snapshot of https://github.com/tldraw/tldraw.git (main, 2026-09-23; blobs > 131072B omitted)
Some checks failed
Deploy analytics worker / Deploy analytics worker to ${{ (github.ref == 'refs/heads/production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Failing after 1s
i18n upload strings / Upload latest strings (push) Failing after 1s
Deploy bemo / Deploy bemo to ${{ (github.ref == 'refs/heads/bemo-production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Failing after 1s
Deploy grafana resources / ${{ github.event_name == 'push' && 'Deploy grafana resources' || 'Validate grafana resources (dry run)' }} (push) Failing after 0s
End to end tests (dotcom) / Check for relevant changes (push) Failing after 0s
End to end tests (examples) / Check for relevant changes (push) Failing after 0s
End to end tests (mermaid readability) / Check for relevant changes (push) Failing after 1s
End to end tests (perf) / Check for relevant changes (push) Failing after 0s
Publish editor extensions / Publish editor extensions (push) Failing after 1s
Publish SDK packages / Remove the publish-packages label (push) Has been skipped
Checks / Tests & checks (push) Has been cancelled
Checks / Build all projects (push) Has been cancelled
Deploy templates / Deploy templates (push) Has been cancelled
End to end tests (dotcom) / End to end tests (dotcom) (push) Has been cancelled
End to end tests (mermaid readability) / End to end tests (mermaid readability) (push) Has been cancelled
End to end tests (perf) / End to end tests (perf) (push) Has been cancelled
Deploy .com / Deploy dotcom to ${{ (github.ref == 'refs/heads/production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Has been cancelled
End to end tests (examples) / End to end tests (examples) (push) Has been cancelled
Publish SDK packages / Publish SDK packages (push) Has been cancelled
Publish SDK packages / Tell user released version (push) Has been cancelled
Publish SDK packages / Publishes code templates to separate repositories (push) Has been cancelled
Publish SDK packages / Deploys the template demos with the new SDK version (push) Has been cancelled
Some checks failed
Deploy analytics worker / Deploy analytics worker to ${{ (github.ref == 'refs/heads/production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Failing after 1s
i18n upload strings / Upload latest strings (push) Failing after 1s
Deploy bemo / Deploy bemo to ${{ (github.ref == 'refs/heads/bemo-production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Failing after 1s
Deploy grafana resources / ${{ github.event_name == 'push' && 'Deploy grafana resources' || 'Validate grafana resources (dry run)' }} (push) Failing after 0s
End to end tests (dotcom) / Check for relevant changes (push) Failing after 0s
End to end tests (examples) / Check for relevant changes (push) Failing after 0s
End to end tests (mermaid readability) / Check for relevant changes (push) Failing after 1s
End to end tests (perf) / Check for relevant changes (push) Failing after 0s
Publish editor extensions / Publish editor extensions (push) Failing after 1s
Publish SDK packages / Remove the publish-packages label (push) Has been skipped
Checks / Tests & checks (push) Has been cancelled
Checks / Build all projects (push) Has been cancelled
Deploy templates / Deploy templates (push) Has been cancelled
End to end tests (dotcom) / End to end tests (dotcom) (push) Has been cancelled
End to end tests (mermaid readability) / End to end tests (mermaid readability) (push) Has been cancelled
End to end tests (perf) / End to end tests (perf) (push) Has been cancelled
Deploy .com / Deploy dotcom to ${{ (github.ref == 'refs/heads/production' && 'production') || (github.ref == 'refs/heads/main' && 'staging') || 'preview' }} (push) Has been cancelled
End to end tests (examples) / End to end tests (examples) (push) Has been cancelled
Publish SDK packages / Publish SDK packages (push) Has been cancelled
Publish SDK packages / Tell user released version (push) Has been cancelled
Publish SDK packages / Publishes code templates to separate repositories (push) Has been cancelled
Publish SDK packages / Deploys the template demos with the new SDK version (push) Has been cancelled
This commit is contained in:
102
internal/scripts/api-check.ts
Executable file
102
internal/scripts/api-check.ts
Executable file
@@ -0,0 +1,102 @@
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
import { join, resolve } from 'path'
|
||||
import { exec } from './lib/exec'
|
||||
import { readFileIfExists } from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { getAllWorkspacePackages } from './lib/workspace'
|
||||
|
||||
const packagesOurTypesCanDependOn = [
|
||||
'@types/lodash.throttle',
|
||||
'@types/lodash.uniq',
|
||||
'@types/lodash.isequal',
|
||||
'@types/lodash.isequalwith',
|
||||
'@types/react',
|
||||
'@types/react-dom',
|
||||
'eventemitter3',
|
||||
'nanoevents',
|
||||
'react',
|
||||
'react-dom',
|
||||
]
|
||||
|
||||
// These packages use subpath exports which require moduleResolution: "bundler".
|
||||
// We pin to the workspace's resolved versions to avoid installing broken releases.
|
||||
const pinnedPackages = [
|
||||
'@tiptap/core',
|
||||
'@tiptap/extension-list',
|
||||
'@tiptap/extension-mention',
|
||||
'@tiptap/pm',
|
||||
'@tiptap/react',
|
||||
'@tiptap/starter-kit',
|
||||
'@tiptap/suggestion',
|
||||
]
|
||||
|
||||
function getPinnedVersions(): string[] {
|
||||
return pinnedPackages.map((pkg) => {
|
||||
const pkgJsonPath = resolve(`./node_modules/${pkg}/package.json`)
|
||||
const { version } = JSON.parse(readFileSync(pkgJsonPath, 'utf8'))
|
||||
return `${pkg}@${version}`
|
||||
})
|
||||
}
|
||||
|
||||
function getTempDeclarationFilename(packageName: string): string {
|
||||
// Keep filenames unique across similarly named packages like `tldraw` and `@tldraw/tldraw`.
|
||||
return (
|
||||
packageName
|
||||
.replace(/^@/, '')
|
||||
.replace(/\//g, '__')
|
||||
.replace(/[^a-zA-Z0-9_.-]/g, '_') + '.d.ts'
|
||||
)
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
async function main() {
|
||||
const tsconfig: any = {
|
||||
compilerOptions: {
|
||||
lib: ['esnext', 'dom'],
|
||||
strict: true,
|
||||
module: 'esnext',
|
||||
moduleResolution: 'bundler',
|
||||
rootDir: '.',
|
||||
paths: {},
|
||||
esModuleInterop: true,
|
||||
},
|
||||
files: [],
|
||||
}
|
||||
|
||||
const tempDir = (await exec('mktemp', ['-d'])).trim()
|
||||
nicelog(`Working in ${tempDir}`)
|
||||
|
||||
const packages = (await getAllWorkspacePackages())
|
||||
.filter(({ packageJson }) => !packageJson.private)
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
|
||||
nicelog(
|
||||
'Checking packages:',
|
||||
packages.map(({ packageJson }) => packageJson.name)
|
||||
)
|
||||
|
||||
for (const { name, relativePath } of packages) {
|
||||
const declarationFilename = getTempDeclarationFilename(name)
|
||||
const dtsFile = await readFileIfExists(join(relativePath, 'api', 'public.d.ts'))
|
||||
if (!dtsFile) {
|
||||
nicelog(`No public.d.ts for ${name}, skipping`)
|
||||
continue
|
||||
}
|
||||
|
||||
writeFileSync(join(tempDir, declarationFilename), dtsFile, 'utf8')
|
||||
tsconfig.compilerOptions.paths[name] = [`./${declarationFilename}`]
|
||||
tsconfig.files.push(`./${declarationFilename}`)
|
||||
}
|
||||
|
||||
nicelog('Checking with tsconfig:', tsconfig)
|
||||
writeFileSync(`${tempDir}/tsconfig.json`, JSON.stringify(tsconfig, null, '\t'), 'utf8')
|
||||
writeFileSync(`${tempDir}/package.json`, JSON.stringify({ dependencies: {} }, null, '\t'), 'utf8')
|
||||
|
||||
await exec('npm', ['install', ...packagesOurTypesCanDependOn, ...getPinnedVersions()], {
|
||||
pwd: tempDir,
|
||||
})
|
||||
await exec(resolve('./node_modules/.bin/tsc'), [], { pwd: tempDir })
|
||||
|
||||
await exec('rm', ['-rf', tempDir])
|
||||
}
|
||||
292
internal/scripts/benchmark-dict-validator.ts
Normal file
292
internal/scripts/benchmark-dict-validator.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
/**
|
||||
* Benchmark script for @tldraw/validate DictValidator
|
||||
*
|
||||
* Usage:
|
||||
* yarn tsx internal/scripts/benchmark-dict-validator.ts
|
||||
*
|
||||
* This script measures the performance of the DictValidator in @tldraw/validate.
|
||||
* It tests both fresh validation and validateUsingKnownGoodVersion scenarios.
|
||||
*/
|
||||
|
||||
import * as T from '../../packages/validate/src/lib/validation'
|
||||
|
||||
const ITERATIONS_PER_RUN = 50_000
|
||||
const TOTAL_RUNS = 200
|
||||
|
||||
interface BenchmarkResult {
|
||||
first100Avg: number
|
||||
last100Avg: number
|
||||
opsPerSecFirst: number
|
||||
opsPerSecLast: number
|
||||
}
|
||||
|
||||
function runBenchmark(name: string, fn: () => void): BenchmarkResult {
|
||||
const times: number[] = []
|
||||
|
||||
for (let run = 0; run < TOTAL_RUNS; run++) {
|
||||
const start = performance.now()
|
||||
for (let i = 0; i < ITERATIONS_PER_RUN; i++) {
|
||||
fn()
|
||||
}
|
||||
times.push(performance.now() - start)
|
||||
}
|
||||
|
||||
const first100 = times.slice(0, 100)
|
||||
const last100 = times.slice(100)
|
||||
const first100Avg = first100.reduce((a, b) => a + b, 0) / first100.length
|
||||
const last100Avg = last100.reduce((a, b) => a + b, 0) / last100.length
|
||||
|
||||
return {
|
||||
first100Avg,
|
||||
last100Avg,
|
||||
opsPerSecFirst: (ITERATIONS_PER_RUN / first100Avg) * 1000,
|
||||
opsPerSecLast: (ITERATIONS_PER_RUN / last100Avg) * 1000,
|
||||
}
|
||||
}
|
||||
|
||||
function formatOps(ops: number): string {
|
||||
if (ops >= 1_000_000) {
|
||||
return `${(ops / 1_000_000).toFixed(2)}M`
|
||||
}
|
||||
if (ops >= 1_000) {
|
||||
return `${(ops / 1_000).toFixed(2)}K`
|
||||
}
|
||||
return ops.toFixed(0)
|
||||
}
|
||||
|
||||
function printResult(name: string, result: BenchmarkResult): void {
|
||||
const warmupChange = ((result.first100Avg / result.last100Avg - 1) * 100).toFixed(1)
|
||||
const warmupDir = Number(warmupChange) > 0 ? 'slower' : 'faster'
|
||||
|
||||
console.log(`--- ${name} ---`)
|
||||
console.log(
|
||||
` First 100 runs: ${result.first100Avg.toFixed(2)}ms (${formatOps(result.opsPerSecFirst)} ops/sec)`
|
||||
)
|
||||
console.log(
|
||||
` Last 100 runs: ${result.last100Avg.toFixed(2)}ms (${formatOps(result.opsPerSecLast)} ops/sec)`
|
||||
)
|
||||
console.log(` JIT stability: First 100 ${Math.abs(Number(warmupChange))}% ${warmupDir}`)
|
||||
console.log('')
|
||||
}
|
||||
|
||||
// Test data for dict validators
|
||||
// Small dict (3 keys) - common case for simple configs
|
||||
const smallDict: Record<string, number> = {
|
||||
width: 100,
|
||||
height: 200,
|
||||
depth: 50,
|
||||
}
|
||||
|
||||
// Medium dict (10 keys) - typical shape props
|
||||
const mediumDict: Record<string, number> = {}
|
||||
for (let i = 0; i < 10; i++) {
|
||||
mediumDict[`key${i}`] = i * 10
|
||||
}
|
||||
|
||||
// Large dict (50 keys) - stress test
|
||||
const largeDict: Record<string, number> = {}
|
||||
for (let i = 0; i < 50; i++) {
|
||||
largeDict[`key${i}`] = i * 10
|
||||
}
|
||||
|
||||
// Line shape points scenario - realistic use case
|
||||
interface LinePoint {
|
||||
id: string
|
||||
index: string
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
const linePoints: Record<string, LinePoint> = {}
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const id = `point${i}`
|
||||
linePoints[id] = { id, index: `a${i}`, x: i * 10, y: i * 20 }
|
||||
}
|
||||
|
||||
// Validators
|
||||
const stringToNumberDict = T.dict(T.string, T.number)
|
||||
const linePointValidator = T.object({
|
||||
id: T.string,
|
||||
index: T.string,
|
||||
x: T.number,
|
||||
y: T.number,
|
||||
})
|
||||
const linePointsDict = T.dict(T.string, linePointValidator)
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('DICT VALIDATOR BENCHMARK')
|
||||
console.log('='.repeat(70))
|
||||
console.log(`Iterations per run: ${ITERATIONS_PER_RUN.toLocaleString()}`)
|
||||
console.log(`Total runs: ${TOTAL_RUNS}`)
|
||||
console.log(`Comparing first 100 runs (warmup) vs last 100 runs (stable)`)
|
||||
console.log('')
|
||||
|
||||
// --- Fresh validation benchmarks ---
|
||||
console.log('='.repeat(70))
|
||||
console.log('FRESH VALIDATION (validate)')
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Small dict fresh validation
|
||||
{
|
||||
const testDicts = [smallDict, { ...smallDict, extra: 999 }, { a: 1, b: 2, c: 3 }]
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Small dict (3 keys)',
|
||||
runBenchmark('SmallDict', () => {
|
||||
stringToNumberDict.validate(testDicts[idx++ % testDicts.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Medium dict fresh validation
|
||||
{
|
||||
const testDicts = [mediumDict, { ...mediumDict }]
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Medium dict (10 keys)',
|
||||
runBenchmark('MediumDict', () => {
|
||||
stringToNumberDict.validate(testDicts[idx++ % testDicts.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Large dict fresh validation
|
||||
{
|
||||
const testDicts = [largeDict, { ...largeDict }]
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Large dict (50 keys)',
|
||||
runBenchmark('LargeDict', () => {
|
||||
stringToNumberDict.validate(testDicts[idx++ % testDicts.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Line points dict (realistic scenario)
|
||||
{
|
||||
const testDicts = [linePoints, { ...linePoints }]
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Line points dict (5 nested objects)',
|
||||
runBenchmark('LinePoints', () => {
|
||||
linePointsDict.validate(testDicts[idx++ % testDicts.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// --- ValidateUsingKnownGoodVersion benchmarks ---
|
||||
console.log('='.repeat(70))
|
||||
console.log('INCREMENTAL VALIDATION (validateUsingKnownGoodVersion)')
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Scenario 1: No changes (should be fast - identity check)
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(smallDict)
|
||||
printResult(
|
||||
'Small dict - no changes (same reference)',
|
||||
runBenchmark('SmallDictNoChange', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, knownGood)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Scenario 2: Same values, different reference
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(smallDict)
|
||||
const newValue = { ...smallDict } // Same values, new object
|
||||
printResult(
|
||||
'Small dict - same values, different reference',
|
||||
runBenchmark('SmallDictSameValues', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, newValue)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Scenario 3: One value changed
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(smallDict)
|
||||
const changed = { ...smallDict, width: 150 }
|
||||
printResult(
|
||||
'Small dict - one value changed',
|
||||
runBenchmark('SmallDictOneChange', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, changed)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Scenario 4: Key added
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(smallDict)
|
||||
const added = { ...smallDict, newKey: 999 }
|
||||
printResult(
|
||||
'Small dict - key added',
|
||||
runBenchmark('SmallDictKeyAdded', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, added)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Scenario 5: Key removed
|
||||
{
|
||||
const { width: _, ...rest } = smallDict
|
||||
const knownGood = stringToNumberDict.validate(smallDict)
|
||||
printResult(
|
||||
'Small dict - key removed',
|
||||
runBenchmark('SmallDictKeyRemoved', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, rest)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Medium dict scenarios
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(mediumDict)
|
||||
const newValue = { ...mediumDict }
|
||||
printResult(
|
||||
'Medium dict - same values, different reference',
|
||||
runBenchmark('MediumDictSameValues', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, newValue)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
{
|
||||
const knownGood = stringToNumberDict.validate(mediumDict)
|
||||
const changed = { ...mediumDict, key5: 999 }
|
||||
printResult(
|
||||
'Medium dict - one value changed',
|
||||
runBenchmark('MediumDictOneChange', () => {
|
||||
stringToNumberDict.validateUsingKnownGoodVersion(knownGood, changed)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Line points incremental
|
||||
{
|
||||
const knownGood = linePointsDict.validate(linePoints)
|
||||
const newValue = { ...linePoints }
|
||||
printResult(
|
||||
'Line points - same values, different reference',
|
||||
runBenchmark('LinePointsSameValues', () => {
|
||||
linePointsDict.validateUsingKnownGoodVersion(knownGood, newValue)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
{
|
||||
const knownGood = linePointsDict.validate(linePoints)
|
||||
const changed = {
|
||||
...linePoints,
|
||||
point2: { ...linePoints.point2, x: 999 },
|
||||
}
|
||||
printResult(
|
||||
'Line points - one point changed',
|
||||
runBenchmark('LinePointsOneChange', () => {
|
||||
linePointsDict.validateUsingKnownGoodVersion(knownGood, changed)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
379
internal/scripts/benchmark-validators.ts
Normal file
379
internal/scripts/benchmark-validators.ts
Normal file
@@ -0,0 +1,379 @@
|
||||
/**
|
||||
* Benchmark script for @tldraw/validate number validators
|
||||
*
|
||||
* Usage:
|
||||
* yarn tsx internal/scripts/benchmark-validators.ts
|
||||
*
|
||||
* This script measures the performance of the numeric validators in @tldraw/validate.
|
||||
* It runs multiple iterations to get stable measurements after JIT warmup.
|
||||
*/
|
||||
|
||||
import * as T from '../../packages/validate/src/lib/validation'
|
||||
|
||||
const ITERATIONS_PER_RUN = 100_000
|
||||
const TOTAL_RUNS = 200
|
||||
|
||||
interface BenchmarkResult {
|
||||
first100Avg: number
|
||||
last100Avg: number
|
||||
opsPerSecFirst: number
|
||||
opsPerSecLast: number
|
||||
}
|
||||
|
||||
function runBenchmark(name: string, fn: () => void): BenchmarkResult {
|
||||
const times: number[] = []
|
||||
|
||||
for (let run = 0; run < TOTAL_RUNS; run++) {
|
||||
const start = performance.now()
|
||||
for (let i = 0; i < ITERATIONS_PER_RUN; i++) {
|
||||
fn()
|
||||
}
|
||||
times.push(performance.now() - start)
|
||||
}
|
||||
|
||||
const first100 = times.slice(0, 100)
|
||||
const last100 = times.slice(100)
|
||||
const first100Avg = first100.reduce((a, b) => a + b, 0) / first100.length
|
||||
const last100Avg = last100.reduce((a, b) => a + b, 0) / last100.length
|
||||
|
||||
return {
|
||||
first100Avg,
|
||||
last100Avg,
|
||||
opsPerSecFirst: (ITERATIONS_PER_RUN / first100Avg) * 1000,
|
||||
opsPerSecLast: (ITERATIONS_PER_RUN / last100Avg) * 1000,
|
||||
}
|
||||
}
|
||||
|
||||
function formatOps(ops: number): string {
|
||||
if (ops >= 1_000_000) {
|
||||
return `${(ops / 1_000_000).toFixed(2)}M`
|
||||
}
|
||||
if (ops >= 1_000) {
|
||||
return `${(ops / 1_000).toFixed(2)}K`
|
||||
}
|
||||
return ops.toFixed(0)
|
||||
}
|
||||
|
||||
function printResult(name: string, result: BenchmarkResult): void {
|
||||
const warmupChange = ((result.first100Avg / result.last100Avg - 1) * 100).toFixed(1)
|
||||
const warmupDir = Number(warmupChange) > 0 ? 'slower' : 'faster'
|
||||
|
||||
console.log(`--- ${name} ---`)
|
||||
console.log(
|
||||
` First 100 runs: ${result.first100Avg.toFixed(2)}ms (${formatOps(result.opsPerSecFirst)} ops/sec)`
|
||||
)
|
||||
console.log(
|
||||
` Last 100 runs: ${result.last100Avg.toFixed(2)}ms (${formatOps(result.opsPerSecLast)} ops/sec)`
|
||||
)
|
||||
console.log(` JIT stability: First 100 ${Math.abs(Number(warmupChange))}% ${warmupDir}`)
|
||||
console.log('')
|
||||
}
|
||||
|
||||
// Test values
|
||||
const testIntegers = [0, 1, 42, -100, 1000000]
|
||||
const testFloats = [0.5, 3.14, -2.5, 100.001, 0.000001]
|
||||
const testPositive = [0, 1, 42, 0.5, 1000000]
|
||||
const testNonZero = [1, 42, 0.5, 100.001, 1000000]
|
||||
const testUnitInterval = [0, 0.25, 0.5, 0.75, 1]
|
||||
const testNonZeroFinite = [1, -1, 0.5, -0.5, 100.001]
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('VALIDATOR BENCHMARK')
|
||||
console.log('='.repeat(70))
|
||||
console.log(`Iterations per run: ${ITERATIONS_PER_RUN.toLocaleString()}`)
|
||||
console.log(`Total runs: ${TOTAL_RUNS}`)
|
||||
console.log(`Comparing first 100 runs (warmup) vs last 100 runs (stable)`)
|
||||
console.log('')
|
||||
|
||||
// T.number
|
||||
{
|
||||
const vals = [...testIntegers, ...testFloats]
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.number',
|
||||
runBenchmark('T.number', () => {
|
||||
T.number.validate(vals[idx++ % vals.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.positiveNumber
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.positiveNumber',
|
||||
runBenchmark('T.positiveNumber', () => {
|
||||
T.positiveNumber.validate(testPositive[idx++ % testPositive.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.nonZeroNumber
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.nonZeroNumber',
|
||||
runBenchmark('T.nonZeroNumber', () => {
|
||||
T.nonZeroNumber.validate(testNonZero[idx++ % testNonZero.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.integer
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.integer',
|
||||
runBenchmark('T.integer', () => {
|
||||
T.integer.validate(testIntegers[idx++ % testIntegers.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.positiveInteger
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.positiveInteger',
|
||||
runBenchmark('T.positiveInteger', () => {
|
||||
T.positiveInteger.validate(testPositive.filter(Number.isInteger)[idx++ % 3])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.nonZeroFiniteNumber
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.nonZeroFiniteNumber',
|
||||
runBenchmark('T.nonZeroFiniteNumber', () => {
|
||||
T.nonZeroFiniteNumber.validate(testNonZeroFinite[idx++ % testNonZeroFinite.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// T.unitInterval
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'T.unitInterval',
|
||||
runBenchmark('T.unitInterval', () => {
|
||||
T.unitInterval.validate(testUnitInterval[idx++ % testUnitInterval.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Union validator benchmarks
|
||||
console.log('='.repeat(70))
|
||||
console.log('UNION VALIDATOR')
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
const circleSchema = T.object({
|
||||
type: T.literal('circle'),
|
||||
radius: T.number,
|
||||
})
|
||||
const squareSchema = T.object({
|
||||
type: T.literal('square'),
|
||||
size: T.number,
|
||||
})
|
||||
const triangleSchema = T.object({
|
||||
type: T.literal('triangle'),
|
||||
base: T.number,
|
||||
height: T.number,
|
||||
})
|
||||
|
||||
const shapeUnion = T.union('type', {
|
||||
circle: circleSchema,
|
||||
square: squareSchema,
|
||||
triangle: triangleSchema,
|
||||
})
|
||||
|
||||
const testShapes = [
|
||||
{ type: 'circle', radius: 10 },
|
||||
{ type: 'square', size: 20 },
|
||||
{ type: 'triangle', base: 15, height: 25 },
|
||||
{ type: 'circle', radius: 5 },
|
||||
{ type: 'square', size: 30 },
|
||||
]
|
||||
|
||||
// Fresh validation
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Union fresh validation',
|
||||
runBenchmark('UnionFresh', () => {
|
||||
shapeUnion.validate(testShapes[idx++ % testShapes.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// validateUsingKnownGoodVersion - same reference (tests Object.is fast path)
|
||||
{
|
||||
const knownGood = shapeUnion.validate({ type: 'circle', radius: 10 })
|
||||
printResult(
|
||||
'Union same reference (Object.is fast path)',
|
||||
runBenchmark('UnionSameRef', () => {
|
||||
shapeUnion.validateUsingKnownGoodVersion(knownGood, knownGood)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// validateUsingKnownGoodVersion - same type, value changed
|
||||
{
|
||||
const knownGood = shapeUnion.validate({ type: 'circle', radius: 10 })
|
||||
const changed = { type: 'circle', radius: 20 }
|
||||
printResult(
|
||||
'Union same type, value changed',
|
||||
runBenchmark('UnionSameType', () => {
|
||||
shapeUnion.validateUsingKnownGoodVersion(knownGood, changed)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// validateUsingKnownGoodVersion - different type
|
||||
{
|
||||
const knownGood = shapeUnion.validate({ type: 'circle', radius: 10 })
|
||||
const different = { type: 'square', size: 20 }
|
||||
printResult(
|
||||
'Union different type',
|
||||
runBenchmark('UnionDiffType', () => {
|
||||
shapeUnion.validateUsingKnownGoodVersion(knownGood, different)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Number union (tests optimized NaN check)
|
||||
const numberUnionSchema = T.numberUnion('version', {
|
||||
1: T.object({ version: T.literal(1), data: T.string }),
|
||||
2: T.object({ version: T.literal(2), data: T.string, extra: T.number }),
|
||||
})
|
||||
|
||||
const testNumberUnions = [
|
||||
{ version: 1, data: 'hello' },
|
||||
{ version: 2, data: 'world', extra: 42 },
|
||||
{ version: 1, data: 'foo' },
|
||||
{ version: 2, data: 'bar', extra: 100 },
|
||||
]
|
||||
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'NumberUnion fresh validation',
|
||||
runBenchmark('NumberUnionFresh', () => {
|
||||
numberUnionSchema.validate(testNumberUnions[idx++ % testNumberUnions.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Realistic scenario: Point validation
|
||||
console.log('='.repeat(70))
|
||||
console.log('REALISTIC SCENARIO: Point validation')
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
const testPoints: unknown[] = []
|
||||
for (let i = 0; i < 10000; i++) {
|
||||
testPoints.push({ x: Math.random() * 1000, y: Math.random() * 1000 })
|
||||
}
|
||||
|
||||
const pointValidator = T.object({
|
||||
x: T.number,
|
||||
y: T.number,
|
||||
})
|
||||
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Point {x, y}',
|
||||
runBenchmark('Point', () => {
|
||||
pointValidator.validate(testPoints[idx++ % testPoints.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Draw shape props scenario
|
||||
const testDrawProps: unknown[] = []
|
||||
for (let i = 0; i < 1000; i++) {
|
||||
testDrawProps.push({
|
||||
scaleX: 1 + Math.random() * 0.5,
|
||||
scaleY: 1 + Math.random() * 0.5,
|
||||
opacity: Math.random(),
|
||||
})
|
||||
}
|
||||
|
||||
const drawPropsValidator = T.object({
|
||||
scaleX: T.nonZeroFiniteNumber,
|
||||
scaleY: T.nonZeroFiniteNumber,
|
||||
opacity: T.unitInterval,
|
||||
})
|
||||
|
||||
{
|
||||
let idx = 0
|
||||
printResult(
|
||||
'DrawProps {scaleX, scaleY, opacity}',
|
||||
runBenchmark('DrawProps', () => {
|
||||
drawPropsValidator.validate(testDrawProps[idx++ % testDrawProps.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Factory method caching benchmarks
|
||||
console.log('='.repeat(70))
|
||||
console.log('FACTORY METHOD CACHING')
|
||||
console.log('='.repeat(70))
|
||||
console.log('')
|
||||
|
||||
// Test repeated .nullable() calls on same validator
|
||||
{
|
||||
printResult(
|
||||
'T.string.nullable() repeated calls',
|
||||
runBenchmark('NullableRepeated', () => {
|
||||
T.string.nullable()
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Test repeated .optional() calls on same validator
|
||||
{
|
||||
printResult(
|
||||
'T.string.optional() repeated calls',
|
||||
runBenchmark('OptionalRepeated', () => {
|
||||
T.string.optional()
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Test chained .nullable().optional() calls
|
||||
{
|
||||
printResult(
|
||||
'T.number.nullable().optional() chain',
|
||||
runBenchmark('NullableOptionalChain', () => {
|
||||
T.number.nullable().optional()
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Realistic: validate with nullable field
|
||||
{
|
||||
const nullableStringValidator = T.string.nullable()
|
||||
const testValues = ['hello', null, 'world', null, 'test']
|
||||
let idx = 0
|
||||
printResult(
|
||||
'Validate with cached nullable',
|
||||
runBenchmark('ValidateCachedNullable', () => {
|
||||
nullableStringValidator.validate(testValues[idx++ % testValues.length])
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
console.log('='.repeat(70))
|
||||
62
internal/scripts/build-api.ts
Normal file
62
internal/scripts/build-api.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import path from 'path'
|
||||
import { glob } from 'glob'
|
||||
import isCI from 'is-ci'
|
||||
import { rimraf } from 'rimraf'
|
||||
import { exec } from './lib/exec'
|
||||
import { sortUnions } from './lib/sort-unions'
|
||||
|
||||
async function buildApi(sourcePackageDir: string) {
|
||||
// this depends on `build-types` being run first, but we'll rely on turbo to
|
||||
// make that happen.
|
||||
|
||||
const relativeSourcePackageDir = path.relative(path.resolve(__dirname, '../..'), sourcePackageDir)
|
||||
|
||||
// we need to copy the .tsbuild dir to another location so we can edit it without busting the cache
|
||||
await rimraf(path.join(sourcePackageDir, '.tsbuild-api'))
|
||||
await exec('cp', ['-R', './.tsbuild', './.tsbuild-api'], { pwd: sourcePackageDir })
|
||||
|
||||
sortUnions(path.join(sourcePackageDir, '.tsbuild-api'))
|
||||
|
||||
// clear api-extractor build files
|
||||
rimraf.sync(glob.sync(path.join(sourcePackageDir, 'api')))
|
||||
// extract public api
|
||||
try {
|
||||
await exec('yarn', ['run', '-T', 'api-extractor', 'run', isCI ? null : '--local'], {
|
||||
pwd: sourcePackageDir,
|
||||
processStderrLine: (line) => {
|
||||
process.stderr.write(`${line}\n`)
|
||||
|
||||
if (process.env.PRINT_GITHUB_ANNOTATIONS) {
|
||||
const errorMatch = line.match(
|
||||
// eslint-disable-next-line no-control-regex
|
||||
/^(?:\u001b\[33m)?(Warning|Error): (.+?):(\d+):(\d+) - \((.*)\) (.*)(?:\u001b\[39m)?$/
|
||||
)
|
||||
if (errorMatch) {
|
||||
const [, errorType, file, line, column, messageTitle, message] = errorMatch
|
||||
|
||||
process.stdout.write(
|
||||
`::${errorType.toLowerCase()} file=${relativeSourcePackageDir}/${file},line=${line},col=${column},title=${messageTitle}::${message}}\n`
|
||||
)
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
} catch (e) {
|
||||
// get git diff
|
||||
await exec('git', [
|
||||
'diff',
|
||||
'--no-color',
|
||||
'--no-ext-diff',
|
||||
'--no-index',
|
||||
'api-report.api.md',
|
||||
'api/temp/api-report.api.md',
|
||||
])
|
||||
|
||||
// @ts-expect-error
|
||||
console.error(e.message)
|
||||
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
buildApi(process.cwd())
|
||||
137
internal/scripts/build-package.ts
Normal file
137
internal/scripts/build-package.ts
Normal file
@@ -0,0 +1,137 @@
|
||||
import { copyFileSync, existsSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { pathToFileURL } from 'url'
|
||||
import { build } from 'esbuild'
|
||||
import { glob } from 'glob'
|
||||
import kleur from 'kleur'
|
||||
import rimraf from 'rimraf'
|
||||
import { addJsExtensions } from './lib/add-extensions'
|
||||
import { readJsonIfExists } from './lib/file'
|
||||
|
||||
interface LibraryInfo {
|
||||
name: string
|
||||
version: string
|
||||
moduleSystem: 'esm' | 'cjs'
|
||||
}
|
||||
|
||||
/** Prepares the package for publishing. the tarball in case it will be written to disk. */
|
||||
async function buildPackage({ sourcePackageDir }: { sourcePackageDir: string }) {
|
||||
// this depends on `build-types` being run first, but we'll rely on turbo to
|
||||
// make that happen.
|
||||
|
||||
if (!existsSync(path.join(sourcePackageDir, 'src/index.ts'))) {
|
||||
throw new Error(`No src/index.ts file found in '${sourcePackageDir}'!`)
|
||||
}
|
||||
|
||||
rimraf.sync(path.join(sourcePackageDir, 'dist'))
|
||||
|
||||
// then copy over the source .ts files
|
||||
const sourceFiles = glob
|
||||
.sync(path.join(sourcePackageDir, 'src/**/*.ts?(x)'))
|
||||
// ignore test files
|
||||
.filter(
|
||||
(file) =>
|
||||
!(file.includes('/__tests__/') || file.includes('.test.') || file.includes('/test/'))
|
||||
)
|
||||
|
||||
const packageJson = await readJsonIfExists(path.join(sourcePackageDir, 'package.json'))
|
||||
if (!packageJson) {
|
||||
throw new Error('No package.json found in source package dir')
|
||||
}
|
||||
|
||||
const packageName = packageJson.name
|
||||
if (typeof packageName !== 'string') throw new Error('package.json name is not a string')
|
||||
const packageVersion = packageJson.version
|
||||
if (typeof packageVersion !== 'string') throw new Error('package.json version is not a string')
|
||||
|
||||
// build CommonJS files to /dist-cjs
|
||||
await buildLibrary({
|
||||
sourceFiles,
|
||||
sourcePackageDir,
|
||||
info: {
|
||||
name: packageName,
|
||||
version: packageVersion,
|
||||
moduleSystem: 'cjs',
|
||||
},
|
||||
})
|
||||
|
||||
// build ES Module files to /dist-esm
|
||||
await buildLibrary({
|
||||
sourceFiles,
|
||||
sourcePackageDir,
|
||||
info: {
|
||||
name: packageName,
|
||||
version: packageVersion,
|
||||
moduleSystem: 'esm',
|
||||
},
|
||||
})
|
||||
|
||||
copyFileSync(
|
||||
path.join(sourcePackageDir, `api/public.d.ts`),
|
||||
path.join(sourcePackageDir, 'dist-cjs/index.d.ts')
|
||||
)
|
||||
|
||||
copyFileSync(
|
||||
path.join(sourcePackageDir, `api/public.d.ts`),
|
||||
path.join(sourcePackageDir, 'dist-esm/index.d.mts')
|
||||
)
|
||||
}
|
||||
|
||||
/** This uses esbuild to build the esm version of the package */
|
||||
async function buildLibrary({
|
||||
sourceFiles,
|
||||
sourcePackageDir,
|
||||
info,
|
||||
}: {
|
||||
sourceFiles: string[]
|
||||
sourcePackageDir: string
|
||||
info: LibraryInfo
|
||||
}) {
|
||||
const dirName = `dist-${info.moduleSystem}`
|
||||
const outdir = path.join(sourcePackageDir, dirName)
|
||||
rimraf.sync(outdir)
|
||||
|
||||
const define: Record<string, string> = {}
|
||||
if (process.env.TLDRAW_BEMO_URL) {
|
||||
define['process.env.TLDRAW_BEMO_URL'] = JSON.stringify(process.env.TLDRAW_BEMO_URL)
|
||||
}
|
||||
|
||||
// we use `globalThis` instead of `process.env` because although it'll get compiled away in
|
||||
// library code, we use the un-compiled versions in our own apps and not every environment
|
||||
// supports process.env.
|
||||
define['globalThis.TLDRAW_LIBRARY_NAME'] = JSON.stringify(info.name)
|
||||
define['globalThis.TLDRAW_LIBRARY_VERSION'] = JSON.stringify(info.version)
|
||||
define['globalThis.TLDRAW_LIBRARY_MODULES'] = JSON.stringify(info.moduleSystem)
|
||||
define['globalThis.TLDRAW_LIBRARY_IS_BUILD'] = JSON.stringify(true)
|
||||
|
||||
const res = await build({
|
||||
entryPoints: sourceFiles,
|
||||
outdir,
|
||||
format: info.moduleSystem,
|
||||
outExtension: info.moduleSystem === 'esm' ? { '.js': '.mjs' } : undefined,
|
||||
bundle: false,
|
||||
platform: 'neutral',
|
||||
sourcemap: true,
|
||||
target: 'es2022',
|
||||
define,
|
||||
})
|
||||
|
||||
if (info.moduleSystem === 'esm') {
|
||||
addJsExtensions(path.join(sourcePackageDir, 'dist-esm'))
|
||||
}
|
||||
|
||||
if (res.errors.length) {
|
||||
console.error(kleur.red('Build failed with errors:'))
|
||||
console.error(res.errors)
|
||||
throw new Error('esm build failed')
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
// module was called directly
|
||||
;(async () => {
|
||||
await buildPackage({
|
||||
sourcePackageDir: process.cwd(),
|
||||
})
|
||||
})()
|
||||
}
|
||||
23
internal/scripts/bump-versions.ts
Normal file
23
internal/scripts/bump-versions.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import { exec } from './lib/exec'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { getLatestTldrawVersionFromNpm, setAllVersions } from './lib/publishing'
|
||||
|
||||
async function main() {
|
||||
const latestVersion = await getLatestTldrawVersionFromNpm()
|
||||
nicelog('Bumping all package versions to', latestVersion.format())
|
||||
|
||||
await setAllVersions(latestVersion.format(), { stageChanges: true })
|
||||
|
||||
// Commit the changes with [skip ci] to prevent triggering other workflows
|
||||
await exec('git', ['commit', '-m', `Bump versions to ${latestVersion.format()} [skip ci]`])
|
||||
|
||||
// Push the changes
|
||||
await exec('git', ['push'])
|
||||
|
||||
nicelog('Successfully bumped versions and pushed changes')
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
nicelog('Error:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
237
internal/scripts/check-circular-deps.ts
Normal file
237
internal/scripts/check-circular-deps.ts
Normal file
@@ -0,0 +1,237 @@
|
||||
import { execFile } from 'child_process'
|
||||
import { access, readdir } from 'fs/promises'
|
||||
import path from 'path'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
const packageEntry = 'src/index.ts'
|
||||
|
||||
async function exists(filePath: string) {
|
||||
try {
|
||||
await access(filePath)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function getPackageEntries() {
|
||||
const packagesDir = path.join(REPO_ROOT, 'packages')
|
||||
const packageNames = (await readdir(packagesDir)).sort()
|
||||
const entries: Record<string, string> = {}
|
||||
|
||||
for (const packageName of packageNames) {
|
||||
const entryPath = path.join(packagesDir, packageName, packageEntry)
|
||||
if (await exists(entryPath)) {
|
||||
entries[packageName] = entryPath
|
||||
}
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
async function getIncludePatternSets(packageName: string) {
|
||||
const fullPackagePatternSet = [[`**/packages/${packageName}/**/*.{js,jsx,ts,tsx}`]]
|
||||
if (packageName !== 'tldraw') return fullPackagePatternSet
|
||||
|
||||
// NOTE:
|
||||
// `vite-plugin-circular-dependency` crashes with "Invalid array length" on tldraw when
|
||||
// `ui/components` and `ui/context` are scanned together in a single build graph.
|
||||
// To keep checks reliable and fast, we build tldraw in a few dynamic batches:
|
||||
// - all non-ui scopes together
|
||||
// - ui/components alone
|
||||
// - ui/context alone
|
||||
// - remaining ui scopes together
|
||||
// These batches are discovered from the filesystem to avoid brittle hardcoded scope lists.
|
||||
const tldrawLibDir = path.join(REPO_ROOT, 'packages', 'tldraw', 'src', 'lib')
|
||||
const tldrawEntries = (await readdir(tldrawLibDir, { withFileTypes: true })).sort((a, b) =>
|
||||
a.name.localeCompare(b.name)
|
||||
)
|
||||
|
||||
const nonUiPatterns: string[] = []
|
||||
let uiComponentsPattern: string | null = null
|
||||
let uiContextPattern: string | null = null
|
||||
const uiOtherPatterns: string[] = []
|
||||
|
||||
for (const entry of tldrawEntries.filter((entry) => !entry.name.startsWith('.'))) {
|
||||
if (!entry.isDirectory()) {
|
||||
if (/\.(ts|tsx|js|jsx)$/.test(entry.name)) {
|
||||
nonUiPatterns.push(`**/packages/tldraw/src/lib/${entry.name}`)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (entry.name !== 'ui') {
|
||||
nonUiPatterns.push(`**/packages/tldraw/src/lib/${entry.name}/**/*.{js,jsx,ts,tsx}`)
|
||||
continue
|
||||
}
|
||||
|
||||
const tldrawUiDir = path.join(tldrawLibDir, 'ui')
|
||||
const tldrawUiEntries = (await readdir(tldrawUiDir, { withFileTypes: true })).sort((a, b) =>
|
||||
a.name.localeCompare(b.name)
|
||||
)
|
||||
for (const uiEntry of tldrawUiEntries.filter((uiEntry) => !uiEntry.name.startsWith('.'))) {
|
||||
const uiPattern = uiEntry.isDirectory()
|
||||
? `**/packages/tldraw/src/lib/ui/${uiEntry.name}/**/*.{js,jsx,ts,tsx}`
|
||||
: /\.(ts|tsx|js|jsx)$/.test(uiEntry.name)
|
||||
? `**/packages/tldraw/src/lib/ui/${uiEntry.name}`
|
||||
: null
|
||||
|
||||
if (!uiPattern) continue
|
||||
if (uiEntry.name === 'components') {
|
||||
uiComponentsPattern = uiPattern
|
||||
} else if (uiEntry.name === 'context') {
|
||||
uiContextPattern = uiPattern
|
||||
} else {
|
||||
uiOtherPatterns.push(uiPattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const includePatternSets = [
|
||||
['**/packages/tldraw/src/index.ts', ...nonUiPatterns],
|
||||
...(uiComponentsPattern ? [['**/packages/tldraw/src/index.ts', uiComponentsPattern]] : []),
|
||||
...(uiContextPattern ? [['**/packages/tldraw/src/index.ts', uiContextPattern]] : []),
|
||||
...(uiOtherPatterns.length > 0
|
||||
? [['**/packages/tldraw/src/index.ts', ...uiOtherPatterns]]
|
||||
: []),
|
||||
]
|
||||
|
||||
return includePatternSets
|
||||
}
|
||||
|
||||
async function checkPackageForCircularDependencies(packageName: string, packageEntryPath: string) {
|
||||
const includePatternSets = await getIncludePatternSets(packageName)
|
||||
|
||||
const script = `
|
||||
import { build } from 'vite'
|
||||
import circularDependencyChecker from 'vite-plugin-circular-dependency'
|
||||
import { mkdirSync, readFileSync, rmSync } from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
const packageName = process.argv[1]
|
||||
const packageEntryPath = process.argv[2]
|
||||
const includePatternSets = JSON.parse(process.argv[3])
|
||||
const maxConcurrency = packageName === 'tldraw' ? 2 : 1
|
||||
|
||||
async function checkIncludeSet(includePatterns) {
|
||||
const outputFilePath = path.join(
|
||||
'.lazy',
|
||||
'circle-deps',
|
||||
packageName + '-' + Date.now() + '-' + Math.random().toString(36).slice(2) + '.json'
|
||||
)
|
||||
mkdirSync(path.dirname(outputFilePath), { recursive: true })
|
||||
|
||||
let cycleReport = {}
|
||||
try {
|
||||
await build({
|
||||
root: ${JSON.stringify(REPO_ROOT)},
|
||||
logLevel: 'error',
|
||||
plugins: [
|
||||
circularDependencyChecker({
|
||||
circleImportThrowErr: false,
|
||||
outputFilePath,
|
||||
include: includePatterns,
|
||||
}),
|
||||
],
|
||||
build: {
|
||||
write: false,
|
||||
emptyOutDir: false,
|
||||
minify: false,
|
||||
target: 'es2022',
|
||||
rollupOptions: {
|
||||
input: {
|
||||
[packageName]: packageEntryPath,
|
||||
},
|
||||
onwarn(warning, warn) {
|
||||
if (warning.code === 'MODULE_LEVEL_DIRECTIVE') return
|
||||
warn(warning)
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
cycleReport = JSON.parse(readFileSync(outputFilePath, 'utf8'))
|
||||
} finally {
|
||||
rmSync(outputFilePath, { force: true })
|
||||
}
|
||||
|
||||
const cycleSignatures = Object.values(cycleReport)
|
||||
.flat()
|
||||
.map((chain) => chain.join(' -> '))
|
||||
|
||||
if (cycleSignatures.length > 0) {
|
||||
console.error(
|
||||
'Found circular dependencies:\\n' +
|
||||
cycleSignatures.map((signature) => ' ' + signature).join('\\n')
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
const queue = [...includePatternSets]
|
||||
const workers = Array.from({ length: Math.min(maxConcurrency, queue.length) }, async () => {
|
||||
while (queue.length > 0) {
|
||||
const includePatterns = queue.shift()
|
||||
if (!includePatterns) return
|
||||
await checkIncludeSet(includePatterns)
|
||||
}
|
||||
})
|
||||
|
||||
await Promise.all(workers)
|
||||
`
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const child = execFile(
|
||||
'node',
|
||||
[
|
||||
'--input-type=module',
|
||||
'-e',
|
||||
script,
|
||||
packageName,
|
||||
packageEntryPath,
|
||||
JSON.stringify(includePatternSets),
|
||||
],
|
||||
{ cwd: REPO_ROOT },
|
||||
(error) => {
|
||||
if (error) reject(error)
|
||||
else resolve()
|
||||
}
|
||||
)
|
||||
|
||||
child.stdout?.on('data', (chunk) => process.stdout.write(chunk))
|
||||
child.stderr?.on('data', (chunk) => process.stderr.write(chunk))
|
||||
})
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const entries = await getPackageEntries()
|
||||
const packageNames = Object.keys(entries)
|
||||
nicelog(`Checking for circular dependencies in packages (${packageNames.length}):`)
|
||||
nicelog(packageNames.join(', '))
|
||||
|
||||
const packageCheckResults = await Promise.allSettled(
|
||||
packageNames.map(async (packageName) => {
|
||||
const packageEntryPath = entries[packageName]
|
||||
nicelog(`Checking ${packageName}...`)
|
||||
await checkPackageForCircularDependencies(packageName, packageEntryPath)
|
||||
})
|
||||
)
|
||||
|
||||
const failedPackages = packageCheckResults.flatMap((result, index) =>
|
||||
result.status === 'rejected' ? [packageNames[index]] : []
|
||||
)
|
||||
|
||||
if (failedPackages.length > 0) {
|
||||
console.error(
|
||||
`Circular dependency checks failed for ${failedPackages.length} package(s): ${failedPackages.join(', ')}`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
nicelog(`No circular dependencies detected across ${packageNames.length} packages.`)
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
618
internal/scripts/check-packages.ts
Normal file
618
internal/scripts/check-packages.ts
Normal file
@@ -0,0 +1,618 @@
|
||||
import { existsSync } from 'fs'
|
||||
import path, { join, relative } from 'path'
|
||||
import kleur from 'kleur'
|
||||
import {
|
||||
REPO_ROOT,
|
||||
readFileIfExists,
|
||||
readJsonIfExists,
|
||||
writeCodeFile,
|
||||
writeJsonFile,
|
||||
writeStringFile,
|
||||
} from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { PRODUCT_CONFIG_KEY } from './lib/types'
|
||||
import { Package, getAllWorkspacePackages, getRootPackage } from './lib/workspace'
|
||||
|
||||
const packagesWithoutTSConfigs: ReadonlySet<string> = new Set(['config'])
|
||||
|
||||
// all packages should have these scripts
|
||||
const expectedPackageJsonScriptsForAll = {
|
||||
lint: tsScript('lint.ts'),
|
||||
}
|
||||
|
||||
const expectedTestScripts = {
|
||||
test: () => 'yarn run -T vitest --passWithNoTests',
|
||||
'test-ci': () => 'yarn run -T vitest run --passWithNoTests',
|
||||
'test-coverage': () => 'yarn run -T vitest run --coverage --passWithNoTests',
|
||||
}
|
||||
|
||||
// packages (in packages/) should have these scripts
|
||||
const expectedPackageJsonScriptsForLibraries = {
|
||||
...expectedPackageJsonScriptsForAll,
|
||||
...expectedTestScripts,
|
||||
}
|
||||
|
||||
const expectedPackageJsonScriptsForApps = {
|
||||
...expectedPackageJsonScriptsForAll,
|
||||
...expectedTestScripts,
|
||||
}
|
||||
|
||||
// published packages should have these scripts
|
||||
const expectedPackageJsonScriptsForPublishedLibraries = {
|
||||
...expectedPackageJsonScriptsForLibraries,
|
||||
build: tsScript('build-package.ts'),
|
||||
'build-api': tsScript('build-api.ts'),
|
||||
prepack: tsScript('prepack.ts'),
|
||||
postpack: (packageDir: string) => scriptPath(packageDir, 'postpack.sh'),
|
||||
'pack-tarball': () => 'yarn pack',
|
||||
}
|
||||
|
||||
// individual packages can have different scripts than the above if needed
|
||||
const packageJsonScriptExceptions: Record<string, Record<string, () => string | undefined>> = {
|
||||
config: {
|
||||
lint: () => undefined,
|
||||
},
|
||||
tsconfig: {
|
||||
lint: () => undefined,
|
||||
},
|
||||
'@tldraw/monorepo': {
|
||||
lint: () => undefined,
|
||||
},
|
||||
'@tldraw/assets': {
|
||||
test: () => undefined,
|
||||
'test-ci': () => undefined,
|
||||
build: () => undefined,
|
||||
'build-api': () => undefined,
|
||||
prepack: () => undefined,
|
||||
postpack: () => undefined,
|
||||
},
|
||||
'create-tldraw': {
|
||||
build: () => './scripts/build.sh',
|
||||
'build-api': () => undefined,
|
||||
prepack: () => 'yarn build',
|
||||
postpack: () => undefined,
|
||||
},
|
||||
}
|
||||
|
||||
async function checkPackageJsonScripts({
|
||||
packages,
|
||||
fix,
|
||||
}: {
|
||||
packages: Package[]
|
||||
fix: boolean
|
||||
}): Promise<boolean> {
|
||||
const needsFix = new Set()
|
||||
|
||||
let errorCount = 0
|
||||
for (const { path: packageDir, relativePath, packageJson, name } of packages) {
|
||||
if (!packageJson.scripts) {
|
||||
packageJson.scripts = {}
|
||||
}
|
||||
const packageScripts = packageJson.scripts
|
||||
|
||||
let expected = relativePath.startsWith('packages/')
|
||||
? packageJson.private
|
||||
? expectedPackageJsonScriptsForLibraries
|
||||
: expectedPackageJsonScriptsForPublishedLibraries
|
||||
: relativePath.startsWith('apps/')
|
||||
? expectedPackageJsonScriptsForApps
|
||||
: expectedPackageJsonScriptsForAll
|
||||
|
||||
if (packageJsonScriptExceptions[name]) {
|
||||
expected = {
|
||||
...expected,
|
||||
...packageJsonScriptExceptions[name],
|
||||
}
|
||||
}
|
||||
|
||||
for (const [scriptName, getExpectedScript] of Object.entries(expected)) {
|
||||
const actualScript = packageScripts[scriptName]
|
||||
const expectedScript = getExpectedScript(packageDir)
|
||||
if (actualScript !== expectedScript) {
|
||||
nicelog(
|
||||
[
|
||||
'❌ ',
|
||||
kleur.red(`${name}: `),
|
||||
kleur.blue(`$ yarn ${scriptName}`),
|
||||
kleur.grey(' -> '),
|
||||
kleur.red(actualScript ?? '<missing>'),
|
||||
kleur.gray(' (expected: '),
|
||||
kleur.green(expectedScript),
|
||||
kleur.gray(')'),
|
||||
].join('')
|
||||
)
|
||||
packageScripts[scriptName] = expectedScript
|
||||
needsFix.add(name)
|
||||
errorCount++
|
||||
} else {
|
||||
nicelog(
|
||||
[
|
||||
'✅ ',
|
||||
kleur.green(`${name}: `),
|
||||
kleur.blue(`$ yarn ${scriptName}`),
|
||||
kleur.grey(' -> '),
|
||||
kleur.green(actualScript ?? '<missing>'),
|
||||
].join('')
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (errorCount) {
|
||||
if (fix) {
|
||||
for (const { packageJson, name, relativePath } of packages) {
|
||||
if (needsFix.has(name)) {
|
||||
nicelog(kleur.yellow(`Fixing ${name}...`))
|
||||
await writeJsonFile(path.join(REPO_ROOT, relativePath, 'package.json'), packageJson)
|
||||
}
|
||||
}
|
||||
nicelog(kleur.yellow(`Fixed ${errorCount} errors`))
|
||||
return true
|
||||
} else {
|
||||
nicelog(kleur.red(`Found ${errorCount} errors`))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
async function checkTsConfigs({
|
||||
packages,
|
||||
fix,
|
||||
}: {
|
||||
fix: boolean
|
||||
packages: Package[]
|
||||
}): Promise<boolean> {
|
||||
let numErrors = 0
|
||||
|
||||
for (const workspace of packages) {
|
||||
const tsconfigPath = join(workspace.path, 'tsconfig.json')
|
||||
if (packagesWithoutTSConfigs.has(workspace.name)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const tsconfig = (await readJsonIfExists(tsconfigPath)) as {
|
||||
references?: { path: string }[]
|
||||
}
|
||||
if (!tsconfig) {
|
||||
throw new Error('No tsconfig.json found at ' + tsconfigPath)
|
||||
}
|
||||
|
||||
const tldrawDeps = Object.keys({
|
||||
...workspace.packageJson.dependencies,
|
||||
...workspace.packageJson.devDependencies,
|
||||
}).filter((dep) => packages.some((p) => p.name === dep))
|
||||
|
||||
const fixedDeps = []
|
||||
const missingRefs = []
|
||||
const currentRefs = new Set<string>([...(tsconfig.references?.map((ref) => ref.path) ?? [])])
|
||||
for (const dep of tldrawDeps) {
|
||||
// construct the expected path to the dependency's tsconfig
|
||||
const matchingWorkspace = packages.find((p) => p.name === dep)
|
||||
if (!matchingWorkspace) {
|
||||
throw new Error(`No workspace found for ${dep}`)
|
||||
}
|
||||
const tsconfigReferencePath = relative(workspace.path, matchingWorkspace.path)
|
||||
fixedDeps.push({ path: tsconfigReferencePath })
|
||||
if (currentRefs.has(tsconfigReferencePath)) {
|
||||
currentRefs.delete(tsconfigReferencePath)
|
||||
} else {
|
||||
missingRefs.push(dep)
|
||||
}
|
||||
}
|
||||
|
||||
fixedDeps.sort((a, b) => a.path.localeCompare(b.path))
|
||||
|
||||
if (currentRefs.size > 0) {
|
||||
if (fix) {
|
||||
tsconfig.references = fixedDeps
|
||||
await writeJsonFile(tsconfigPath, tsconfig)
|
||||
} else {
|
||||
numErrors++
|
||||
nicelog(
|
||||
[
|
||||
'❌ ',
|
||||
kleur.red(`${workspace.name}: `),
|
||||
kleur.blue(relative(process.cwd(), tsconfigPath)),
|
||||
kleur.grey(' has unnecessary reference(s) to '),
|
||||
kleur.red([...currentRefs].join(', ')),
|
||||
].join('')
|
||||
)
|
||||
}
|
||||
}
|
||||
if (missingRefs.length) {
|
||||
if (fix) {
|
||||
tsconfig.references = fixedDeps
|
||||
await writeJsonFile(tsconfigPath, tsconfig)
|
||||
} else {
|
||||
numErrors++
|
||||
nicelog(
|
||||
[
|
||||
'❌ ',
|
||||
kleur.red(`${workspace.name}: `),
|
||||
kleur.blue(relative(process.cwd(), tsconfigPath)),
|
||||
kleur.grey(' is missing reference(s) to '),
|
||||
kleur.red(missingRefs.join(', ')),
|
||||
].join('')
|
||||
)
|
||||
nicelog('The references entry should look like this:')
|
||||
nicelog('"references": ' + JSON.stringify(fixedDeps, null, 2))
|
||||
}
|
||||
}
|
||||
|
||||
if (currentRefs.size === 0 && missingRefs.length === 0) {
|
||||
nicelog(['✅ ', kleur.green(`${workspace.name}`)].join(''))
|
||||
}
|
||||
}
|
||||
if (numErrors > 0) {
|
||||
nicelog('Run `yarn check-tsconfigs --fix` to fix these problems')
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
function scriptPath(packageDir: string, scriptName: string) {
|
||||
return path.relative(packageDir, path.join(__dirname, scriptName))
|
||||
}
|
||||
|
||||
function tsScript(scriptName: string) {
|
||||
return (packageDir: string) => `yarn run -T tsx ${scriptPath(packageDir, scriptName)}`
|
||||
}
|
||||
|
||||
async function checkLibraryContents({
|
||||
fix,
|
||||
packages,
|
||||
}: {
|
||||
fix: boolean
|
||||
packages: Package[]
|
||||
}): Promise<boolean> {
|
||||
let errorCount = 0
|
||||
|
||||
for (const { packageJson, name, path } of packages) {
|
||||
if (packageJson.private) continue
|
||||
|
||||
const sourceFilePath = join(path, 'src', 'index.ts')
|
||||
const sourceFileContents = await readFileIfExists(sourceFilePath)
|
||||
if (!sourceFileContents) {
|
||||
nicelog(['⏩ ', kleur.blue(`${name}: `), 'skipped (no src/index.ts)'].join(''))
|
||||
continue
|
||||
}
|
||||
|
||||
const search = [
|
||||
'registerTldrawLibraryVersion(',
|
||||
'\t(globalThis as any).TLDRAW_LIBRARY_NAME,',
|
||||
'\t(globalThis as any).TLDRAW_LIBRARY_VERSION,',
|
||||
'\t(globalThis as any).TLDRAW_LIBRARY_MODULES',
|
||||
')',
|
||||
].join('\n')
|
||||
|
||||
if (sourceFileContents.includes(search)) {
|
||||
nicelog(['✅ ', kleur.green(name)].join(''))
|
||||
continue
|
||||
}
|
||||
|
||||
errorCount++
|
||||
if (fix) {
|
||||
const newSourceFileContents = [
|
||||
"import {registerTldrawLibraryVersion} from '@tldraw/utils'",
|
||||
sourceFileContents,
|
||||
'',
|
||||
'registerTldrawLibraryVersion(',
|
||||
'(globalThis as any).TLDRAW_LIBRARY_NAME,',
|
||||
'(globalThis as any).TLDRAW_LIBRARY_VERSION,',
|
||||
'(globalThis as any).TLDRAW_LIBRARY_NAME',
|
||||
')',
|
||||
].join('\n')
|
||||
|
||||
await writeCodeFile(null, 'typescript', sourceFilePath, newSourceFileContents)
|
||||
nicelog(
|
||||
[
|
||||
'⚠️ ',
|
||||
kleur.yellow(`${name}: `),
|
||||
'added call to ',
|
||||
kleur.blue('registerTldrawLibraryVersion'),
|
||||
].join('')
|
||||
)
|
||||
} else {
|
||||
nicelog(
|
||||
[
|
||||
'❌ ',
|
||||
kleur.red(`${name}: `),
|
||||
'missing call to ',
|
||||
kleur.blue('registerTldrawLibraryVersion'),
|
||||
].join('')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (errorCount) {
|
||||
if (fix) {
|
||||
nicelog(kleur.yellow(`Fixed ${errorCount} errors`))
|
||||
return true
|
||||
} else {
|
||||
nicelog(kleur.red(`Found ${errorCount} errors`))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
const LICENSE_POINTER_CONTENTS =
|
||||
'This code is licensed under the [tldraw license](https://github.com/tldraw/tldraw/blob/main/LICENSE.md)\n'
|
||||
|
||||
// The stable id convention is documented in internal/docs/product-stable-ids.md. Every published
|
||||
// package in packages/ must declare which commercial component it belongs to, and premium
|
||||
// components must name the license key flag that entitles them.
|
||||
async function checkProductMetadata({
|
||||
packages,
|
||||
fix,
|
||||
}: {
|
||||
packages: Package[]
|
||||
fix: boolean
|
||||
}): Promise<boolean> {
|
||||
let errorCount = 0
|
||||
const error = (name: string, message: string) => {
|
||||
errorCount++
|
||||
nicelog(['❌ ', kleur.red(`${name}: `), message].join(''))
|
||||
}
|
||||
|
||||
const licenseManagerSource = await readFileIfExists(
|
||||
join(REPO_ROOT, 'packages/editor/src/lib/license/LicenseManager.ts')
|
||||
)
|
||||
if (!licenseManagerSource) {
|
||||
throw new Error('Could not read LicenseManager.ts to validate license flags')
|
||||
}
|
||||
const knownLicenseFlags = new Set(
|
||||
[...licenseManagerSource.matchAll(/^\t(FEAT_[A-Z0-9_]+):/gm)].map((m) => m[1])
|
||||
)
|
||||
|
||||
const byStableId = new Map<string, { name: string; config: string }[]>()
|
||||
|
||||
for (const { packageJson, name, relativePath, path: packageDir } of packages) {
|
||||
if (!relativePath.startsWith('packages/') || packageJson.private) continue
|
||||
|
||||
// published packages that point at LICENSE.md must actually ship one, since npm only
|
||||
// includes license files that exist inside the package directory
|
||||
if (packageJson.license === 'SEE LICENSE IN LICENSE.md') {
|
||||
const licensePath = join(packageDir, 'LICENSE.md')
|
||||
if (!existsSync(licensePath)) {
|
||||
if (fix) {
|
||||
await writeStringFile(licensePath, LICENSE_POINTER_CONTENTS)
|
||||
nicelog(['⚠️ ', kleur.yellow(`${name}: `), 'added missing LICENSE.md'].join(''))
|
||||
} else {
|
||||
error(name, 'is missing LICENSE.md (run `yarn check-packages --fix`)')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const product = packageJson[PRODUCT_CONFIG_KEY]
|
||||
if (!product) {
|
||||
error(
|
||||
name,
|
||||
`is missing the "${PRODUCT_CONFIG_KEY}" field in package.json. ` +
|
||||
'See internal/docs/product-stable-ids.md for how to choose a stable id.'
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if (!/^tldraw:[a-z0-9-]+$/.test(product.stableId)) {
|
||||
error(name, `has invalid stableId "${product.stableId}" (expected tldraw:<kebab-case>)`)
|
||||
}
|
||||
if (product.premium && !product.licenseFlag) {
|
||||
error(name, 'is premium but has no licenseFlag')
|
||||
}
|
||||
if (!product.premium && product.licenseFlag) {
|
||||
error(name, 'has a licenseFlag but is not premium')
|
||||
}
|
||||
if (product.licenseFlag && !knownLicenseFlags.has(product.licenseFlag)) {
|
||||
error(
|
||||
name,
|
||||
`has licenseFlag "${product.licenseFlag}" which does not exist in LicenseManager FLAGS`
|
||||
)
|
||||
}
|
||||
|
||||
if (product.type === 'feature' && !product.parent) {
|
||||
error(name, 'is a feature but has no parent component')
|
||||
}
|
||||
if (product.type === 'product' && product.parent) {
|
||||
error(name, 'is a top-level product but declares a parent')
|
||||
}
|
||||
if (product.parent === product.stableId) {
|
||||
error(name, 'is its own parent')
|
||||
}
|
||||
|
||||
// all packages sharing a stable id must agree on the rest of the metadata, since they
|
||||
// describe the same commercial component
|
||||
const configKey = JSON.stringify([
|
||||
product.name,
|
||||
product.type,
|
||||
product.parent ?? null,
|
||||
product.premium,
|
||||
product.licenseFlag ?? null,
|
||||
])
|
||||
const others = byStableId.get(product.stableId) ?? []
|
||||
others.push({ name, config: configKey })
|
||||
byStableId.set(product.stableId, others)
|
||||
}
|
||||
|
||||
for (const [stableId, entries] of byStableId) {
|
||||
if (new Set(entries.map((e) => e.config)).size > 1) {
|
||||
error(
|
||||
stableId,
|
||||
`packages disagree on product metadata: ${entries.map((e) => e.name).join(', ')}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// every parent must be a component that actually exists, or the order form would reference a
|
||||
// line item nothing defines
|
||||
for (const { packageJson, name, relativePath } of packages) {
|
||||
if (!relativePath.startsWith('packages/') || packageJson.private) continue
|
||||
const parent = packageJson[PRODUCT_CONFIG_KEY]?.parent
|
||||
if (parent && !byStableId.has(parent)) {
|
||||
error(name, `has parent "${parent}" which is not a known component`)
|
||||
}
|
||||
}
|
||||
|
||||
if (errorCount) {
|
||||
nicelog(kleur.red(`Found ${errorCount} errors`))
|
||||
return false
|
||||
}
|
||||
|
||||
nicelog(['✅ ', kleur.green('product metadata ok')].join(''))
|
||||
return true
|
||||
}
|
||||
|
||||
async function group<T>(name: string, cb: () => Promise<T>) {
|
||||
console.group(name)
|
||||
try {
|
||||
return await cb()
|
||||
} finally {
|
||||
console.groupEnd()
|
||||
console.log('')
|
||||
}
|
||||
}
|
||||
|
||||
// Every workspace uses the same version range for a dependency, so the lockfile resolves one
|
||||
// copy. To let a dependency diverge, list the workspaces that step off the shared range here:
|
||||
// they must agree with each other, and everything else must agree with everything else.
|
||||
const ALLOWED_VERSION_DIVERGENCE: Record<string, { workspaces: string[]; reason: string }> = {
|
||||
typescript: {
|
||||
workspaces: ['templates/', 'apps/mcp-app'],
|
||||
reason:
|
||||
"templates are independently published starters, and mcp-app's extract-editor-api.ts " +
|
||||
"needs the TS 5 compiler API that TS 7 doesn't export",
|
||||
},
|
||||
}
|
||||
|
||||
// Node 22.12 is the first version where `require()` of an ES module works unflagged, so
|
||||
// published packages can depend on ESM-only modules without breaking CommonJS consumers.
|
||||
// Earlier versions throw `ERR_REQUIRE_ESM`.
|
||||
const PUBLISHED_NODE_ENGINE = '>=22.12.0'
|
||||
|
||||
const DEPENDENCY_FIELDS = [
|
||||
'dependencies',
|
||||
'devDependencies',
|
||||
'optionalDependencies',
|
||||
'peerDependencies',
|
||||
] as const
|
||||
type DependencyField = (typeof DEPENDENCY_FIELDS)[number]
|
||||
|
||||
async function checkDependencyVersions({
|
||||
packages,
|
||||
fix,
|
||||
}: {
|
||||
packages: Package[]
|
||||
fix: boolean
|
||||
}): Promise<boolean> {
|
||||
let errorCount = 0
|
||||
const changed = new Set<Package>()
|
||||
const report = (name: string, message: string) => {
|
||||
errorCount++
|
||||
nicelog([fix ? '⚠️ ' : '❌ ', (fix ? kleur.yellow : kleur.red)(`${name}: `), message].join(''))
|
||||
}
|
||||
|
||||
const root = await getRootPackage()
|
||||
|
||||
// Peer ranges are deliberately wider than installed ranges, so they're only compared with
|
||||
// each other.
|
||||
const usages = new Map<string, { pkg: Package; field: DependencyField; range: string }[]>()
|
||||
for (const pkg of [root, ...packages]) {
|
||||
for (const field of DEPENDENCY_FIELDS) {
|
||||
for (const [dep, range] of Object.entries(pkg.packageJson[field] ?? {})) {
|
||||
const diverges = ALLOWED_VERSION_DIVERGENCE[dep]?.workspaces.some((prefix) =>
|
||||
pkg.relativePath.startsWith(prefix)
|
||||
)
|
||||
const kind = field === 'peerDependencies' ? 'peer' : 'installed'
|
||||
const key = `${dep}\0${kind}\0${diverges ? 'diverged' : 'shared'}`
|
||||
if (!usages.has(key)) usages.set(key, [])
|
||||
usages.get(key)!.push({ pkg, field, range })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, group] of usages) {
|
||||
const counts = new Map<string, number>()
|
||||
for (const { range } of group) counts.set(range, (counts.get(range) ?? 0) + 1)
|
||||
if (counts.size === 1) continue
|
||||
|
||||
const dep = key.split('\0')[0]
|
||||
const [expected] = [...counts].sort((a, b) => b[1] - a[1])[0]
|
||||
for (const { pkg, field, range } of group) {
|
||||
if (range === expected) continue
|
||||
report(pkg.name, `${field}.${dep} is ${range}, but other workspaces use ${expected}`)
|
||||
if (fix) {
|
||||
pkg.packageJson[field]![dep] = expected
|
||||
changed.add(pkg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const pkg of packages) {
|
||||
if ('packageManager' in pkg.packageJson) {
|
||||
report(pkg.name, 'only the root package.json may set packageManager')
|
||||
if (fix) {
|
||||
delete pkg.packageJson.packageManager
|
||||
changed.add(pkg)
|
||||
}
|
||||
}
|
||||
|
||||
if (!pkg.relativePath.startsWith('packages/')) continue
|
||||
const engines = pkg.packageJson.engines ?? {}
|
||||
if (engines.node !== PUBLISHED_NODE_ENGINE) {
|
||||
report(pkg.name, `engines.node must be ${PUBLISHED_NODE_ENGINE}`)
|
||||
if (fix) {
|
||||
pkg.packageJson.engines = { ...engines, node: PUBLISHED_NODE_ENGINE }
|
||||
changed.add(pkg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const pkg of changed) {
|
||||
await writeJsonFile(join(pkg.path, 'package.json'), pkg.packageJson)
|
||||
}
|
||||
|
||||
if (errorCount) {
|
||||
nicelog(
|
||||
fix
|
||||
? kleur.yellow(`Fixed ${errorCount} errors. Run \`yarn\` to update the lockfile.`)
|
||||
: kleur.red(`Found ${errorCount} errors. Run \`yarn check-packages --fix\` to fix them.`)
|
||||
)
|
||||
return fix
|
||||
}
|
||||
|
||||
nicelog(' ✅ dependency versions ok')
|
||||
return true
|
||||
}
|
||||
|
||||
async function main({ fix }: { fix: boolean }) {
|
||||
const packages = await getAllWorkspacePackages()
|
||||
|
||||
const scriptsOk = await group('Checking package.json scripts...', () =>
|
||||
checkPackageJsonScripts({ packages, fix })
|
||||
)
|
||||
const tsConfigsOk = await group('Checking tsconfig.json files...', () =>
|
||||
checkTsConfigs({ packages, fix })
|
||||
)
|
||||
const libsOk = await group('Checking library source files...', () =>
|
||||
checkLibraryContents({ packages, fix })
|
||||
)
|
||||
const productMetadataOk = await group('Checking product metadata...', () =>
|
||||
checkProductMetadata({ packages, fix })
|
||||
)
|
||||
|
||||
const dependencyVersionsOk = await group('Checking dependency versions...', () =>
|
||||
checkDependencyVersions({ packages, fix })
|
||||
)
|
||||
|
||||
if (!scriptsOk || !tsConfigsOk || !libsOk || !productMetadataOk || !dependencyVersionsOk) {
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main({
|
||||
fix: process.argv.includes('--fix'),
|
||||
})
|
||||
65
internal/scripts/check-worker-bundle.ts
Normal file
65
internal/scripts/check-worker-bundle.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
import { existsSync, readFileSync, rmSync } from 'fs'
|
||||
import { resolve } from 'path'
|
||||
import kleur from 'kleur'
|
||||
import minimist from 'minimist'
|
||||
import { exec } from './lib/exec'
|
||||
import { WORKER_EXTERNAL_DEPS } from './lib/worker-externals'
|
||||
|
||||
const args = minimist(process.argv.slice(2))
|
||||
const sizeLimit = Number(args['size-limit-bytes'])
|
||||
if (!isFinite(sizeLimit) || sizeLimit < 1) {
|
||||
console.error(
|
||||
'Invalid usage. Usage: yarn tsx check-worker-bundle.ts --size-limit-bytes <size> --entry <entry>'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const entry = args['entry']
|
||||
if (!existsSync(entry)) {
|
||||
console.error('Entry file does not exist:', entry)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const bundleMetaFileName = '.bundle-meta.json'
|
||||
|
||||
interface Meta {
|
||||
outputs: {
|
||||
[fileName: string]: {
|
||||
bytes: number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function checkBundleSize() {
|
||||
rmSync(bundleMetaFileName, { force: true })
|
||||
|
||||
console.log('checking bundle size')
|
||||
await exec('yarn', [
|
||||
'esbuild',
|
||||
entry,
|
||||
'--bundle',
|
||||
'--outfile=/dev/null',
|
||||
'--minify',
|
||||
'--metafile=' + bundleMetaFileName,
|
||||
...WORKER_EXTERNAL_DEPS.map((dep) => '--external:' + dep),
|
||||
])
|
||||
|
||||
console.log(kleur.cyan().bold('Checking bundle size'), 'of', resolve(entry) + '\n')
|
||||
|
||||
const meta = JSON.parse(readFileSync(bundleMetaFileName).toString()) as Meta
|
||||
const totalSize = Object.values(meta.outputs).reduce((acc, output) => acc + output.bytes, 0)
|
||||
if (totalSize === 0) {
|
||||
throw new Error('Invalid bundle meta in ' + process.cwd() + '/' + bundleMetaFileName)
|
||||
}
|
||||
|
||||
if (totalSize > sizeLimit) {
|
||||
console.error(
|
||||
`${kleur.red().bold('ERROR')} Bundle size exceeds limit: ${totalSize} > ${sizeLimit}`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(`Bundle size is within limit: ${totalSize} < ${sizeLimit}`)
|
||||
}
|
||||
|
||||
checkBundleSize()
|
||||
41
internal/scripts/clean.sh
Executable file
41
internal/scripts/clean.sh
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
yarn run lazy run clean || true
|
||||
|
||||
# a function called 'goodbye' that takes a string as an argument
|
||||
function goodbye() {
|
||||
rm -rf $1
|
||||
rm -rf */$1
|
||||
rm -rf */*/$1
|
||||
rm -rf */*/*/$1
|
||||
rm -rf */*/*/*/$1
|
||||
rm -rf */*/*/*/*/$1
|
||||
rm -rf **/$1
|
||||
}
|
||||
|
||||
rm -rf node_modules
|
||||
goodbye node_modules
|
||||
goodbye .turbo
|
||||
goodbye tmp-assets
|
||||
goodbye tsconfig.tsbuildinfo
|
||||
goodbye tsconfig.build.tsbuildinfo
|
||||
goodbye tsconfig.build.json
|
||||
goodbye .lazy
|
||||
goodbye dist-esm
|
||||
goodbye dist-cjs
|
||||
goodbye .tsbuild
|
||||
goodbye .tsbuild-pub
|
||||
goodbye .tsbuild-dev
|
||||
goodbye .tsbuild-api
|
||||
goodbye .next
|
||||
|
||||
rm -rf packages/*/api
|
||||
rm -rf {packages,apps}/*/*.tgz
|
||||
rm -rf {packages,apps}/vscode/extension/temp
|
||||
rm -rf {packages,apps}/vscode/extension/editor
|
||||
rm -rf apps/docs/content.json
|
||||
rm -rf apps/dotcom/client/e2e/.auth
|
||||
|
||||
npm i -g corepack
|
||||
yarn
|
||||
104
internal/scripts/cloudflare/dns-check.ts
Normal file
104
internal/scripts/cloudflare/dns-check.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { execFileSync } from 'child_process'
|
||||
import { makeEnv } from '../lib/makeEnv'
|
||||
import { nicelog } from '../lib/nicelog'
|
||||
|
||||
// Check DNS records in a zone: what's there already, and which pr-NNNN-* records are orphaned.
|
||||
// Read-only. Usage: CLOUDFLARE_API_TOKEN=xxx yarn tsx internal/scripts/cloudflare/dns-check.ts <zone_name>
|
||||
// Token needs "Zone: Read" + "DNS: Read"; gh CLI must be authed for tldraw/tldraw.
|
||||
|
||||
const env = makeEnv(['CLOUDFLARE_API_TOKEN'])
|
||||
|
||||
const [zoneName] = process.argv.slice(2)
|
||||
if (!zoneName) {
|
||||
nicelog('usage: dns-check.ts <zone_name>')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const API = 'https://api.cloudflare.com/client/v4'
|
||||
|
||||
async function cfApi<T>(endpoint: string): Promise<T> {
|
||||
const res = await fetch(`${API}${endpoint}`, {
|
||||
headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}` },
|
||||
})
|
||||
if (!res.ok) throw new Error(`GET ${endpoint}: ${res.status} ${res.statusText}`)
|
||||
const data = (await res.json()) as { success: boolean; errors: unknown; result: T }
|
||||
if (!data.success) throw new Error(`GET ${endpoint}: ${JSON.stringify(data.errors)}`)
|
||||
return data.result
|
||||
}
|
||||
|
||||
const prStateCache = new Map<string, string>()
|
||||
function prState(pr: string): string {
|
||||
let state = prStateCache.get(pr)
|
||||
if (!state) {
|
||||
try {
|
||||
state = execFileSync('gh', ['api', `repos/tldraw/tldraw/pulls/${pr}`, '--jq', '.state'], {
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
} catch (err: any) {
|
||||
if (String(err.stderr).includes('HTTP 404')) {
|
||||
state = 'closed'
|
||||
} else {
|
||||
throw new Error(`gh api failed for PR #${pr}: ${err.stderr ?? err}`)
|
||||
}
|
||||
}
|
||||
prStateCache.set(pr, state)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
interface DnsRecord {
|
||||
type: string
|
||||
name: string
|
||||
content: string
|
||||
proxied: boolean
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const zones = await cfApi<{ id: string }[]>(`/zones?name=${zoneName}`)
|
||||
if (!zones[0]) throw new Error(`zone not found: ${zoneName}`)
|
||||
const zoneId = zones[0].id
|
||||
|
||||
const records: DnsRecord[] = []
|
||||
for (let page = 1; ; page++) {
|
||||
const result = await cfApi<DnsRecord[]>(
|
||||
`/zones/${zoneId}/dns_records?per_page=100&page=${page}`
|
||||
)
|
||||
records.push(...result)
|
||||
if (result.length < 100) break
|
||||
}
|
||||
|
||||
const fmt = (r: DnsRecord) =>
|
||||
`${r.type.padEnd(6)} ${r.name.padEnd(36)} ${String(r.content).slice(0, 50).padEnd(52)} proxied=${r.proxied}`
|
||||
|
||||
const isPreview = (r: DnsRecord) => /^pr-\d+-/.test(r.name)
|
||||
|
||||
nicelog('=== non-PR records (existing infra) ===')
|
||||
for (const r of records
|
||||
.filter((r) => !isPreview(r))
|
||||
.sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
nicelog(fmt(r))
|
||||
}
|
||||
|
||||
nicelog('\n=== PR preview records ===')
|
||||
let ok = 0
|
||||
let orphaned = 0
|
||||
for (const r of records.filter(isPreview)) {
|
||||
const pr = r.name.match(/^pr-(\d+)-/)![1]
|
||||
if (prState(pr) === 'closed') {
|
||||
nicelog(`ORPHAN (PR #${pr} closed) ${fmt(r)}`)
|
||||
orphaned++
|
||||
} else {
|
||||
nicelog(`ok (PR #${pr} open) ${fmt(r)}`)
|
||||
ok++
|
||||
}
|
||||
}
|
||||
|
||||
nicelog('\n=== summary ===')
|
||||
nicelog(`total records: ${records.length}`)
|
||||
nicelog(`pr records: ok=${ok} orphaned=${orphaned}`)
|
||||
const hasWildcard = records.some((r) => r.name === `*.${zoneName}`)
|
||||
nicelog(`wildcard *.${zoneName}: ${hasWildcard ? 'EXISTS' : 'not present'}`)
|
||||
}
|
||||
|
||||
main()
|
||||
142
internal/scripts/cloudflare/ssl-cleanup.ts
Normal file
142
internal/scripts/cloudflare/ssl-cleanup.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
import { execFileSync } from 'child_process'
|
||||
import { makeEnv } from '../lib/makeEnv'
|
||||
import { nicelog } from '../lib/nicelog'
|
||||
|
||||
// Delete orphaned per-PR advanced cert packs (pr-NNNN-* hostnames) for CLOSED PRs only.
|
||||
// Usage: CLOUDFLARE_API_TOKEN=xxx yarn tsx internal/scripts/cloudflare/ssl-cleanup.ts <zone_name> [--delete]
|
||||
// Default is dry-run. Token needs "Zone: Read" + "SSL and Certificates: Edit";
|
||||
// gh CLI must be authed for tldraw/tldraw.
|
||||
|
||||
const env = makeEnv(['CLOUDFLARE_API_TOKEN'])
|
||||
|
||||
const [zoneName, mode] = process.argv.slice(2)
|
||||
if (!zoneName || (mode && mode !== '--delete')) {
|
||||
nicelog('usage: ssl-cleanup.ts <zone_name> [--delete]')
|
||||
process.exit(1)
|
||||
}
|
||||
const doDelete = mode === '--delete'
|
||||
|
||||
const API = 'https://api.cloudflare.com/client/v4'
|
||||
|
||||
async function cfApi<T>(endpoint: string, options: RequestInit = {}, attempt = 0): Promise<T> {
|
||||
const res = await fetch(`${API}${endpoint}`, {
|
||||
...options,
|
||||
headers: { Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}` },
|
||||
})
|
||||
if (res.status === 429 && attempt < 3) {
|
||||
const waitSeconds = Number(res.headers.get('retry-after')) || 10
|
||||
nicelog(`rate limited, retrying in ${waitSeconds}s...`)
|
||||
await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1000))
|
||||
return cfApi(endpoint, options, attempt + 1)
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new Error(`${options.method ?? 'GET'} ${endpoint}: ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const data = (await res.json()) as { success: boolean; errors: unknown; result: T }
|
||||
if (!data.success) {
|
||||
throw new Error(`${options.method ?? 'GET'} ${endpoint}: ${JSON.stringify(data.errors)}`)
|
||||
}
|
||||
return data.result
|
||||
}
|
||||
|
||||
// Only an HTTP 404 (deleted PR) counts as closed; any other gh failure throws
|
||||
// so a broken token can't authorize deleting open-PR packs. Uses the gh CLI so
|
||||
// local runs reuse the developer's existing auth.
|
||||
const prStateCache = new Map<string, string>()
|
||||
function prState(pr: string): string {
|
||||
let state = prStateCache.get(pr)
|
||||
if (!state) {
|
||||
try {
|
||||
state = execFileSync('gh', ['api', `repos/tldraw/tldraw/pulls/${pr}`, '--jq', '.state'], {
|
||||
encoding: 'utf-8',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
}).trim()
|
||||
} catch (err: any) {
|
||||
if (String(err.stderr).includes('HTTP 404')) {
|
||||
state = 'closed'
|
||||
} else {
|
||||
throw new Error(`gh api failed for PR #${pr}: ${err.stderr ?? err}`)
|
||||
}
|
||||
}
|
||||
prStateCache.set(pr, state)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
interface CertPack {
|
||||
id: string
|
||||
type: string
|
||||
status: string
|
||||
hosts: string[]
|
||||
}
|
||||
|
||||
function isPreviewPack(p: CertPack) {
|
||||
// status=all also returns packs that are already deleted/pending deletion, skip those
|
||||
return (
|
||||
p.type === 'advanced' &&
|
||||
p.status !== 'deleted' &&
|
||||
p.status !== 'pending_deletion' &&
|
||||
p.hosts.some((h) => /^pr-\d+-/.test(h))
|
||||
)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const zones = await cfApi<{ id: string }[]>(`/zones?name=${zoneName}`)
|
||||
if (!zones[0]) throw new Error(`zone not found: ${zoneName}`)
|
||||
const zoneId = zones[0].id
|
||||
|
||||
let deleted = 0
|
||||
let kept = 0
|
||||
let failed = 0
|
||||
|
||||
// Process one page at a time instead of fetching everything up front, so an
|
||||
// abort partway through (rate limit, network) loses nothing already deleted
|
||||
// and a rerun picks up where it left off. Deleting shifts later items into
|
||||
// the current page, so after a page with deletions we refetch the same page
|
||||
// (skipping packs we've already handled) and only advance once it yields
|
||||
// nothing new. The endpoint caps per_page at 50.
|
||||
const processed = new Set<string>()
|
||||
let page = 1
|
||||
while (true) {
|
||||
const result = await cfApi<CertPack[]>(
|
||||
`/zones/${zoneId}/ssl/certificate_packs?status=all&per_page=50&page=${page}`
|
||||
)
|
||||
let deletedThisPage = 0
|
||||
for (const pack of result.filter(isPreviewPack)) {
|
||||
if (processed.has(pack.id)) continue
|
||||
processed.add(pack.id)
|
||||
const pr = pack.hosts.find((h) => /^pr-\d+-/.test(h))!.match(/^pr-(\d+)-/)![1]
|
||||
const hosts = pack.hosts.join(',')
|
||||
if (prState(pr) !== 'closed') {
|
||||
nicelog(`keeping (PR #${pr} open) ${hosts}`)
|
||||
kept++
|
||||
continue
|
||||
}
|
||||
if (doDelete) {
|
||||
try {
|
||||
await cfApi(`/zones/${zoneId}/ssl/certificate_packs/${pack.id}`, { method: 'DELETE' })
|
||||
nicelog(`deleted (PR #${pr} closed) ${pack.id} ${hosts}`)
|
||||
deleted++
|
||||
deletedThisPage++
|
||||
} catch (err) {
|
||||
nicelog(`FAILED to delete ${pack.id} ${hosts}: ${err}`)
|
||||
failed++
|
||||
}
|
||||
} else {
|
||||
nicelog(`would delete (PR #${pr} closed) ${pack.id} ${hosts}`)
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
if (result.length < 50) break
|
||||
if (deletedThisPage === 0) page++
|
||||
}
|
||||
|
||||
nicelog('---')
|
||||
nicelog(
|
||||
`${deleted} closed-PR packs ${doDelete ? 'deleted' : 'to delete'}, ${kept} open-PR packs kept, ${failed} failed`
|
||||
)
|
||||
if (!doDelete) nicelog('(dry run — rerun with --delete to remove)')
|
||||
if (failed > 0) process.exit(1)
|
||||
}
|
||||
|
||||
main()
|
||||
220
internal/scripts/create-docs-reference.ts
Normal file
220
internal/scripts/create-docs-reference.ts
Normal file
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env tsx
|
||||
|
||||
/**
|
||||
* Creates a reference document for Pandoc with custom styling:
|
||||
* - Times New Roman for all prose
|
||||
* - Consolas for all code
|
||||
* - Tighter margins and spacing for density
|
||||
*/
|
||||
|
||||
import { writeFileSync } from 'fs'
|
||||
import { dirname, resolve } from 'path'
|
||||
import {
|
||||
convertInchesToTwip,
|
||||
Document,
|
||||
HeadingLevel,
|
||||
Packer,
|
||||
PageOrientation,
|
||||
Paragraph,
|
||||
TextRun,
|
||||
} from 'docx'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
const DOCS_DIR = resolve(dirname(dirname(dirname(__filename))), 'documentation')
|
||||
const REFERENCE_FILE = resolve(DOCS_DIR, 'reference.docx')
|
||||
|
||||
// Font settings
|
||||
const BODY_FONT = 'Times New Roman'
|
||||
const CODE_FONT = 'Consolas'
|
||||
const BODY_SIZE = 22 // 11pt in half-points
|
||||
const CODE_SIZE = 20 // 10pt in half-points
|
||||
const HEADING1_SIZE = 32 // 16pt
|
||||
const HEADING2_SIZE = 28 // 14pt
|
||||
const HEADING3_SIZE = 24 // 12pt
|
||||
|
||||
async function main() {
|
||||
nicelog('📄 Creating reference document for Pandoc...')
|
||||
|
||||
const doc = new Document({
|
||||
styles: {
|
||||
default: {
|
||||
document: {
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: BODY_SIZE,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
after: 120, // 6pt after paragraphs
|
||||
line: 240, // Single line spacing (240 twips = 1 line)
|
||||
},
|
||||
},
|
||||
},
|
||||
heading1: {
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: HEADING1_SIZE,
|
||||
bold: true,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
before: 240,
|
||||
after: 120,
|
||||
},
|
||||
},
|
||||
},
|
||||
heading2: {
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: HEADING2_SIZE,
|
||||
bold: true,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
before: 200,
|
||||
after: 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
heading3: {
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: HEADING3_SIZE,
|
||||
bold: true,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
before: 160,
|
||||
after: 80,
|
||||
},
|
||||
},
|
||||
},
|
||||
heading4: {
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: BODY_SIZE,
|
||||
bold: true,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
before: 120,
|
||||
after: 60,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
paragraphStyles: [
|
||||
{
|
||||
id: 'Normal',
|
||||
name: 'Normal',
|
||||
run: {
|
||||
font: BODY_FONT,
|
||||
size: BODY_SIZE,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
after: 120,
|
||||
line: 240,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'SourceCode',
|
||||
name: 'Source Code',
|
||||
basedOn: 'Normal',
|
||||
run: {
|
||||
font: CODE_FONT,
|
||||
size: CODE_SIZE,
|
||||
},
|
||||
paragraph: {
|
||||
spacing: {
|
||||
after: 0,
|
||||
line: 220,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'VerbatimChar',
|
||||
name: 'Verbatim Char',
|
||||
basedOn: 'Normal',
|
||||
run: {
|
||||
font: CODE_FONT,
|
||||
size: CODE_SIZE,
|
||||
},
|
||||
},
|
||||
],
|
||||
characterStyles: [
|
||||
{
|
||||
id: 'VerbatimChar',
|
||||
name: 'Verbatim Char',
|
||||
run: {
|
||||
font: CODE_FONT,
|
||||
size: CODE_SIZE,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
sections: [
|
||||
{
|
||||
properties: {
|
||||
page: {
|
||||
margin: {
|
||||
top: convertInchesToTwip(0.75),
|
||||
right: convertInchesToTwip(0.75),
|
||||
bottom: convertInchesToTwip(0.75),
|
||||
left: convertInchesToTwip(0.75),
|
||||
},
|
||||
size: {
|
||||
orientation: PageOrientation.PORTRAIT,
|
||||
},
|
||||
},
|
||||
},
|
||||
children: [
|
||||
// Sample content to establish styles
|
||||
new Paragraph({
|
||||
text: 'Heading 1',
|
||||
heading: HeadingLevel.HEADING_1,
|
||||
}),
|
||||
new Paragraph({
|
||||
text: 'Heading 2',
|
||||
heading: HeadingLevel.HEADING_2,
|
||||
}),
|
||||
new Paragraph({
|
||||
text: 'Heading 3',
|
||||
heading: HeadingLevel.HEADING_3,
|
||||
}),
|
||||
new Paragraph({
|
||||
children: [
|
||||
new TextRun({
|
||||
text: 'Normal body text in Times New Roman.',
|
||||
font: BODY_FONT,
|
||||
size: BODY_SIZE,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
new Paragraph({
|
||||
children: [
|
||||
new TextRun({
|
||||
text: 'const code = "Consolas font";',
|
||||
font: CODE_FONT,
|
||||
size: CODE_SIZE,
|
||||
}),
|
||||
],
|
||||
style: 'SourceCode',
|
||||
}),
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const buffer = await Packer.toBuffer(doc)
|
||||
writeFileSync(REFERENCE_FILE, buffer)
|
||||
|
||||
nicelog(`✅ Created: ${REFERENCE_FILE}`)
|
||||
nicelog(' Use with: pandoc --reference-doc=documentation/reference.docx')
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
nicelog(`❌ Failed: ${error}`)
|
||||
process.exit(1)
|
||||
})
|
||||
110
internal/scripts/deploy-analytics.ts
Normal file
110
internal/scripts/deploy-analytics.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import assert from 'assert'
|
||||
import { readFileSync } from 'fs'
|
||||
import path, { join } from 'path'
|
||||
import toml from 'toml'
|
||||
import {
|
||||
createGithubDeployment,
|
||||
getDeployInfo,
|
||||
setWranglerPreviewConfig,
|
||||
wranglerDeploy,
|
||||
} from './lib/deploy'
|
||||
import { Discord } from './lib/discord'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
const workerDir = path.relative(process.cwd(), path.resolve(REPO_ROOT, './apps/analytics-worker'))
|
||||
|
||||
// Do not use `process.env` directly in this script. Add your variable to `makeEnv` and use it via
|
||||
// `env` instead. This makes sure that all required env vars are present.
|
||||
const env = makeEnv([
|
||||
'CLOUDFLARE_ACCOUNT_ID',
|
||||
'CLOUDFLARE_API_TOKEN',
|
||||
'DISCORD_DEPLOY_WEBHOOK_URL',
|
||||
'RELEASE_COMMIT_HASH',
|
||||
'TLDRAW_ENV',
|
||||
'GH_TOKEN',
|
||||
])
|
||||
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
shouldNotify: env.TLDRAW_ENV === 'production',
|
||||
totalSteps: 2,
|
||||
messagePrefix: '[ANALYTICS]',
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
|
||||
const { previewId, sha } = getDeployInfo()
|
||||
|
||||
async function main() {
|
||||
assert(
|
||||
env.TLDRAW_ENV === 'staging' || env.TLDRAW_ENV === 'production' || env.TLDRAW_ENV === 'preview',
|
||||
'TLDRAW_ENV must be staging or production or preview'
|
||||
)
|
||||
|
||||
await discord.message(`--- **${env.TLDRAW_ENV} analytics worker deploy pre-flight** ---`)
|
||||
|
||||
await discord.step('cloudflare deploy dry run', async () => {
|
||||
await deployAnalyticsWorker({ dryRun: true })
|
||||
})
|
||||
|
||||
// --- point of no return! do the deploy for real --- //
|
||||
|
||||
await discord.message(`--- **pre-flight complete, starting real analytics worker deploy** ---`)
|
||||
|
||||
await discord.step('deploying analytics worker to cloudflare', async () => {
|
||||
await deployAnalyticsWorker({ dryRun: false })
|
||||
})
|
||||
|
||||
// Get the worker name from wrangler.toml
|
||||
const wranglerConfig = toml.parse(readFileSync(join(workerDir, 'wrangler.toml')).toString())
|
||||
const workerName = wranglerConfig?.env?.[env.TLDRAW_ENV]?.name || wranglerConfig?.name
|
||||
if (!workerName) {
|
||||
throw new Error('Could not find worker name in wrangler.toml')
|
||||
}
|
||||
|
||||
// Analytics worker uses default workers.dev domain
|
||||
const deploymentUrl = `https://${workerName}.${env.CLOUDFLARE_ACCOUNT_ID}.workers.dev`
|
||||
|
||||
nicelog('Creating deployment for', deploymentUrl)
|
||||
await createGithubDeployment(env, {
|
||||
app: 'analytics-worker',
|
||||
deploymentUrl,
|
||||
sha,
|
||||
})
|
||||
|
||||
await discord.message(`**Deploy complete!**`)
|
||||
}
|
||||
|
||||
let didUpdateAnalyticsWorker = false
|
||||
async function deployAnalyticsWorker({ dryRun }: { dryRun: boolean }) {
|
||||
const workerId = `${previewId ?? env.TLDRAW_ENV}-tldraw-consent`
|
||||
if (previewId && !didUpdateAnalyticsWorker) {
|
||||
await setWranglerPreviewConfig(workerDir, {
|
||||
name: workerId,
|
||||
routeHostname: `${previewId}-consent.tldraw.xyz`,
|
||||
})
|
||||
didUpdateAnalyticsWorker = true
|
||||
}
|
||||
|
||||
await wranglerDeploy({
|
||||
location: workerDir,
|
||||
dryRun,
|
||||
env: env.TLDRAW_ENV,
|
||||
vars: {
|
||||
WORKER_NAME: workerId,
|
||||
TLDRAW_ENV: env.TLDRAW_ENV,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
main().catch(async (err) => {
|
||||
// don't notify discord on preview builds
|
||||
if (env.TLDRAW_ENV !== 'preview') {
|
||||
await discord.message(`${Discord.AT_TEAM_MENTION} Deploy failed: ${err.stack}`, {
|
||||
always: true,
|
||||
})
|
||||
}
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
123
internal/scripts/deploy-bemo.ts
Normal file
123
internal/scripts/deploy-bemo.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
import assert from 'assert'
|
||||
import { readFileSync } from 'fs'
|
||||
import path, { join } from 'path'
|
||||
import toml from 'toml'
|
||||
import {
|
||||
createGithubDeployment,
|
||||
getDeployInfo,
|
||||
setWranglerPreviewConfig,
|
||||
wranglerDeploy,
|
||||
} from './lib/deploy'
|
||||
import { Discord } from './lib/discord'
|
||||
import { exec } from './lib/exec'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
const workerDir = path.relative(process.cwd(), path.resolve(REPO_ROOT, './apps/bemo-worker'))
|
||||
|
||||
// Do not use `process.env` directly in this script. Add your variable to `makeEnv` and use it via
|
||||
// `env` instead. This makes sure that all required env vars are present.
|
||||
const env = makeEnv([
|
||||
'CLOUDFLARE_ACCOUNT_ID',
|
||||
'CLOUDFLARE_API_TOKEN',
|
||||
'DISCORD_DEPLOY_WEBHOOK_URL',
|
||||
'RELEASE_COMMIT_HASH',
|
||||
'TLDRAW_ENV',
|
||||
'GH_TOKEN',
|
||||
'SENTRY_AUTH_TOKEN',
|
||||
'SENTRY_BEMO_WORKER_DSN',
|
||||
])
|
||||
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
shouldNotify: env.TLDRAW_ENV === 'production',
|
||||
totalSteps: 3,
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
|
||||
const { previewId, sha } = getDeployInfo()
|
||||
|
||||
async function main() {
|
||||
assert(
|
||||
env.TLDRAW_ENV === 'staging' || env.TLDRAW_ENV === 'production' || env.TLDRAW_ENV === 'preview',
|
||||
'TLDRAW_ENV must be staging or production or preview'
|
||||
)
|
||||
|
||||
await discord.message(`--- **${env.TLDRAW_ENV} bemo deploy pre-flight** ---`)
|
||||
|
||||
await discord.step('setting up deploy', async () => {
|
||||
await exec('yarn', ['lazy', 'prebuild'])
|
||||
})
|
||||
|
||||
await discord.step('cloudflare deploy dry run', async () => {
|
||||
await deployBemoWorker({ dryRun: true })
|
||||
})
|
||||
|
||||
// --- point of no return! do the deploy for real --- //
|
||||
|
||||
await discord.message(`--- **pre-flight complete, starting real bemo deploy** ---`)
|
||||
|
||||
// 2. deploy the cloudflare workers:
|
||||
await discord.step('deploying bemo-worker to cloudflare', async () => {
|
||||
await deployBemoWorker({ dryRun: false })
|
||||
})
|
||||
|
||||
// the deploy hostname comes from the env's route pattern in wrangler.toml
|
||||
const routePattern = toml.parse(readFileSync(join(workerDir, 'wrangler.toml')).toString())?.env[
|
||||
env.TLDRAW_ENV
|
||||
]?.routes?.[0]?.pattern
|
||||
if (!routePattern) {
|
||||
throw new Error('Could not find the route pattern in wrangler.toml')
|
||||
}
|
||||
|
||||
// preview route patterns carry a path glob ("host/*"); the deployment URL wants just the host
|
||||
const deploymentUrl = `https://${routePattern.split('/')[0]}`
|
||||
|
||||
nicelog('Creating deployment for', deploymentUrl)
|
||||
await createGithubDeployment(env, {
|
||||
app: 'bemo',
|
||||
deploymentUrl,
|
||||
sha,
|
||||
})
|
||||
|
||||
await discord.message(`**Deploy complete!**`)
|
||||
}
|
||||
|
||||
let didUpdateBemoWorker = false
|
||||
async function deployBemoWorker({ dryRun }: { dryRun: boolean }) {
|
||||
const workerId = `${previewId ?? env.TLDRAW_ENV}-bemo`
|
||||
if (previewId && !didUpdateBemoWorker) {
|
||||
await setWranglerPreviewConfig(workerDir, {
|
||||
name: workerId,
|
||||
routeHostname: `${previewId}-demo.tldraw.xyz`,
|
||||
})
|
||||
didUpdateBemoWorker = true
|
||||
}
|
||||
|
||||
await wranglerDeploy({
|
||||
location: workerDir,
|
||||
dryRun,
|
||||
env: env.TLDRAW_ENV,
|
||||
vars: {
|
||||
WORKER_NAME: workerId,
|
||||
TLDRAW_ENV: env.TLDRAW_ENV,
|
||||
SENTRY_DSN: env.SENTRY_BEMO_WORKER_DSN,
|
||||
},
|
||||
sentry: {
|
||||
authToken: env.SENTRY_AUTH_TOKEN,
|
||||
project: 'bemo-worker',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
main().catch(async (err) => {
|
||||
// don't notify discord on preview builds
|
||||
if (env.TLDRAW_ENV !== 'preview') {
|
||||
await discord.message(`${Discord.AT_TEAM_MENTION} Deploy failed: ${err.stack}`, {
|
||||
always: true,
|
||||
})
|
||||
}
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
1093
internal/scripts/deploy-dotcom.ts
Normal file
1093
internal/scripts/deploy-dotcom.ts
Normal file
File diff suppressed because it is too large
Load Diff
100
internal/scripts/deploy-template.ts
Normal file
100
internal/scripts/deploy-template.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { readdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { exec } from './lib/exec'
|
||||
import { readJsonIfExists, REPO_ROOT } from './lib/file'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { EXPORT_CONFIG_KEY, PackageJson } from './lib/types'
|
||||
|
||||
const TEMPLATE_DIR = join(REPO_ROOT, 'templates')
|
||||
|
||||
/**
|
||||
* Like `makeEnv`, but also rejects empty values. Github expands an unconfigured secret or variable
|
||||
* to the empty string rather than leaving it unset, so `makeEnv` alone would let a missing license
|
||||
* key through and we'd deploy unlicensed demos on a green CI run — the exact thing this script
|
||||
* exists to prevent.
|
||||
*/
|
||||
function requireEnv<const Keys extends readonly string[]>(keys: Keys) {
|
||||
const env = makeEnv(keys)
|
||||
const emptyVars = keys.filter((key) => env[key as keyof typeof env] === '')
|
||||
if (emptyVars.length > 0) {
|
||||
throw new Error(`Empty environment variables: ${emptyVars.join(', ')}`)
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// The license key for the demos deployed from here. Don't set one in the cloudflare dashboard as
|
||||
// well: a key configured there is invisible to this repo, so nothing catches it silently expiring.
|
||||
//
|
||||
// The templates themselves never reference it. tldraw's LicenseProvider picks it up from the build
|
||||
// environment (see getLicenseKeyFromEnv in packages/editor/src/lib/license/LicenseProvider.tsx), so
|
||||
// the exported public templates stay free of our key.
|
||||
const env = requireEnv(['TEMPLATES_TLDRAW_LICENSE_KEY'])
|
||||
|
||||
// every template deployed from here builds with vite. A next.js template would need
|
||||
// NEXT_PUBLIC_TLDRAW_LICENSE_KEY instead.
|
||||
const licenseEnv = { VITE_TLDRAW_LICENSE_KEY: env.TEMPLATES_TLDRAW_LICENSE_KEY }
|
||||
|
||||
async function main() {
|
||||
const templateName = process.argv[2]
|
||||
if (!templateName) {
|
||||
nicelog('Usage: tsx deploy-template.ts <template-name>')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const possibleTemplates = (await readdir(TEMPLATE_DIR, { withFileTypes: true }))
|
||||
.filter((d) => d.isDirectory())
|
||||
.map((d) => d.name)
|
||||
|
||||
if (!possibleTemplates.includes(templateName)) {
|
||||
nicelog(`Template "${templateName}" not found.`)
|
||||
nicelog(`Possible templates: ${possibleTemplates.join(', ')}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const templateDir = join(TEMPLATE_DIR, templateName)
|
||||
const packageJsonRaw = await readJsonIfExists(join(templateDir, 'package.json'))
|
||||
if (!packageJsonRaw) {
|
||||
nicelog('No package.json found')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const packageJson = PackageJson.validate(packageJsonRaw)
|
||||
const deployConfig = packageJson[EXPORT_CONFIG_KEY]?.deploy
|
||||
if (!deployConfig) {
|
||||
nicelog(`Template "${templateName}" has no deploy config. Skipping.`)
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
await deployToCloudflare(templateName, templateDir)
|
||||
|
||||
nicelog(`Deployed ${templateName} to https://${deployConfig.host}`)
|
||||
}
|
||||
|
||||
async function deployToCloudflare(templateName: string, templateDir: string) {
|
||||
const cloudflareEnv = requireEnv(['CLOUDFLARE_ACCOUNT_ID', 'CLOUDFLARE_API_TOKEN'])
|
||||
|
||||
nicelog(`Building ${templateName}...`)
|
||||
await exec('yarn', ['run', 'build'], {
|
||||
pwd: templateDir,
|
||||
env: { ...licenseEnv, NODE_ENV: 'production' },
|
||||
})
|
||||
|
||||
// The route (including the custom domain) comes from the template's own wrangler.toml, which
|
||||
// ships to users too — so the deploy target is described in one place rather than here.
|
||||
nicelog(`Deploying ${templateName} to cloudflare...`)
|
||||
await exec('yarn', ['wrangler', 'deploy'], {
|
||||
pwd: templateDir,
|
||||
env: {
|
||||
...cloudflareEnv,
|
||||
NODE_ENV: 'production',
|
||||
// wrangler needs CI=1 set to prevent it from trying to do interactive prompts
|
||||
CI: '1',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
45
internal/scripts/dev-template.sh
Executable file
45
internal/scripts/dev-template.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/bin/bash
|
||||
|
||||
template_name="$1"
|
||||
shift || true
|
||||
vite_args=("$@")
|
||||
workspace_root="$(git rev-parse --show-toplevel)"
|
||||
|
||||
if [ -z "$template_name" ]; then
|
||||
echo "Usage: yarn dev-template <template_name>"
|
||||
echo "Available templates: $(ls templates | awk 'ORS=", "' | sed 's/, $//')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "templates/$template_name" ]; then
|
||||
echo "Template $template_name does not exist"
|
||||
echo "Available templates: $(ls templates | awk 'ORS=", "' | sed 's/, $//')"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ${#vite_args[@]} -eq 0 ]; then
|
||||
LAZYREPO_PRETTY_OUTPUT=0 lazy run dev --filter="templates/$template_name" --filter='packages/tldraw' --filter='apps/bemo-worker'
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Running vite directly skips lazy's `dev` prerequisites. Only refresh-assets feeds templates, so
|
||||
# run it here first (usually a cache hit) and bail on failure like the normal `lazy run dev` path.
|
||||
LAZYREPO_PRETTY_OUTPUT=0 lazy run refresh-assets || exit 1
|
||||
|
||||
# lazy doesn't forward extra args, so run vite directly in the template below. Run the shared deps
|
||||
# in their own process group (set -m) so cleanup can kill lazy and its watchers, which lazy itself
|
||||
# won't (it installs no signal handlers, so killing just its pid would orphan them).
|
||||
set -m
|
||||
LAZYREPO_PRETTY_OUTPUT=0 lazy run dev --filter='packages/tldraw' --filter='apps/bemo-worker' &
|
||||
lazy_pid=$!
|
||||
disown
|
||||
cleanup() {
|
||||
# negative pid signals the whole process group
|
||||
kill -- "-$lazy_pid" 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
cd "$workspace_root/templates/$template_name" || exit 1
|
||||
# Args are appended to the template's `dev` script, so they only reach vite for plain-vite templates.
|
||||
# For concurrently-wrapped or next.js templates they hit the wrapper/next instead.
|
||||
yarn dev "${vite_args[@]}"
|
||||
72
internal/scripts/dotcom/cleanup-test-files.ts
Normal file
72
internal/scripts/dotcom/cleanup-test-files.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import { DeleteObjectCommand, S3Client } from '@aws-sdk/client-s3'
|
||||
import { Client } from 'pg'
|
||||
import { makeEnv } from '../lib/makeEnv'
|
||||
|
||||
const env = makeEnv([
|
||||
'CLOUDFLARE_ACCOUNT_ID',
|
||||
'R2_ACCESS_KEY_ID',
|
||||
'R2_ACCESS_KEY_SECRET',
|
||||
'SUPABASE_STAGING_DB_URL',
|
||||
'STAGING_OWNER_ID',
|
||||
])
|
||||
|
||||
const R2_URL = `https://${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`
|
||||
const R2_STAGING_BUCKET = 'rooms-preview'
|
||||
|
||||
const R2 = new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: R2_URL,
|
||||
credentials: {
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_ACCESS_KEY_SECRET,
|
||||
},
|
||||
})
|
||||
|
||||
async function cleanupTestFiles() {
|
||||
const stagingClient = new Client({
|
||||
connectionString: env.SUPABASE_STAGING_DB_URL,
|
||||
})
|
||||
await stagingClient.connect()
|
||||
|
||||
try {
|
||||
const selectQuery = `SELECT id FROM file WHERE file."owningGroupId" = $1 AND id LIKE 'test_%'`
|
||||
const selectResult = await stagingClient.query(selectQuery, [env.STAGING_OWNER_ID])
|
||||
|
||||
const fileIds = selectResult.rows.map((row: any) => row.id)
|
||||
|
||||
for (const fileId of fileIds) {
|
||||
try {
|
||||
const deleteParams = {
|
||||
Bucket: R2_STAGING_BUCKET,
|
||||
Key: `app_rooms/${fileId}`,
|
||||
}
|
||||
|
||||
const deleteResponse = await R2.send(new DeleteObjectCommand(deleteParams))
|
||||
if (
|
||||
!deleteResponse.$metadata.httpStatusCode ||
|
||||
(deleteResponse.$metadata.httpStatusCode !== 200 &&
|
||||
deleteResponse.$metadata.httpStatusCode !== 204)
|
||||
) {
|
||||
console.error('Failed to delete file from R2 staging bucket')
|
||||
continue
|
||||
}
|
||||
|
||||
const deleteFileQuery = `DELETE FROM file WHERE id = $1`
|
||||
const deleteResult = await stagingClient.query(deleteFileQuery, [fileId])
|
||||
|
||||
if (deleteResult.rowCount === 0) {
|
||||
console.error('Failed to delete file from Supabase database - no rows affected')
|
||||
continue
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to delete file', error)
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Successfully cleaned up ${fileIds.length} test files`)
|
||||
} finally {
|
||||
await stagingClient.end()
|
||||
}
|
||||
}
|
||||
|
||||
cleanupTestFiles().catch(console.error)
|
||||
182
internal/scripts/dotcom/copy-random-files-to-staging.ts
Normal file
182
internal/scripts/dotcom/copy-random-files-to-staging.ts
Normal file
@@ -0,0 +1,182 @@
|
||||
import { writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
|
||||
import type { TlaFile } from '@tldraw/dotcom-shared'
|
||||
import { nanoid } from 'nanoid'
|
||||
import { Client } from 'pg'
|
||||
import { makeEnv } from '../lib/makeEnv'
|
||||
|
||||
const env = makeEnv([
|
||||
'CLOUDFLARE_ACCOUNT_ID',
|
||||
'R2_ACCESS_KEY_ID',
|
||||
'R2_ACCESS_KEY_SECRET',
|
||||
'SUPABASE_PRODUCTION_DB_URL',
|
||||
'SUPABASE_STAGING_DB_URL',
|
||||
'STAGING_OWNER_ID',
|
||||
])
|
||||
|
||||
const R2_URL = `https://${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`
|
||||
const R2_PRODUCTION_BUCKET = 'rooms'
|
||||
const R2_STAGING_BUCKET = 'rooms-preview'
|
||||
|
||||
const R2 = new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: R2_URL,
|
||||
credentials: {
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_ACCESS_KEY_SECRET,
|
||||
},
|
||||
})
|
||||
|
||||
async function getRandomFileIds(n: number): Promise<string[]> {
|
||||
const client = new Client({
|
||||
connectionString: env.SUPABASE_PRODUCTION_DB_URL,
|
||||
})
|
||||
await client.connect()
|
||||
|
||||
const threeDays = 3 * 24 * 60 * 60 * 1000
|
||||
|
||||
const query = `
|
||||
SELECT * FROM file
|
||||
TABLESAMPLE BERNOULLI(30)
|
||||
WHERE "shared" = true
|
||||
AND ("updatedAt" - "createdAt") >= $1
|
||||
LIMIT $2;
|
||||
`
|
||||
|
||||
const { rows } = await client.query(query, [threeDays, n])
|
||||
await client.end()
|
||||
|
||||
if (!rows || rows.length === 0) {
|
||||
console.log('No files found matching the criteria')
|
||||
return []
|
||||
}
|
||||
|
||||
return rows.map((r) => r.id)
|
||||
}
|
||||
|
||||
async function copyFilesToStaging(fileIds: string[]) {
|
||||
const copiedIds: string[] = []
|
||||
|
||||
// Create one staging client for all database operations
|
||||
const stagingClient = new Client({
|
||||
connectionString: env.SUPABASE_STAGING_DB_URL,
|
||||
})
|
||||
await stagingClient.connect()
|
||||
|
||||
try {
|
||||
for (const fileId of fileIds) {
|
||||
try {
|
||||
const getParams = {
|
||||
Bucket: R2_PRODUCTION_BUCKET,
|
||||
Key: `app_rooms/${fileId}`,
|
||||
}
|
||||
|
||||
const getResponse = await R2.send(new GetObjectCommand(getParams))
|
||||
|
||||
if (!getResponse.Body) {
|
||||
console.error('Failed to download file - no body')
|
||||
continue
|
||||
}
|
||||
|
||||
const fileContent = await getResponse.Body.transformToByteArray()
|
||||
|
||||
const newId = `test_${nanoid()}`
|
||||
|
||||
const putParams = {
|
||||
Bucket: R2_STAGING_BUCKET,
|
||||
Key: `app_rooms/${newId}`,
|
||||
Body: Buffer.from(fileContent),
|
||||
ContentType: 'application/octet-stream',
|
||||
}
|
||||
|
||||
const putResponse = await R2.send(new PutObjectCommand(putParams))
|
||||
if (!putResponse.$metadata.httpStatusCode || putResponse.$metadata.httpStatusCode !== 200) {
|
||||
console.error('Failed to upload file to staging bucket')
|
||||
continue
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const testFile: TlaFile = {
|
||||
id: newId,
|
||||
name: 'Test File',
|
||||
ownerName: 'Test User',
|
||||
thumbnail: '',
|
||||
shared: false,
|
||||
sharedLinkType: 'link',
|
||||
published: false,
|
||||
lastPublished: 0,
|
||||
publishedSlug: nanoid(),
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
isEmpty: false,
|
||||
isDeleted: false,
|
||||
createSource: null,
|
||||
// the staging user's home workspace: home group id is the user's own id. Must be set —
|
||||
// file_owner_xor_check rejects a row with neither owner, and cleanup-test-files finds
|
||||
// these rows by owningGroupId.
|
||||
owningGroupId: env.STAGING_OWNER_ID,
|
||||
}
|
||||
|
||||
const insertQuery = `
|
||||
INSERT INTO file (id, name, "ownerName", thumbnail, shared, "sharedLinkType", published, "lastPublished", "publishedSlug", "createdAt", "updatedAt", "isEmpty", "isDeleted", "owningGroupId")
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
|
||||
`
|
||||
await stagingClient.query(insertQuery, [
|
||||
testFile.id,
|
||||
testFile.name,
|
||||
testFile.ownerName,
|
||||
testFile.thumbnail,
|
||||
testFile.shared,
|
||||
testFile.sharedLinkType,
|
||||
testFile.published,
|
||||
testFile.lastPublished,
|
||||
testFile.publishedSlug,
|
||||
testFile.createdAt,
|
||||
testFile.updatedAt,
|
||||
testFile.isEmpty,
|
||||
testFile.isDeleted,
|
||||
testFile.owningGroupId,
|
||||
])
|
||||
|
||||
await stagingClient.query(
|
||||
`INSERT INTO group_file ("fileId", "groupId", "createdAt", "updatedAt", "index")
|
||||
VALUES ($1, $2, $3, $4, NULL)`,
|
||||
[testFile.id, testFile.owningGroupId, now, now]
|
||||
)
|
||||
|
||||
copiedIds.push(newId)
|
||||
} catch (error) {
|
||||
console.error('Error copying file:', error)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await stagingClient.end()
|
||||
}
|
||||
|
||||
console.log(`Successfully copied ${copiedIds.length} files to staging`)
|
||||
|
||||
if (copiedIds.length > 0) {
|
||||
const testFilesPath = join(
|
||||
__dirname,
|
||||
'../../../apps/dotcom/client/e2e/fixtures/test-files.json'
|
||||
)
|
||||
writeFileSync(testFilesPath, JSON.stringify(copiedIds, null, 2))
|
||||
console.log('Updated test files')
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const n = parseInt(process.env.STAGING_TEST_FILES_COUNT || '10')
|
||||
console.log(`Getting ${n} random file IDs...`)
|
||||
const fileIds = await getRandomFileIds(n)
|
||||
if (fileIds.length === 0) {
|
||||
console.log('No files found to copy')
|
||||
return
|
||||
}
|
||||
|
||||
console.log('\nCopying files to staging...')
|
||||
await copyFilesToStaging(fileIds)
|
||||
}
|
||||
|
||||
main().catch(console.error)
|
||||
170
internal/scripts/export-template.ts
Normal file
170
internal/scripts/export-template.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
import { copyFile, mkdir, mkdtemp, readdir, readFile, rm } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { dirname, join } from 'path'
|
||||
import { exec } from './lib/exec'
|
||||
import { readJsonIfExists, REPO_ROOT, writeCodeFile, writeJsonFile } from './lib/file'
|
||||
import { EXPORT_CONFIG_KEY, PackageJson, TsConfigJson } from './lib/types'
|
||||
|
||||
const TEMPLATE_DIR = join(REPO_ROOT, 'templates')
|
||||
|
||||
async function main() {
|
||||
const templateName = process.argv[2]
|
||||
if (!templateName) {
|
||||
console.log('Usage: tsx prepare-template-for-export.ts <template-name>')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const possibleTemplates = (await readdir(TEMPLATE_DIR, { withFileTypes: true }))
|
||||
.filter((d) => d.isDirectory())
|
||||
.map((d) => d.name)
|
||||
|
||||
if (!possibleTemplates.includes(templateName)) {
|
||||
console.log(`Template "${templateName}" not found.`)
|
||||
console.log(`Possible templates: ${possibleTemplates.join(', ')}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const sourceDir = join(REPO_ROOT, 'templates', templateName)
|
||||
const githubAuth = getGithubAuth()
|
||||
|
||||
// make a temporary working dir:
|
||||
const workingDir = await mkdtemp(join(tmpdir(), `template-export-${templateName}`))
|
||||
console.log(`Working in ${workingDir}`)
|
||||
|
||||
// read the package.json:
|
||||
const packageJsonRaw = await readJsonIfExists(join(sourceDir, 'package.json'))
|
||||
if (!packageJsonRaw) {
|
||||
console.log('No package.json found')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const packageJson = PackageJson.validate(packageJsonRaw)
|
||||
const exportConfig = packageJson[EXPORT_CONFIG_KEY]
|
||||
if (!exportConfig) {
|
||||
console.log('No export config found in package.json.')
|
||||
console.log(`Please add a "${EXPORT_CONFIG_KEY}" key to your package.json.`)
|
||||
console.log('Skipping for now.')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
// clone the template repo:
|
||||
const repoUrl = `https://${githubAuth}github.com/${exportConfig.repo}.git`
|
||||
console.log(`Cloning ${repoUrl}...`)
|
||||
await exec('git', ['clone', repoUrl, workingDir, '--depth', '1'])
|
||||
|
||||
console.log('Clearing old files...')
|
||||
const oldFiles = (await readdir(workingDir)).filter((f) => f !== '.git')
|
||||
for (const file of oldFiles) {
|
||||
await rm(join(workingDir, file), { recursive: true })
|
||||
}
|
||||
|
||||
console.log('Copying files to export dir...')
|
||||
const templateRepoPrefix = `templates/${templateName}/`
|
||||
const templateFiles = (await exec('git', ['ls-files', templateRepoPrefix]))
|
||||
.split('\n')
|
||||
.map((s) => s.replace(templateRepoPrefix, ''))
|
||||
.filter((s) => s.trim())
|
||||
// Monorepo-only Vercel deploy config; its commands don't work outside this workspace.
|
||||
.filter((s) => s !== 'vercel.json')
|
||||
|
||||
await mkdir(workingDir, { recursive: true })
|
||||
|
||||
for (const file of templateFiles) {
|
||||
const targetFile = join(workingDir, file)
|
||||
await mkdir(dirname(targetFile), { recursive: true })
|
||||
if (isTypeScriptSourceFile(file)) {
|
||||
const source = await readFile(join(REPO_ROOT, templateRepoPrefix, file), 'utf-8')
|
||||
const cleanedSource = source.replace(
|
||||
/\/\* EXCLUDE_FROM_TEMPLATE_EXPORT_START \*\/(.*?)\/\* EXCLUDE_FROM_TEMPLATE_EXPORT_END \*\//gs,
|
||||
''
|
||||
)
|
||||
await writeCodeFile(null, 'typescript', targetFile, cleanedSource)
|
||||
} else {
|
||||
await copyFile(join(REPO_ROOT, templateRepoPrefix, file), targetFile)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Cleaning package.json...')
|
||||
delete packageJson[EXPORT_CONFIG_KEY]
|
||||
|
||||
if (exportConfig?.scripts) {
|
||||
if (!packageJson.scripts) {
|
||||
packageJson.scripts = {}
|
||||
}
|
||||
for (const [name, script] of Object.entries(exportConfig.scripts)) {
|
||||
if (script === null) {
|
||||
delete packageJson.scripts[name]
|
||||
} else {
|
||||
packageJson.scripts[name] = script
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await setWorkspaceDependenciesToLatest(packageJson.dependencies)
|
||||
await setWorkspaceDependenciesToLatest(packageJson.devDependencies)
|
||||
await setWorkspaceDependenciesToLatest(packageJson.peerDependencies)
|
||||
|
||||
await writeJsonFile(join(workingDir, 'package.json'), packageJson)
|
||||
|
||||
console.log('Cleaning tsconfig.json...')
|
||||
const tsconfigJsonRaw = await readJsonIfExists(join(workingDir, 'tsconfig.json'))
|
||||
if (!tsconfigJsonRaw) {
|
||||
console.log('No tsconfig.json found')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const tsconfigJson = TsConfigJson.validate(tsconfigJsonRaw)
|
||||
if (tsconfigJson.references) {
|
||||
tsconfigJson.references = tsconfigJson.references.filter((ref) => !ref.path.startsWith('.'))
|
||||
if (tsconfigJson.references.length === 0) {
|
||||
delete tsconfigJson.references
|
||||
}
|
||||
}
|
||||
|
||||
await writeJsonFile(join(workingDir, 'tsconfig.json'), tsconfigJson)
|
||||
|
||||
console.log('Committing...')
|
||||
const latestTldrawVersion = await getLatestPackageVersion('tldraw')
|
||||
await exec('git', ['add', '.'], { pwd: workingDir })
|
||||
await exec(
|
||||
'git',
|
||||
['commit', '--allow-empty', '-m', `update from tldraw ${latestTldrawVersion}`],
|
||||
{ pwd: workingDir }
|
||||
)
|
||||
|
||||
console.log('Pushing...')
|
||||
await exec('git', ['push', repoUrl, 'main'], { pwd: workingDir })
|
||||
|
||||
console.log('Cleaning up temp dir...')
|
||||
await rm(workingDir, { recursive: true })
|
||||
|
||||
console.log('Done!')
|
||||
}
|
||||
|
||||
function getGithubAuth() {
|
||||
const githubToken = process.env.GITHUB_TOKEN
|
||||
if (!githubToken) return ''
|
||||
|
||||
return `huppy-bot:${githubToken}@`
|
||||
}
|
||||
|
||||
async function setWorkspaceDependenciesToLatest(dependencies?: Record<string, string>) {
|
||||
if (!dependencies) return
|
||||
for (const [name, version] of Object.entries(dependencies)) {
|
||||
if (version.startsWith('workspace:')) {
|
||||
const latestVersion = await getLatestPackageVersion(name)
|
||||
dependencies[name] = `^${latestVersion}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getLatestPackageVersion(packageName: string) {
|
||||
const latestVersion = await exec('npm', ['view', `${packageName}@latest`, 'version'])
|
||||
return latestVersion.trim()
|
||||
}
|
||||
|
||||
function isTypeScriptSourceFile(path: string) {
|
||||
return path.endsWith('.ts') || path.endsWith('.tsx')
|
||||
}
|
||||
|
||||
main()
|
||||
258
internal/scripts/extract-draft-changelog.tsx
Normal file
258
internal/scripts/extract-draft-changelog.tsx
Normal file
@@ -0,0 +1,258 @@
|
||||
import { join } from 'path'
|
||||
import { exec } from './lib/exec'
|
||||
import { REPO_ROOT, writeStringFile } from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
function convertPRLinks(text: string): string {
|
||||
// Convert (#1234) to ([#1234](https://github.com/tldraw/tldraw/issues/1234))
|
||||
return text.replace(/\(#(\d+)\)/g, '([#$1](https://github.com/tldraw/tldraw/issues/$1))')
|
||||
}
|
||||
|
||||
function extractRelevantSections(message: string): string {
|
||||
// Extract Release Notes and API Changes sections (case-insensitive, flexible markdown heading levels)
|
||||
const lines = message.split('\n')
|
||||
const relevantSections: string[] = []
|
||||
let inRelevantSection = false
|
||||
let currentSection: string[] = []
|
||||
|
||||
for (const line of lines) {
|
||||
// Check if this is a heading line (one or more #s followed by text)
|
||||
const headingMatch = line.match(/^(#{1,6})\s+(.+)$/i)
|
||||
if (headingMatch) {
|
||||
const headingText = headingMatch[2].trim().toLowerCase()
|
||||
|
||||
// If we were in a relevant section, save it
|
||||
if (inRelevantSection && currentSection.length > 0) {
|
||||
relevantSections.push(currentSection.join('\n'))
|
||||
currentSection = []
|
||||
}
|
||||
|
||||
// Check if this is a Release Notes or API Changes heading
|
||||
if (headingText.includes('release note') || headingText.includes('api change')) {
|
||||
inRelevantSection = true
|
||||
// Include the heading itself in the section
|
||||
currentSection.push(line)
|
||||
} else {
|
||||
inRelevantSection = false
|
||||
}
|
||||
} else if (inRelevantSection) {
|
||||
// We're in a relevant section, collect the line
|
||||
currentSection.push(line)
|
||||
}
|
||||
}
|
||||
|
||||
// Don't forget the last section
|
||||
if (inRelevantSection && currentSection.length > 0) {
|
||||
relevantSections.push(currentSection.join('\n'))
|
||||
}
|
||||
|
||||
// If we found relevant sections, return them joined together
|
||||
if (relevantSections.length > 0) {
|
||||
// Remove the section headings themselves (first line of each section)
|
||||
const sectionsWithoutHeadings = relevantSections
|
||||
.map((section) => {
|
||||
const sectionLines = section.split('\n')
|
||||
// Remove the first line (the heading) and keep the rest
|
||||
return sectionLines.slice(1).join('\n').trim()
|
||||
})
|
||||
.filter((section) => section.length > 0)
|
||||
|
||||
return sectionsWithoutHeadings.join('\n\n').trim()
|
||||
}
|
||||
|
||||
// Otherwise, return empty string (no content to include)
|
||||
return ''
|
||||
}
|
||||
|
||||
async function getLatestMinorRelease() {
|
||||
// otherwise get the latest minor release
|
||||
const tags = (await exec('git', ['tag', '--list', `v*.*.0`, '--sort=-version:refname']))
|
||||
.toString()
|
||||
.trim()
|
||||
return tags.split('\n')[0]
|
||||
}
|
||||
|
||||
export async function extractChangelog(startRef: string, endRef: string): Promise<string> {
|
||||
nicelog(`Extracting changelog from ${startRef} to ${endRef}`)
|
||||
|
||||
const commitHashes = (await exec('git', ['cherry', startRef, endRef]))
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter((line) => !line.startsWith('-'))
|
||||
.map((line) => line.split(' ')[1])
|
||||
|
||||
nicelog(`Found ${commitHashes.length} commits`)
|
||||
|
||||
// Extract full commit messages for each commit
|
||||
const entries: Array<{
|
||||
firstLine: string
|
||||
restOfMessage: string
|
||||
changeType: string
|
||||
changedFolders: string[]
|
||||
}> = []
|
||||
|
||||
for (const hash of commitHashes) {
|
||||
const fullMessage = await exec('git', ['log', '-1', '--pretty=format:%B', hash], {
|
||||
processStdoutLine: () => {}, // Suppress output
|
||||
processStderrLine: () => {},
|
||||
})
|
||||
|
||||
const lines = fullMessage.trim().split('\n')
|
||||
const firstLine = lines[0] || hash
|
||||
|
||||
// Skip commits with [skip ci] in the title
|
||||
if (firstLine.includes('[skip ci]')) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Get changed files for this commit
|
||||
const changedFiles = await exec('git', ['show', '--name-only', '--pretty=format:', hash], {
|
||||
processStdoutLine: () => {}, // Suppress output
|
||||
processStderrLine: () => {},
|
||||
})
|
||||
|
||||
// Extract unique folders (max 2 levels deep) from changed files
|
||||
const folders = new Set<string>()
|
||||
changedFiles
|
||||
.split('\n')
|
||||
.filter((file) => file.trim())
|
||||
.forEach((file) => {
|
||||
const parts = file.split('/')
|
||||
if (parts.length >= 2) {
|
||||
folders.add(`${parts[0]}/${parts[1]}`)
|
||||
} else if (parts.length === 1) {
|
||||
folders.add(parts[0])
|
||||
}
|
||||
})
|
||||
|
||||
let restOfMessage = lines.slice(1).join('\n').trim()
|
||||
|
||||
// Extract change type from checkboxes
|
||||
let changeType = 'other'
|
||||
const changeTypes = ['bugfix', 'improvement', 'feature', 'api', 'other']
|
||||
|
||||
for (const type of changeTypes) {
|
||||
if (restOfMessage.includes(`- [x] \`${type}\``)) {
|
||||
changeType = type
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Remove the Change type section and all checkboxes
|
||||
restOfMessage = restOfMessage
|
||||
// remove cursor summary
|
||||
.replace(/<!-- CURSOR_SUMMARY -->(.|\s)*/gm, '')
|
||||
// remove Change type section
|
||||
.replace(/### Change type[\s\n]*/g, '')
|
||||
// remove checkboxes
|
||||
.replace(/- \[[ x]\] `(bugfix|improvement|feature|api|other)`[\s\n]*/g, '')
|
||||
// remove co-authored-by lines
|
||||
.replace(/^co-authored-by: .*$/gim, '')
|
||||
// Clean up excessive newlines
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
// remove trailing horizontal rule
|
||||
.trim()
|
||||
.replace(/-{3,}$/, '')
|
||||
.trim()
|
||||
|
||||
// Extract only Release Notes and API Changes sections
|
||||
const extractedContent = extractRelevantSections(restOfMessage)
|
||||
|
||||
// Convert PR links like (#1234) to full URLs
|
||||
const contentWithLinks = convertPRLinks(extractedContent)
|
||||
|
||||
// Only include entries that touch packages/ folder (but not internal packages)
|
||||
const excludedPackages = ['packages/dotcom-shared', 'packages/worker-shared']
|
||||
const touchesPackages = Array.from(folders).some(
|
||||
(folder) => folder.startsWith('packages/') && !excludedPackages.includes(folder)
|
||||
)
|
||||
if (touchesPackages) {
|
||||
entries.push({
|
||||
firstLine: convertPRLinks(firstLine),
|
||||
restOfMessage: contentWithLinks,
|
||||
changeType,
|
||||
changedFolders: Array.from(folders).sort(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Group entries by change type
|
||||
const groupedEntries: Record<string, typeof entries> = {
|
||||
bugfix: [],
|
||||
improvement: [],
|
||||
feature: [],
|
||||
api: [],
|
||||
other: [],
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
groupedEntries[entry.changeType].push(entry)
|
||||
}
|
||||
|
||||
// Generate output.md content
|
||||
let output = `Generated from commits between \`${startRef}\` and \`${endRef}\`\n\n`
|
||||
|
||||
// Output each change type section
|
||||
const changeTypeLabels = {
|
||||
bugfix: 'Bug Fixes',
|
||||
improvement: 'Improvements',
|
||||
feature: 'Features',
|
||||
api: 'API Changes',
|
||||
other: 'Other Changes',
|
||||
}
|
||||
|
||||
for (const [changeType, label] of Object.entries(changeTypeLabels)) {
|
||||
const entriesForType = groupedEntries[changeType]
|
||||
if (entriesForType.length === 0) continue
|
||||
|
||||
output += `## ${label}\n\n`
|
||||
|
||||
for (const entry of entriesForType) {
|
||||
output += `### ${entry.firstLine}\n\n`
|
||||
|
||||
if (entry.restOfMessage) {
|
||||
output += `${entry.restOfMessage}\n\n`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nicelog(`Extracted ${entries.length} changelog entries`)
|
||||
return output
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2)
|
||||
|
||||
await exec('git', ['fetch', 'origin'])
|
||||
await exec('git', ['fetch', 'origin', '--tags'])
|
||||
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.error('Usage: yarn extract-draft-changelog [start-ref] [end-ref]')
|
||||
console.error('Example: yarn extract-draft-changelog v2.0.0 HEAD')
|
||||
console.error('Example: yarn extract-draft-changelog v2.0.0 # uses current HEAD')
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const startRef = args[0] || (await getLatestMinorRelease())
|
||||
const endRef = args[1] || 'origin/production'
|
||||
|
||||
try {
|
||||
const output = await extractChangelog(startRef, endRef)
|
||||
|
||||
// Write to changelog.md in the repo root
|
||||
const outputPath = join(REPO_ROOT, 'changelog.md')
|
||||
await writeStringFile(outputPath, output)
|
||||
|
||||
nicelog(`Draft changelog written to changelog.md`)
|
||||
} catch (error) {
|
||||
console.error('Error extracting changelog:', error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error('Unhandled error:', error)
|
||||
process.exit(1)
|
||||
})
|
||||
}
|
||||
126
internal/scripts/fetch-fly-logs.ts
Normal file
126
internal/scripts/fetch-fly-logs.ts
Normal file
@@ -0,0 +1,126 @@
|
||||
/**
|
||||
* Fetch logs from Fly.io's VictoriaLogs API.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts [options]
|
||||
*
|
||||
* Examples:
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --last 2h
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --last 30m --filter "Purged"
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --start 2026-04-15T15:00:00Z --end 2026-04-15T18:00:00Z
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --last 1h --filter "NOT DEBUG" -o logs.jsonl
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --last 6h --filter "Purged" --pretty
|
||||
* FLY_TOKEN=fo1_xxx npx tsx internal/scripts/fetch-fly-logs.ts --app production-zero-rm --last 1h
|
||||
* npx tsx internal/scripts/fetch-fly-logs.ts --token fo1_xxx --app production-zero-rm --last 1h
|
||||
*
|
||||
* See fly-common.ts for app names and token configuration.
|
||||
*/
|
||||
|
||||
import { FLY_ORG_SLUG, getFlyToken, parseDuration } from './fly-common'
|
||||
|
||||
const BASE_URL = `https://api.fly.io/victorialogs/${FLY_ORG_SLUG}/select/logsql/query`
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2)
|
||||
const opts: Record<string, string> = {}
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--token' || args[i] === '-t') opts.token = args[++i]
|
||||
else if (args[i] === '--app' || args[i] === '-a') opts.app = args[++i]
|
||||
else if (args[i] === '--start') opts.start = args[++i]
|
||||
else if (args[i] === '--end') opts.end = args[++i]
|
||||
else if (args[i] === '--last' || args[i] === '-l') opts.last = args[++i]
|
||||
else if (args[i] === '--filter' || args[i] === '-f') opts.filter = args[++i]
|
||||
else if (args[i] === '--limit') opts.limit = args[++i]
|
||||
else if (args[i] === '--output' || args[i] === '-o') opts.output = args[++i]
|
||||
else if (args[i] === '--pretty' || args[i] === '-p') opts.pretty = 'true'
|
||||
else if (args[i] === '--help' || args[i] === '-h') {
|
||||
console.log(
|
||||
[
|
||||
'Usage: npx tsx internal/scripts/fetch-fly-logs.ts [options]',
|
||||
'',
|
||||
'Options:',
|
||||
' --token, -t Fly API token (falls back to FLY_TOKEN env var)',
|
||||
' --app, -a Fly app name (required)',
|
||||
' --last, -l Duration to look back, e.g. 30m, 2h, 1d (default: 1h)',
|
||||
' --start Start time (ISO 8601), overrides --last',
|
||||
' --end End time (ISO 8601), defaults to now',
|
||||
' --filter, -f Additional LogsQL filter, e.g. "Purged", "NOT DEBUG"',
|
||||
' --limit Max lines (default: unlimited)',
|
||||
' --output, -o Write to file instead of stdout',
|
||||
' --pretty, -p Pretty-print: timestamp + message only',
|
||||
' --help, -h Show this help',
|
||||
].join('\n')
|
||||
)
|
||||
process.exit(0)
|
||||
} else {
|
||||
console.error(`Unknown option: ${args[i]}`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
if (!opts.app) {
|
||||
console.error('--app is required')
|
||||
process.exit(1)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs()
|
||||
|
||||
const now = new Date()
|
||||
const end = opts.end ?? now.toISOString()
|
||||
let start: string
|
||||
if (opts.start) {
|
||||
start = opts.start
|
||||
} else {
|
||||
const ms = parseDuration(opts.last ?? '1h', 'ms')
|
||||
start = new Date(now.getTime() - ms).toISOString()
|
||||
}
|
||||
|
||||
let query = `fly.app.name: ${opts.app}`
|
||||
if (opts.filter) query += ` AND ${opts.filter}`
|
||||
|
||||
const params = new URLSearchParams({ query, start, end })
|
||||
if (opts.limit) params.set('limit', opts.limit)
|
||||
|
||||
const token = getFlyToken(opts.token)
|
||||
const url = `${BASE_URL}?${params}`
|
||||
|
||||
const res = await fetch(url, { headers: { Authorization: `FlyV1 ${token}` } })
|
||||
if (!res.ok) {
|
||||
console.error(`HTTP ${res.status}: ${await res.text()}`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const body = await res.text()
|
||||
if (!body.trim()) {
|
||||
console.error('No logs found.')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
let output: string
|
||||
if (opts.pretty) {
|
||||
output = body
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
const entry = JSON.parse(line)
|
||||
return `${entry._time} ${entry._msg}`
|
||||
})
|
||||
.sort()
|
||||
.join('\n')
|
||||
} else {
|
||||
output = body
|
||||
}
|
||||
|
||||
if (opts.output) {
|
||||
const fs = await import('fs')
|
||||
fs.writeFileSync(opts.output, output + '\n')
|
||||
const lines = output.trim().split('\n').length
|
||||
console.error(`Wrote ${lines} lines to ${opts.output}`)
|
||||
} else {
|
||||
process.stdout.write(output + '\n')
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
201
internal/scripts/fetch-fly-metrics.ts
Normal file
201
internal/scripts/fetch-fly-metrics.ts
Normal file
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* Fetch CPU and memory metrics from Fly.io's Prometheus API.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx internal/scripts/fetch-fly-metrics.ts [options]
|
||||
*
|
||||
* Examples:
|
||||
* npx tsx internal/scripts/fetch-fly-metrics.ts --app production-zero-rm --last 1h
|
||||
* npx tsx internal/scripts/fetch-fly-metrics.ts --app production-zero-rm --last 6h --step 5m
|
||||
* npx tsx internal/scripts/fetch-fly-metrics.ts --token fo1_xxx --app production-zero-rm --last 2h
|
||||
* FLY_TOKEN=fo1_xxx npx tsx internal/scripts/fetch-fly-metrics.ts --app production-zero-rm --last 1h
|
||||
*
|
||||
* See fly-common.ts for app names and token configuration.
|
||||
*/
|
||||
|
||||
import { FLY_ORG_SLUG, getFlyToken, parseDuration } from './fly-common'
|
||||
|
||||
const PROM_URL = `https://api.fly.io/prometheus/${FLY_ORG_SLUG}/api/v1/query_range`
|
||||
|
||||
function parseArgs() {
|
||||
const args = process.argv.slice(2)
|
||||
const opts: Record<string, string> = {}
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--token' || args[i] === '-t') opts.token = args[++i]
|
||||
else if (args[i] === '--app' || args[i] === '-a') opts.app = args[++i]
|
||||
else if (args[i] === '--last' || args[i] === '-l') opts.last = args[++i]
|
||||
else if (args[i] === '--step' || args[i] === '-s') opts.step = args[++i]
|
||||
else if (args[i] === '--output' || args[i] === '-o') opts.output = args[++i]
|
||||
else if (args[i] === '--help' || args[i] === '-h') {
|
||||
console.log(
|
||||
[
|
||||
'Usage: npx tsx internal/scripts/fetch-fly-metrics.ts [options]',
|
||||
'',
|
||||
'Options:',
|
||||
' --token, -t Fly API token (falls back to FLY_TOKEN env var)',
|
||||
' --app, -a Fly app name (required)',
|
||||
' --last, -l Duration to look back, e.g. 30m, 2h, 1d (default: 1h)',
|
||||
' --step, -s Query resolution step, e.g. 1m, 5m (default: 1m)',
|
||||
' --output, -o Write to file instead of stdout',
|
||||
' --help, -h Show this help',
|
||||
].join('\n')
|
||||
)
|
||||
process.exit(0)
|
||||
} else {
|
||||
console.error(`Unknown option: ${args[i]}`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
if (!opts.app) {
|
||||
console.error('--app is required')
|
||||
process.exit(1)
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
interface PromResult {
|
||||
metric: Record<string, string>
|
||||
values: [number, string][]
|
||||
}
|
||||
|
||||
async function queryRange(
|
||||
token: string,
|
||||
query: string,
|
||||
start: number,
|
||||
end: number,
|
||||
step: string
|
||||
): Promise<PromResult[]> {
|
||||
const params = new URLSearchParams({
|
||||
query,
|
||||
start: start.toString(),
|
||||
end: end.toString(),
|
||||
step,
|
||||
})
|
||||
const res = await fetch(`${PROM_URL}?${params}`, {
|
||||
headers: { Authorization: `FlyV1 ${token}` },
|
||||
})
|
||||
if (!res.ok) {
|
||||
console.error(`HTTP ${res.status}: ${await res.text()}`)
|
||||
process.exit(1)
|
||||
}
|
||||
const json = (await res.json()) as { data: { result: PromResult[] } }
|
||||
return json.data.result
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)}GB`
|
||||
if (bytes >= 1e6) return `${(bytes / 1e6).toFixed(1)}MB`
|
||||
return `${(bytes / 1e3).toFixed(1)}KB`
|
||||
}
|
||||
|
||||
function formatTime(epoch: number): string {
|
||||
return new Date(epoch * 1000)
|
||||
.toISOString()
|
||||
.replace('T', ' ')
|
||||
.replace(/\.\d+Z/, 'Z')
|
||||
}
|
||||
|
||||
function seriesLabel(metric: Record<string, string>): string {
|
||||
const instance = metric.instance ?? 'unknown'
|
||||
const region = metric.region ?? ''
|
||||
return region ? `${instance} (${region})` : instance
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs()
|
||||
const token = getFlyToken(opts.token)
|
||||
const step = opts.step ?? '1m'
|
||||
|
||||
const now = Math.floor(Date.now() / 1000)
|
||||
const rangeSec = parseDuration(opts.last ?? '1h')
|
||||
const start = now - rangeSec
|
||||
const app = opts.app
|
||||
|
||||
// Query A: CPU usage % (excludes idle + steal, normalized by core count)
|
||||
const cpuQuery = `sum(increase(fly_instance_cpu{app="${app}", mode!="idle", mode!="steal"}[60s]))by(instance, region)/60 / sum(count(fly_instance_cpu{app="${app}", mode="idle"})without(cpu))by(instance, region)`
|
||||
// Query B: CPU baseline (shared CPU fraction)
|
||||
const baselineQuery = `mode(fly_instance_cpu_baseline{app="${app}"}) / mode(count(fly_instance_cpu{app="${app}", mode="idle"})without(cpu_id, mode)) * 100`
|
||||
// Query C: CPU throttle indicator
|
||||
const throttleQuery = `avg(rate(fly_instance_cpu_throttle{app="${app}"}[60s])) / count(fly_instance_cpu{app="${app}", mode="idle"})without(cpu_id, mode)`
|
||||
// Memory used
|
||||
const memQuery = `fly_instance_memory_mem_total{app="${app}"} - fly_instance_memory_mem_available{app="${app}"}`
|
||||
const memTotalQuery = `fly_instance_memory_mem_total{app="${app}"}`
|
||||
|
||||
const [cpuResults, baselineResults, throttleResults, memResults, memTotalResults] =
|
||||
await Promise.all([
|
||||
queryRange(token, cpuQuery, start, now, step),
|
||||
queryRange(token, baselineQuery, start, now, step),
|
||||
queryRange(token, throttleQuery, start, now, step),
|
||||
queryRange(token, memQuery, start, now, step),
|
||||
queryRange(token, memTotalQuery, start, now, step),
|
||||
])
|
||||
|
||||
if (!cpuResults.length && !memResults.length) {
|
||||
console.error('No metrics found.')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
const lines: string[] = []
|
||||
lines.push(`Metrics for ${app} (last ${opts.last ?? '1h'}, step ${step})`)
|
||||
|
||||
// CPU time series
|
||||
for (const series of cpuResults) {
|
||||
const label = seriesLabel(series.metric)
|
||||
const values = series.values.map((v) => parseFloat(v[1]))
|
||||
const avg = values.reduce((a, b) => a + b, 0) / values.length
|
||||
const max = Math.max(...values)
|
||||
lines.push('')
|
||||
lines.push(`--- CPU [${label}] avg=${avg.toFixed(1)}% max=${max.toFixed(1)}% ---`)
|
||||
for (const [ts, val] of series.values) {
|
||||
lines.push(` ${formatTime(ts)} ${parseFloat(val).toFixed(1)}%`)
|
||||
}
|
||||
}
|
||||
|
||||
// CPU baseline
|
||||
for (const series of baselineResults) {
|
||||
const label = seriesLabel(series.metric)
|
||||
const values = series.values.map((v) => parseFloat(v[1]))
|
||||
const avg = values.reduce((a, b) => a + b, 0) / values.length
|
||||
lines.push('')
|
||||
lines.push(`--- CPU baseline [${label}] avg=${avg.toFixed(1)}% ---`)
|
||||
}
|
||||
|
||||
// CPU throttle
|
||||
for (const series of throttleResults) {
|
||||
const values = series.values.map((v) => parseFloat(v[1]))
|
||||
const throttled = values.filter((v) => v > 1).length
|
||||
if (throttled > 0) {
|
||||
lines.push('')
|
||||
lines.push(`--- CPU throttled: ${throttled}/${values.length} samples ---`)
|
||||
}
|
||||
}
|
||||
|
||||
// Memory time series
|
||||
for (const series of memResults) {
|
||||
const label = seriesLabel(series.metric)
|
||||
const values = series.values.map((v) => parseFloat(v[1]))
|
||||
const avg = values.reduce((a, b) => a + b, 0) / values.length
|
||||
const max = Math.max(...values)
|
||||
const totalSeries = memTotalResults.find((t) => t.metric.instance === series.metric.instance)
|
||||
const total = totalSeries?.values[0] ? parseFloat(totalSeries.values[0][1]) : 0
|
||||
const totalStr = total ? ` / ${formatBytes(total)}` : ''
|
||||
lines.push('')
|
||||
lines.push(`--- MEM [${label}] avg=${formatBytes(avg)} max=${formatBytes(max)}${totalStr} ---`)
|
||||
for (const [ts, val] of series.values) {
|
||||
const pct = total ? ` (${((parseFloat(val) / total) * 100).toFixed(1)}%)` : ''
|
||||
lines.push(` ${formatTime(ts)} ${formatBytes(parseFloat(val))}${pct}`)
|
||||
}
|
||||
}
|
||||
|
||||
const output = lines.join('\n')
|
||||
|
||||
if (opts.output) {
|
||||
const fs = await import('fs')
|
||||
fs.writeFileSync(opts.output, output + '\n')
|
||||
console.error(`Wrote to ${opts.output}`)
|
||||
} else {
|
||||
process.stdout.write(output + '\n')
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
321
internal/scripts/fetch-screenshot-metrics.ts
Normal file
321
internal/scripts/fetch-screenshot-metrics.ts
Normal file
@@ -0,0 +1,321 @@
|
||||
/**
|
||||
* Fetch board screenshot telemetry (the mcp_shared_board_screenshot dataset events) from the
|
||||
* Cloudflare Analytics Engine SQL API and report failure rate, timeout rate, cache hit rate, and
|
||||
* Browser Run render time per request source (mcp = MCP tool, og = GET og-image route, queue = async
|
||||
* render consumer).
|
||||
*
|
||||
* Render time is wall-clock time around each Browser Run capture (double3), not Cloudflare's billed
|
||||
* browser milliseconds: the BROWSER binding does not surface a per-render billed-ms figure (double4 is
|
||||
* always -1), so wall-clock render time is the available proxy for Browser Run spend.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx internal/scripts/fetch-screenshot-metrics.ts [options]
|
||||
*
|
||||
* Examples:
|
||||
* npx tsx internal/scripts/fetch-screenshot-metrics.ts --last 24h
|
||||
* npx tsx internal/scripts/fetch-screenshot-metrics.ts --worker main-tldraw-multiplayer --last 7d
|
||||
* npx tsx internal/scripts/fetch-screenshot-metrics.ts --check --max-failure-rate 0.2 --max-render-minutes 300
|
||||
*
|
||||
* With --check the script exits non-zero when a threshold is breached, so it can back a scheduled
|
||||
* alert (cron CI job, uptime monitor, etc.) without any extra infrastructure.
|
||||
*
|
||||
* Auth: set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_ANALYTICS_API_TOKEN (an API token with the
|
||||
* "Account Analytics: Read" permission; CLOUDFLARE_API_TOKEN works as a fallback).
|
||||
*/
|
||||
|
||||
const EVENT_NAME = 'mcp_shared_board_screenshot'
|
||||
|
||||
interface Options {
|
||||
lastHours: number
|
||||
worker?: string
|
||||
dataset: string
|
||||
json: boolean
|
||||
check: boolean
|
||||
maxFailureRate?: number
|
||||
maxTimeoutRate?: number
|
||||
minCacheHitRate?: number
|
||||
maxRenderMinutes?: number
|
||||
}
|
||||
|
||||
interface SourceMetrics {
|
||||
source: string
|
||||
requests: number
|
||||
failures: number
|
||||
timeouts: number
|
||||
rateLimited: number
|
||||
cacheHits: number
|
||||
cacheStale: number
|
||||
cacheMisses: number
|
||||
captures: number
|
||||
renderMs: number
|
||||
}
|
||||
|
||||
function parseArgs(): Options {
|
||||
const args = process.argv.slice(2)
|
||||
const opts: Options = {
|
||||
lastHours: 24,
|
||||
dataset: 'MEASURE',
|
||||
json: false,
|
||||
check: false,
|
||||
}
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i]
|
||||
if (arg === '--last' || arg === '-l') opts.lastHours = parseDurationHours(args[++i])
|
||||
else if (arg === '--worker' || arg === '-w') opts.worker = args[++i]
|
||||
else if (arg === '--dataset' || arg === '-d') opts.dataset = args[++i]
|
||||
else if (arg === '--json') opts.json = true
|
||||
else if (arg === '--check') opts.check = true
|
||||
else if (arg === '--max-failure-rate') opts.maxFailureRate = Number(args[++i])
|
||||
else if (arg === '--max-timeout-rate') opts.maxTimeoutRate = Number(args[++i])
|
||||
else if (arg === '--min-cache-hit-rate') opts.minCacheHitRate = Number(args[++i])
|
||||
// --max-browser-minutes is the former name; kept as an alias.
|
||||
else if (arg === '--max-render-minutes' || arg === '--max-browser-minutes')
|
||||
opts.maxRenderMinutes = Number(args[++i])
|
||||
else if (arg === '--help' || arg === '-h') {
|
||||
console.log(
|
||||
[
|
||||
'Usage: npx tsx internal/scripts/fetch-screenshot-metrics.ts [options]',
|
||||
'',
|
||||
'Options:',
|
||||
' --last, -l Look-back window, e.g. 1h, 24h, 7d (default: 24h)',
|
||||
' --worker, -w Filter to one worker name, e.g. main-tldraw-multiplayer',
|
||||
' --dataset, -d Analytics Engine dataset name (default: MEASURE)',
|
||||
' --json Print raw per-source metrics as JSON',
|
||||
' --check Exit non-zero when a threshold below is breached',
|
||||
' --max-failure-rate Threshold, 0-1 (default with --check: 0.2)',
|
||||
' --max-timeout-rate Threshold, 0-1 (default with --check: 0.1)',
|
||||
' --min-cache-hit-rate Threshold, 0-1 (no default)',
|
||||
' --max-render-minutes Browser Run render minutes in the window (no default)',
|
||||
' (alias: --max-browser-minutes)',
|
||||
' --help, -h Show this help',
|
||||
].join('\n')
|
||||
)
|
||||
process.exit(0)
|
||||
} else {
|
||||
console.error(`Unknown option: ${arg}`)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
if (opts.check) {
|
||||
opts.maxFailureRate ??= 0.2
|
||||
opts.maxTimeoutRate ??= 0.1
|
||||
}
|
||||
return opts
|
||||
}
|
||||
|
||||
function parseDurationHours(value: string | undefined): number {
|
||||
const match = value?.match(/^(\d+)([hd])$/)
|
||||
if (!match) {
|
||||
console.error(`Invalid duration: ${value}. Use e.g. 6h, 24h, 7d.`)
|
||||
process.exit(1)
|
||||
}
|
||||
const amount = Number(match[1])
|
||||
return match[2] === 'd' ? amount * 24 : amount
|
||||
}
|
||||
|
||||
function getAuth() {
|
||||
const accountId = process.env.CLOUDFLARE_ACCOUNT_ID
|
||||
const apiToken = process.env.CLOUDFLARE_ANALYTICS_API_TOKEN ?? process.env.CLOUDFLARE_API_TOKEN
|
||||
if (!accountId || !apiToken) {
|
||||
console.error(
|
||||
'Set CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_ANALYTICS_API_TOKEN (or CLOUDFLARE_API_TOKEN).'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
return { accountId, apiToken }
|
||||
}
|
||||
|
||||
async function queryAnalyticsEngine(sql: string): Promise<Record<string, string | number>[]> {
|
||||
const { accountId, apiToken } = getAuth()
|
||||
const response = await fetch(
|
||||
`https://api.cloudflare.com/client/v4/accounts/${accountId}/analytics_engine/sql`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: { Authorization: `Bearer ${apiToken}` },
|
||||
body: sql,
|
||||
}
|
||||
)
|
||||
if (!response.ok) {
|
||||
throw new Error(`Analytics Engine query failed (${response.status}): ${await response.text()}`)
|
||||
}
|
||||
const result = (await response.json()) as { data: Record<string, string | number>[] }
|
||||
return result.data
|
||||
}
|
||||
|
||||
// Only [0-9a-zA-Z_-] survives; every dynamic value interpolated into SQL goes through this.
|
||||
function sqlSafe(value: string) {
|
||||
return value.replace(/[^0-9a-zA-Z_-]/g, '')
|
||||
}
|
||||
|
||||
async function fetchMetrics(opts: Options): Promise<SourceMetrics[]> {
|
||||
const workerFilter = opts.worker ? `AND blob2 = '${sqlSafe(opts.worker)}'` : ''
|
||||
const where = `blob1 = '${EVENT_NAME}' ${workerFilter} AND timestamp > NOW() - INTERVAL '${opts.lastHours}' HOUR`
|
||||
const dataset = sqlSafe(opts.dataset)
|
||||
|
||||
// blob3 = source, blob4 = cache status, blob5 = failure reason, blob6 = rate limit decision.
|
||||
// _sample_interval corrects for Analytics Engine sampling.
|
||||
const rows = await queryAnalyticsEngine(`
|
||||
SELECT
|
||||
blob3 AS source,
|
||||
blob4 AS cache,
|
||||
blob5 AS failure,
|
||||
blob6 AS rate_limit,
|
||||
SUM(_sample_interval) AS requests
|
||||
FROM ${dataset}
|
||||
WHERE ${where}
|
||||
GROUP BY source, cache, failure, rate_limit
|
||||
FORMAT JSON
|
||||
`)
|
||||
|
||||
// double3 is the wall-clock render duration, positive only on rows that actually invoked Browser
|
||||
// Run (cache hits and pre-render failures leave it at -1). It is the spend proxy: double4 holds
|
||||
// billed browser ms, which the BROWSER binding does not surface, so it is always -1.
|
||||
//
|
||||
// Failed captures count here too, deliberately: a render that threw still created a browser and
|
||||
// held it, often for the whole timeout, so excluding it would understate spend by exactly the
|
||||
// renders most worth knowing the cost of. Queue rows are per delivery, so a job that retried
|
||||
// three times contributes three captures — again, that is what it spent.
|
||||
const spendRows = await queryAnalyticsEngine(`
|
||||
SELECT
|
||||
blob3 AS source,
|
||||
SUM(_sample_interval) AS captures,
|
||||
SUM(double3 * _sample_interval) AS render_ms
|
||||
FROM ${dataset}
|
||||
WHERE ${where} AND double3 > 0
|
||||
GROUP BY source
|
||||
FORMAT JSON
|
||||
`)
|
||||
|
||||
const bySource = new Map<string, SourceMetrics>()
|
||||
const getSource = (source: string) => {
|
||||
let metrics = bySource.get(source)
|
||||
if (!metrics) {
|
||||
metrics = {
|
||||
source,
|
||||
requests: 0,
|
||||
failures: 0,
|
||||
timeouts: 0,
|
||||
rateLimited: 0,
|
||||
cacheHits: 0,
|
||||
cacheStale: 0,
|
||||
cacheMisses: 0,
|
||||
captures: 0,
|
||||
renderMs: 0,
|
||||
}
|
||||
bySource.set(source, metrics)
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
const metrics = getSource(String(row.source).replace(/^source:/, ''))
|
||||
const requests = Number(row.requests)
|
||||
const failure = String(row.failure).replace(/^failure:/, '')
|
||||
const cache = String(row.cache).replace(/^cache:/, '')
|
||||
metrics.requests += requests
|
||||
if (failure !== 'none') metrics.failures += requests
|
||||
if (/timeout|abort/i.test(failure)) metrics.timeouts += requests
|
||||
if (String(row.rate_limit) === 'rate_limit:blocked') metrics.rateLimited += requests
|
||||
if (cache === 'hit') metrics.cacheHits += requests
|
||||
if (cache === 'stale') metrics.cacheStale += requests
|
||||
if (cache === 'miss') metrics.cacheMisses += requests
|
||||
}
|
||||
for (const row of spendRows) {
|
||||
const metrics = getSource(String(row.source).replace(/^source:/, ''))
|
||||
metrics.captures += Number(row.captures)
|
||||
metrics.renderMs += Number(row.render_ms)
|
||||
}
|
||||
|
||||
return [...bySource.values()].sort((a, b) => a.source.localeCompare(b.source))
|
||||
}
|
||||
|
||||
function rate(part: number, total: number) {
|
||||
return total === 0 ? 0 : part / total
|
||||
}
|
||||
|
||||
function formatPercent(value: number) {
|
||||
return `${(value * 100).toFixed(1)}%`
|
||||
}
|
||||
|
||||
function main() {
|
||||
const opts = parseArgs()
|
||||
return fetchMetrics(opts).then((metrics) => {
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(metrics, null, 2))
|
||||
} else {
|
||||
console.log(
|
||||
`Screenshot telemetry, last ${opts.lastHours}h${opts.worker ? ` (worker ${opts.worker})` : ''}:`
|
||||
)
|
||||
if (metrics.length === 0) console.log(' no events in window')
|
||||
for (const m of metrics) {
|
||||
console.log(
|
||||
[
|
||||
` ${m.source.padEnd(6)}`,
|
||||
`requests ${m.requests}`,
|
||||
`failure ${formatPercent(rate(m.failures, m.requests))}`,
|
||||
`timeout ${formatPercent(rate(m.timeouts, m.requests))}`,
|
||||
`cache hit ${formatPercent(rate(m.cacheHits, m.cacheHits + m.cacheStale + m.cacheMisses))}`,
|
||||
`rate-limited ${m.rateLimited}`,
|
||||
`captures ${m.captures}`,
|
||||
`render ${(m.renderMs / 60_000).toFixed(1)}min`,
|
||||
].join(' ')
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if (!opts.check) return
|
||||
|
||||
const totals = metrics.reduce(
|
||||
(sum, m) => ({
|
||||
requests: sum.requests + m.requests,
|
||||
failures: sum.failures + m.failures,
|
||||
timeouts: sum.timeouts + m.timeouts,
|
||||
cacheHits: sum.cacheHits + m.cacheHits,
|
||||
cacheLookups: sum.cacheLookups + m.cacheHits + m.cacheStale + m.cacheMisses,
|
||||
renderMs: sum.renderMs + m.renderMs,
|
||||
}),
|
||||
{ requests: 0, failures: 0, timeouts: 0, cacheHits: 0, cacheLookups: 0, renderMs: 0 }
|
||||
)
|
||||
|
||||
const breaches: string[] = []
|
||||
const failureRate = rate(totals.failures, totals.requests)
|
||||
const timeoutRate = rate(totals.timeouts, totals.requests)
|
||||
const cacheHitRate = rate(totals.cacheHits, totals.cacheLookups)
|
||||
const renderMinutes = totals.renderMs / 60_000
|
||||
if (opts.maxFailureRate !== undefined && failureRate > opts.maxFailureRate) {
|
||||
breaches.push(
|
||||
`failure rate ${formatPercent(failureRate)} > ${formatPercent(opts.maxFailureRate)}`
|
||||
)
|
||||
}
|
||||
if (opts.maxTimeoutRate !== undefined && timeoutRate > opts.maxTimeoutRate) {
|
||||
breaches.push(
|
||||
`timeout rate ${formatPercent(timeoutRate)} > ${formatPercent(opts.maxTimeoutRate)}`
|
||||
)
|
||||
}
|
||||
if (
|
||||
opts.minCacheHitRate !== undefined &&
|
||||
totals.cacheLookups > 0 &&
|
||||
cacheHitRate < opts.minCacheHitRate
|
||||
) {
|
||||
breaches.push(
|
||||
`cache hit rate ${formatPercent(cacheHitRate)} < ${formatPercent(opts.minCacheHitRate)}`
|
||||
)
|
||||
}
|
||||
if (opts.maxRenderMinutes !== undefined && renderMinutes > opts.maxRenderMinutes) {
|
||||
breaches.push(
|
||||
`Browser Run render time ${renderMinutes.toFixed(1)}min > ${opts.maxRenderMinutes}min`
|
||||
)
|
||||
}
|
||||
|
||||
if (breaches.length > 0) {
|
||||
console.error(`ALERT: ${breaches.join('; ')}`)
|
||||
process.exit(1)
|
||||
}
|
||||
console.log('All thresholds OK')
|
||||
})
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : error)
|
||||
process.exit(1)
|
||||
})
|
||||
38
internal/scripts/fly-common.ts
Normal file
38
internal/scripts/fly-common.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* Shared utilities for Fly.io scripts.
|
||||
*
|
||||
* App names:
|
||||
* production-zero-rm Production replication manager
|
||||
* production-zero-vs Production view syncer
|
||||
* staging-zero-rm Staging replication manager
|
||||
* staging-zero-vs Staging view syncer
|
||||
* pr-NNNN-zero-cache PR preview (single zero-cache process)
|
||||
*/
|
||||
|
||||
export const FLY_ORG_SLUG = process.env.FLY_ORG_SLUG ?? 'tldraw-gb-ltd'
|
||||
|
||||
export function getFlyToken(explicit?: string): string {
|
||||
let token = explicit ?? process.env.FLY_TOKEN
|
||||
if (!token) {
|
||||
console.error(
|
||||
'No token provided. Use --token, FLY_TOKEN env var, or create one with: fly tokens create readonly'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
// Strip "FlyV1 " prefix if present — callers add it themselves
|
||||
if (token.startsWith('FlyV1 ')) token = token.slice(6)
|
||||
return token
|
||||
}
|
||||
|
||||
export function parseDuration(s: string, unit: 'ms' | 's' = 's'): number {
|
||||
const match = s.match(/^(\d+)(m|h|d)$/)
|
||||
if (!match) {
|
||||
console.error(`Invalid duration: ${s}. Use e.g. 30m, 2h, 1d`)
|
||||
process.exit(1)
|
||||
}
|
||||
const n = parseInt(match[1])
|
||||
const u = match[2]
|
||||
const multipliers: Record<string, number> = { m: 60, h: 3_600, d: 86_400 }
|
||||
const sec = n * multipliers[u]
|
||||
return unit === 'ms' ? sec * 1000 : sec
|
||||
}
|
||||
257
internal/scripts/generate-test-licenses.ts
Normal file
257
internal/scripts/generate-test-licenses.ts
Normal file
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env npx tsx
|
||||
|
||||
/**
|
||||
* Test License Key Generator
|
||||
*
|
||||
* This script generates various license keys for manual testing of the LicenseManager.
|
||||
* Uses the existing test utilities from LicenseManager.test.ts
|
||||
*/
|
||||
|
||||
import crypto from 'crypto'
|
||||
import { FLAGS } from '../../packages/editor/src/lib/license/LicenseManager'
|
||||
import { str2ab } from '../../packages/editor/src/lib/utils/licensing'
|
||||
// Utility function copied from LicenseManager.test.ts to avoid import issues
|
||||
function ab2str(buf: ArrayBuffer) {
|
||||
return String.fromCharCode.apply(null, Array.from(new Uint8Array(buf)))
|
||||
}
|
||||
|
||||
// Utility functions copied from LicenseManager.test.ts (not exported)
|
||||
async function generateLicenseKey(
|
||||
message: string,
|
||||
keyPair: { publicKey: string; privateKey: string }
|
||||
) {
|
||||
const enc = new TextEncoder()
|
||||
const encodedMsg = enc.encode(message)
|
||||
const privateKey = await importPrivateKey(keyPair.privateKey)
|
||||
|
||||
const signedLicenseKeyBuffer = await crypto.subtle.sign(
|
||||
{
|
||||
name: 'ECDSA',
|
||||
hash: { name: 'SHA-256' },
|
||||
},
|
||||
privateKey,
|
||||
encodedMsg
|
||||
)
|
||||
|
||||
const signature = btoa(ab2str(signedLicenseKeyBuffer))
|
||||
const prefix = 'tldraw-'
|
||||
const licenseKey = `${prefix}/${btoa(message)}.${signature}`
|
||||
|
||||
return licenseKey
|
||||
}
|
||||
|
||||
function importPrivateKey(pemContents: string) {
|
||||
const binaryDerString = atob(pemContents)
|
||||
const binaryDer = str2ab(binaryDerString)
|
||||
|
||||
return crypto.subtle.importKey(
|
||||
'pkcs8',
|
||||
new Uint8Array(binaryDer),
|
||||
{
|
||||
name: 'ECDSA',
|
||||
namedCurve: 'P-256',
|
||||
},
|
||||
true,
|
||||
['sign']
|
||||
)
|
||||
}
|
||||
|
||||
async function exportCryptoKey(key: CryptoKey, isPublic = false) {
|
||||
const exported = await crypto.subtle.exportKey(isPublic ? 'spki' : 'pkcs8', key)
|
||||
return btoa(ab2str(exported))
|
||||
}
|
||||
|
||||
// Generate a fresh key pair for each run
|
||||
async function generateKeyPair() {
|
||||
const keyPair = await crypto.subtle.generateKey(
|
||||
{
|
||||
name: 'ECDSA',
|
||||
namedCurve: 'P-256',
|
||||
},
|
||||
true,
|
||||
['sign', 'verify']
|
||||
)
|
||||
const publicKey = await exportCryptoKey(keyPair.publicKey, true /* isPublic */)
|
||||
const privateKey = await exportCryptoKey(keyPair.privateKey)
|
||||
|
||||
return { publicKey, privateKey }
|
||||
}
|
||||
|
||||
function createLicenseInfo(id: string, hosts: string[], flags: number, expiryDate: string) {
|
||||
return JSON.stringify([id, hosts, flags, expiryDate])
|
||||
}
|
||||
|
||||
function getDateOffset(days: number): string {
|
||||
const date = new Date()
|
||||
date.setDate(date.getDate() + days)
|
||||
return date.toISOString().slice(0, 10)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('🔑 Generating test license keys for LicenseManager testing...\n')
|
||||
|
||||
console.log('Generating fresh key pair for testing...')
|
||||
const keyPair = await generateKeyPair()
|
||||
console.log('Public Key:', keyPair.publicKey)
|
||||
console.log('Private Key:', keyPair.privateKey)
|
||||
console.log('')
|
||||
|
||||
const testCases = [
|
||||
// === VALID LICENSES ===
|
||||
{
|
||||
name: 'Valid Annual License',
|
||||
info: createLicenseInfo(
|
||||
'test-annual-valid',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE,
|
||||
getDateOffset(60) // 2 months future
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Valid Perpetual License',
|
||||
info: createLicenseInfo(
|
||||
'test-perpetual-valid',
|
||||
['localhost'],
|
||||
FLAGS.PERPETUAL_LICENSE,
|
||||
getDateOffset(365) // 1 year future
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Valid Evaluation License',
|
||||
info: createLicenseInfo(
|
||||
'test-eval-valid',
|
||||
['localhost'],
|
||||
FLAGS.EVALUATION_LICENSE,
|
||||
getDateOffset(14) // 2 weeks future
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Valid Internal License',
|
||||
info: createLicenseInfo(
|
||||
'test-internal-valid',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE | FLAGS.INTERNAL_LICENSE,
|
||||
getDateOffset(60) // 2 months future
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'License with Watermark Flag',
|
||||
info: createLicenseInfo(
|
||||
'test-watermark-only',
|
||||
['localhost'],
|
||||
FLAGS.WITH_WATERMARK,
|
||||
getDateOffset(60) // 2 months future
|
||||
),
|
||||
},
|
||||
|
||||
// === FEATURE FLAGS ===
|
||||
{
|
||||
name: 'License with Commenting Feature',
|
||||
info: createLicenseInfo(
|
||||
'test-feat-commenting',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE | FLAGS.FEAT_COMMENTING,
|
||||
getDateOffset(60) // 2 months future
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'License with Collaboration Feature (grants commenting)',
|
||||
info: createLicenseInfo(
|
||||
'test-feat-collaboration',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE | FLAGS.FEAT_COLLABORATION,
|
||||
getDateOffset(60) // 2 months future
|
||||
),
|
||||
},
|
||||
|
||||
// === GRACE PERIOD TESTING ===
|
||||
{
|
||||
name: 'Annual License (expired 15 days - within grace period)',
|
||||
info: createLicenseInfo(
|
||||
'test-grace-15days',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE,
|
||||
getDateOffset(-15) // 15 days past
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Annual License (expired 45 days - beyond grace period)',
|
||||
info: createLicenseInfo(
|
||||
'test-annual-expired-45days',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE,
|
||||
getDateOffset(-45) // 45 days past
|
||||
),
|
||||
},
|
||||
|
||||
// === EXPIRED INTERNAL LICENSES ===
|
||||
{
|
||||
name: 'Internal License (expired 15 days - within grace period)',
|
||||
info: createLicenseInfo(
|
||||
'test-internal-expired-15days',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE | FLAGS.INTERNAL_LICENSE,
|
||||
getDateOffset(-15) // 15 days past
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Internal License (expired 45 days - beyond grace period)',
|
||||
info: createLicenseInfo(
|
||||
'test-internal-expired-45days',
|
||||
['localhost'],
|
||||
FLAGS.ANNUAL_LICENSE | FLAGS.INTERNAL_LICENSE,
|
||||
getDateOffset(-45) // 45 days past
|
||||
),
|
||||
},
|
||||
|
||||
// === EXPIRED EVALUATION LICENSES ===
|
||||
{
|
||||
name: 'Evaluation License (expired 1 day)',
|
||||
info: createLicenseInfo(
|
||||
'test-eval-expired-1day',
|
||||
['localhost'],
|
||||
FLAGS.EVALUATION_LICENSE,
|
||||
getDateOffset(-1) // 1 day past
|
||||
),
|
||||
},
|
||||
{
|
||||
name: 'Evaluation License (expired 15 days)',
|
||||
info: createLicenseInfo(
|
||||
'test-eval-expired-15days',
|
||||
['localhost'],
|
||||
FLAGS.EVALUATION_LICENSE,
|
||||
getDateOffset(-15) // 15 days past
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
console.log('🎫 Generated Test License Keys:\n')
|
||||
|
||||
for (const testCase of testCases) {
|
||||
try {
|
||||
const licenseKey = await generateLicenseKey(testCase.info, keyPair)
|
||||
console.log(`## ${testCase.name}`)
|
||||
console.log(`License Info: ${testCase.info}`)
|
||||
console.log(`License Key: ${licenseKey}`)
|
||||
console.log('')
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to generate license for ${testCase.name}:`, error)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('📝 Usage Instructions:')
|
||||
console.log('')
|
||||
console.log('🚨 DO NOT COMMIT THESE KEYS TO THE REPOSITORY! 🚨')
|
||||
console.log('These are temporary test keys - generate fresh ones for each session')
|
||||
console.log('')
|
||||
console.log('Option 1 - Update LicenseManager (development):')
|
||||
console.log(` Replace publicKey in LicenseManager.ts with: "${keyPair.publicKey}"`)
|
||||
console.log(' Then use: <Tldraw licenseKey="license-key-from-above" />')
|
||||
console.log('')
|
||||
console.log('Option 2 - Pass testPublicKey (automated testing):')
|
||||
console.log(` new LicenseManager("license-key", "${keyPair.publicKey}", "production")`)
|
||||
console.log('')
|
||||
console.log('ℹ️ For production licenses: sales@tldraw.com')
|
||||
}
|
||||
|
||||
main().catch(console.error)
|
||||
334
internal/scripts/generate-tldraw-package-docs.ts
Normal file
334
internal/scripts/generate-tldraw-package-docs.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { fileURLToPath, pathToFileURL } from 'url'
|
||||
|
||||
const DOCS_SECTION_IDS = ['getting-started', 'docs', 'starter-kits', 'sdk-features', 'community']
|
||||
const RELEASES_SECTION_ID = 'releases'
|
||||
const RELEASE_NOTES_FILE_NAME = /^v\d+\.\d+\.\d+\.mdx$/
|
||||
const NEXT_RELEASE_NOTES_FILE_NAME = 'next.mdx'
|
||||
const UNWRAPPED_MDX_TAGS = new Set(['Callout', 'Feature', 'FocusLines', 'div'])
|
||||
const OMITTED_MDX_TAGS = new Set(['Image', 'MaskedArticleImage', 'StarterKitBento', 'img'])
|
||||
|
||||
interface Article {
|
||||
content: string
|
||||
fileName: string
|
||||
order: number | null
|
||||
sectionId: string
|
||||
title: string
|
||||
version: [number, number, number] | null
|
||||
}
|
||||
|
||||
interface Section {
|
||||
id: string
|
||||
title: string
|
||||
description?: string
|
||||
}
|
||||
|
||||
export function generateTldrawPackageDocs({
|
||||
sourcePackageDir,
|
||||
repoRoot = path.resolve(sourcePackageDir, '../..'),
|
||||
}: {
|
||||
sourcePackageDir: string
|
||||
repoRoot?: string
|
||||
}) {
|
||||
const packageJsonPath = path.join(sourcePackageDir, 'package.json')
|
||||
if (!existsSync(packageJsonPath)) {
|
||||
throw new Error(`No package.json file found in '${sourcePackageDir}'!`)
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(readFileSync(packageJsonPath, 'utf8'))
|
||||
const packageVersion = manifest.version
|
||||
const contentDir = path.join(repoRoot, 'apps/docs/content')
|
||||
const sections = readSections(contentDir)
|
||||
|
||||
writeFileSync(
|
||||
path.join(sourcePackageDir, 'DOCS.md'),
|
||||
createDocsRollup({ contentDir, packageVersion, sections })
|
||||
)
|
||||
writeFileSync(
|
||||
path.join(sourcePackageDir, 'RELEASE_NOTES.md'),
|
||||
createReleaseNotesRollup({ contentDir, packageVersion, sections })
|
||||
)
|
||||
}
|
||||
|
||||
export function createDocsRollup({
|
||||
contentDir,
|
||||
packageVersion,
|
||||
sections,
|
||||
}: {
|
||||
contentDir: string
|
||||
packageVersion: string
|
||||
sections: Section[]
|
||||
}) {
|
||||
const result = [
|
||||
'# tldraw Documentation',
|
||||
'',
|
||||
`Version: \`${packageVersion}\``,
|
||||
'',
|
||||
'This file is generated during package publishing from the tldraw docs content for this exact package version.',
|
||||
]
|
||||
|
||||
for (const sectionId of DOCS_SECTION_IDS) {
|
||||
const section = sectionById(sections, sectionId)
|
||||
const articles = readArticles(contentDir, sectionId)
|
||||
if (!articles.length) continue
|
||||
|
||||
result.push('', `## ${section.title}`)
|
||||
if (section.description) result.push('', section.description)
|
||||
|
||||
for (const article of articles) {
|
||||
result.push('', `### ${article.title}`, '', normalizeMdxForMarkdown(article.content, 2))
|
||||
}
|
||||
}
|
||||
|
||||
return result.join('\n').trimEnd() + '\n'
|
||||
}
|
||||
|
||||
export function createReleaseNotesRollup({
|
||||
contentDir,
|
||||
packageVersion,
|
||||
sections,
|
||||
}: {
|
||||
contentDir: string
|
||||
packageVersion: string
|
||||
sections: Section[]
|
||||
}) {
|
||||
const section = sectionById(sections, RELEASES_SECTION_ID)
|
||||
const releaseNotes = readReleaseNotesArticles(contentDir, packageVersion)
|
||||
.filter((article): article is Article & { version: [number, number, number] } =>
|
||||
Boolean(article.version)
|
||||
)
|
||||
.sort((a, b) => compareVersions(b.version, a.version))
|
||||
|
||||
const result = [
|
||||
`# ${section.title}`,
|
||||
'',
|
||||
`Version: \`${packageVersion}\``,
|
||||
'',
|
||||
'This file is generated during package publishing from the tldraw release notes content for this exact package version.',
|
||||
]
|
||||
|
||||
for (const article of releaseNotes) {
|
||||
result.push('', `## ${article.title}`, '', normalizeMdxForMarkdown(article.content, 1))
|
||||
}
|
||||
|
||||
return result.join('\n').trimEnd() + '\n'
|
||||
}
|
||||
|
||||
function readReleaseNotesArticles(contentDir: string, packageVersion: string) {
|
||||
const releaseNotes = readArticles(contentDir, RELEASES_SECTION_ID)
|
||||
const packageVersionTuple = getStableVersionTuple(packageVersion)
|
||||
|
||||
if (packageVersionTuple) {
|
||||
const packageVersionTitle = `v${packageVersionTuple.join('.')}`
|
||||
const nextReleaseNotes = readArticle(
|
||||
path.join(contentDir, RELEASES_SECTION_ID, NEXT_RELEASE_NOTES_FILE_NAME),
|
||||
RELEASES_SECTION_ID,
|
||||
NEXT_RELEASE_NOTES_FILE_NAME
|
||||
)
|
||||
if (
|
||||
!releaseNotes.some((article) => article.title === packageVersionTitle) &&
|
||||
nextReleaseNotes
|
||||
) {
|
||||
releaseNotes.push({
|
||||
...nextReleaseNotes,
|
||||
fileName: `${packageVersionTitle}.mdx`,
|
||||
title: packageVersionTitle,
|
||||
version: packageVersionTuple,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return releaseNotes
|
||||
}
|
||||
|
||||
function readSections(contentDir: string): Section[] {
|
||||
return JSON.parse(readFileSync(path.join(contentDir, 'sections.json'), 'utf8'))
|
||||
}
|
||||
|
||||
function sectionById(sections: Section[], sectionId: string) {
|
||||
const section = sections.find((section) => section.id === sectionId)
|
||||
if (!section) {
|
||||
throw new Error(`Could not find docs section '${sectionId}'`)
|
||||
}
|
||||
return section
|
||||
}
|
||||
|
||||
function readArticles(contentDir: string, sectionId: string) {
|
||||
const sectionDir = path.join(contentDir, sectionId)
|
||||
return readdirSync(sectionDir)
|
||||
.filter((fileName) => fileName.endsWith('.mdx') || fileName.endsWith('.md'))
|
||||
.filter(
|
||||
(fileName) => sectionId !== RELEASES_SECTION_ID || RELEASE_NOTES_FILE_NAME.test(fileName)
|
||||
)
|
||||
.map((fileName) => readArticle(path.join(sectionDir, fileName), sectionId, fileName))
|
||||
.filter((article) => article !== null)
|
||||
.sort(sortArticles)
|
||||
}
|
||||
|
||||
function readArticle(filePath: string, sectionId: string, fileName: string): Article | null {
|
||||
const { data, content } = parseFrontMatter(readFileSync(filePath, 'utf8'))
|
||||
if (data.status !== 'published') return null
|
||||
|
||||
const title = data.title ?? path.basename(fileName, path.extname(fileName))
|
||||
const order = data.order ? Number(data.order) : null
|
||||
const versionMatch = fileName.match(/^v(\d+)\.(\d+)\.(\d+)\.mdx$/)
|
||||
|
||||
return {
|
||||
content,
|
||||
fileName,
|
||||
order: Number.isFinite(order) ? order : null,
|
||||
sectionId,
|
||||
title,
|
||||
version: versionMatch
|
||||
? [Number(versionMatch[1]), Number(versionMatch[2]), Number(versionMatch[3])]
|
||||
: null,
|
||||
}
|
||||
}
|
||||
|
||||
function parseFrontMatter(raw: string): { data: Record<string, string>; content: string } {
|
||||
const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/)
|
||||
if (!match) return { data: {}, content: raw }
|
||||
|
||||
const data: Record<string, string> = {}
|
||||
for (const line of match[1].split(/\r?\n/)) {
|
||||
const lineMatch = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/)
|
||||
if (!lineMatch) continue
|
||||
|
||||
const [, key, rawValue] = lineMatch
|
||||
let value = rawValue.trim()
|
||||
if (
|
||||
(value.startsWith("'") && value.endsWith("'")) ||
|
||||
(value.startsWith('"') && value.endsWith('"'))
|
||||
) {
|
||||
value = value.slice(1, -1)
|
||||
}
|
||||
data[key] = value
|
||||
}
|
||||
|
||||
return { data, content: raw.slice(match[0].length).trim() }
|
||||
}
|
||||
|
||||
function sortArticles(a: Article, b: Article) {
|
||||
const orderA = a.order ?? Number.MAX_SAFE_INTEGER
|
||||
const orderB = b.order ?? Number.MAX_SAFE_INTEGER
|
||||
if (orderA !== orderB) return orderA - orderB
|
||||
return a.title.localeCompare(b.title)
|
||||
}
|
||||
|
||||
function compareVersions(a: [number, number, number], b: [number, number, number]) {
|
||||
return a[0] - b[0] || a[1] - b[1] || a[2] - b[2]
|
||||
}
|
||||
|
||||
function getStableVersionTuple(packageVersion: string): [number, number, number] | null {
|
||||
const match = packageVersion.match(/^(\d+)\.(\d+)\.(\d+)$/)
|
||||
return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null
|
||||
}
|
||||
|
||||
function normalizeMdxForMarkdown(content: string, headingDemotion: number) {
|
||||
const result: string[] = []
|
||||
let inFence = false
|
||||
let omittedMdxTag: string | null = null
|
||||
|
||||
for (const line of content.split(/\r?\n/)) {
|
||||
const trimmed = line.trim()
|
||||
if (/^(```|~~~)/.test(trimmed)) {
|
||||
inFence = !inFence
|
||||
result.push(line)
|
||||
continue
|
||||
}
|
||||
|
||||
if (!inFence) {
|
||||
if (omittedMdxTag) {
|
||||
if (trimmed.includes('/>') || trimmed.includes(`</${omittedMdxTag}>`)) {
|
||||
omittedMdxTag = null
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (/^import\s.+\sfrom\s/.test(trimmed)) continue
|
||||
if (/^export\s+(const|function|class|type|interface|default|\{)/.test(trimmed)) continue
|
||||
|
||||
const mdxLine = normalizeMdxLine(trimmed)
|
||||
if (mdxLine.kind === 'omit') {
|
||||
omittedMdxTag = mdxLine.tag
|
||||
continue
|
||||
}
|
||||
if (mdxLine.kind === 'skip') continue
|
||||
if (mdxLine.kind === 'replace') {
|
||||
result.push(rewriteLinks(demoteHeading(mdxLine.content, headingDemotion)))
|
||||
continue
|
||||
}
|
||||
|
||||
result.push(rewriteLinks(demoteHeading(line, headingDemotion)))
|
||||
} else {
|
||||
result.push(line)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
.join('\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim()
|
||||
}
|
||||
|
||||
function normalizeMdxLine(
|
||||
trimmed: string
|
||||
):
|
||||
| { kind: 'keep' }
|
||||
| { kind: 'skip' }
|
||||
| { kind: 'omit'; tag: string }
|
||||
| { kind: 'replace'; content: string } {
|
||||
const checkItemMatch = trimmed.match(/^<CheckItem>(.*?)<\/CheckItem>$/)
|
||||
if (checkItemMatch) {
|
||||
return { kind: 'replace', content: `- ${checkItemMatch[1]}` }
|
||||
}
|
||||
|
||||
const tagName = getMdxTagName(trimmed)
|
||||
if (!tagName) return { kind: 'keep' }
|
||||
if (tagName === 'br') return { kind: 'skip' }
|
||||
if (OMITTED_MDX_TAGS.has(tagName)) {
|
||||
return trimmed.includes('/>') || trimmed.includes(`</${tagName}>`)
|
||||
? { kind: 'skip' }
|
||||
: { kind: 'omit', tag: tagName }
|
||||
}
|
||||
if (UNWRAPPED_MDX_TAGS.has(tagName)) {
|
||||
const title = tagName === 'Feature' ? getMdxStringProp(trimmed, 'title') : null
|
||||
return title ? { kind: 'replace', content: `**${title}**` } : { kind: 'skip' }
|
||||
}
|
||||
if (/^<\/?[A-Z][A-Za-z0-9.]*\b/.test(trimmed)) {
|
||||
return { kind: 'skip' }
|
||||
}
|
||||
|
||||
return { kind: 'keep' }
|
||||
}
|
||||
|
||||
function getMdxTagName(trimmed: string) {
|
||||
const match = trimmed.match(/^<\/?([A-Za-z][A-Za-z0-9.]*)\b/)
|
||||
return match?.[1] ?? null
|
||||
}
|
||||
|
||||
function getMdxStringProp(trimmed: string, propName: string) {
|
||||
const match = trimmed.match(new RegExp(`\\b${propName}=(?:"([^"]+)"|'([^']+)')`))
|
||||
return match?.[1] ?? match?.[2] ?? null
|
||||
}
|
||||
|
||||
function demoteHeading(line: string, headingDemotion: number) {
|
||||
return line.replace(/^(#{1,6})(\s+)/, (_, hashes, spacing) => {
|
||||
return '#'.repeat(Math.min(6, hashes.length + headingDemotion)) + spacing
|
||||
})
|
||||
}
|
||||
|
||||
function rewriteLinks(line: string) {
|
||||
return line
|
||||
.replace(/\[`?([^[\]]*?)`?\]\(\?\)/g, '`$1`')
|
||||
.replace(/\]\((\/[^)\s]+)\)/g, '](https://tldraw.dev$1)')
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const scriptDir = path.dirname(fileURLToPath(import.meta.url))
|
||||
generateTldrawPackageDocs({
|
||||
sourcePackageDir: path.resolve(path.normalize(process.argv[2] ?? process.cwd())),
|
||||
repoRoot: path.resolve(scriptDir, '../..'),
|
||||
})
|
||||
}
|
||||
219
internal/scripts/i18n-download-strings.ts
Normal file
219
internal/scripts/i18n-download-strings.ts
Normal file
@@ -0,0 +1,219 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import JSZip from 'jszip'
|
||||
|
||||
const LOKALISE_API_BASE = 'https://api.lokalise.com/api2'
|
||||
const ASYNC_EXPORT_POLL_INTERVAL_MS = 2000
|
||||
const ASYNC_EXPORT_MAX_POLLS = 120
|
||||
|
||||
function formatError(error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
return error.stack ?? error.message
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.stringify(error, null, 2)
|
||||
} catch {
|
||||
return String(error)
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
function getEnv(name: string) {
|
||||
const value = process.env[name]
|
||||
if (!value) throw new Error(`Missing required env var: ${name}`)
|
||||
return value
|
||||
}
|
||||
|
||||
async function lokaliseRequest<T>(path: string, init: RequestInit, apiKey: string): Promise<T> {
|
||||
const response = await fetch(`${LOKALISE_API_BASE}${path}`, {
|
||||
...init,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Api-Token': apiKey,
|
||||
...(init.headers ?? {}),
|
||||
},
|
||||
})
|
||||
|
||||
const body = (await response.json()) as unknown
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Lokalise request failed (${response.status}): ${JSON.stringify(body, null, 2)}`
|
||||
)
|
||||
}
|
||||
|
||||
return body as T
|
||||
}
|
||||
|
||||
async function getAsyncBundleUrl(
|
||||
projectId: string,
|
||||
options: Record<string, unknown>,
|
||||
apiKey: string
|
||||
) {
|
||||
const start = await lokaliseRequest<{ process_id?: string }>(
|
||||
`/projects/${projectId}/files/async-download`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: JSON.stringify(options),
|
||||
},
|
||||
apiKey
|
||||
)
|
||||
|
||||
if (!start.process_id) {
|
||||
throw new Error(`Async export did not return process_id: ${JSON.stringify(start, null, 2)}`)
|
||||
}
|
||||
|
||||
for (let i = 0; i < ASYNC_EXPORT_MAX_POLLS; i++) {
|
||||
const processResult = await lokaliseRequest<{
|
||||
process?: {
|
||||
status?: string
|
||||
message?: string
|
||||
details?: { download_url?: string; bundle_url?: string }
|
||||
}
|
||||
}>(`/projects/${projectId}/processes/${start.process_id}`, { method: 'GET' }, apiKey)
|
||||
|
||||
const status = processResult.process?.status
|
||||
const details = processResult.process?.details
|
||||
const bundleUrl = details?.download_url ?? details?.bundle_url
|
||||
|
||||
if (status === 'finished') {
|
||||
if (!bundleUrl) {
|
||||
throw new Error(
|
||||
`Async export finished but no bundle URL was returned: ${JSON.stringify(processResult, null, 2)}`
|
||||
)
|
||||
}
|
||||
return bundleUrl
|
||||
}
|
||||
|
||||
if (status === 'failed' || status === 'cancelled') {
|
||||
throw new Error(
|
||||
`Async export ${status}: ${processResult.process?.message ?? 'No message'}\n${JSON.stringify(processResult, null, 2)}`
|
||||
)
|
||||
}
|
||||
|
||||
await sleep(ASYNC_EXPORT_POLL_INTERVAL_MS)
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Async export timed out after ${ASYNC_EXPORT_MAX_POLLS} polls for project ${projectId}`
|
||||
)
|
||||
}
|
||||
|
||||
async function fetchAndSave(
|
||||
url: string,
|
||||
dirPath: string,
|
||||
sortJson: boolean,
|
||||
fileResolver: (fileName: string) => string,
|
||||
space: string | number,
|
||||
skipEnglish = false
|
||||
) {
|
||||
const bundleZip = await fetch(url)
|
||||
// Need to unzip the bundle
|
||||
const zipData = await bundleZip.arrayBuffer()
|
||||
const zip = new JSZip()
|
||||
const bundle = await zip.loadAsync(zipData)
|
||||
const locales = Object.keys(bundle.files)
|
||||
for (const locale of locales) {
|
||||
if (bundle.files[locale].dir) {
|
||||
continue
|
||||
}
|
||||
const fileName = fileResolver(bundle.files[locale].name)
|
||||
if (skipEnglish && fileName === 'en.json') {
|
||||
continue
|
||||
}
|
||||
const filePath = path.resolve(dirPath, fileName)
|
||||
const file = await bundle.files[locale].async('text')
|
||||
const json = JSON.parse(file)
|
||||
|
||||
let finalJson = json
|
||||
if (sortJson) {
|
||||
// Sort of annoying, the property order is not guaranteed, so we need to alphabetize the keys.
|
||||
const sortedJson: { [key: string]: Record<string, string> } = {}
|
||||
// We have to go into the subobject to sort the keys.
|
||||
for (const key of Object.keys(json)) {
|
||||
const subJson = json[key]
|
||||
const sortedSubJson: Record<string, string> = {}
|
||||
for (const subKey of Object.keys(subJson).sort()) {
|
||||
sortedSubJson[subKey] = subJson[subKey]
|
||||
}
|
||||
sortedJson[key] = sortedSubJson
|
||||
}
|
||||
finalJson = sortedJson
|
||||
}
|
||||
fs.writeFileSync(filePath, JSON.stringify(finalJson, null, space) + '\n')
|
||||
console.log(`Wrote ${filePath}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function i18nDownloadStrings() {
|
||||
await i18nDownloadTldrawStrings()
|
||||
await i18nDownloadDotcomStrings()
|
||||
}
|
||||
|
||||
async function i18nDownloadTldrawStrings() {
|
||||
const projectId = getEnv('LOKALISE_TLDRAW_PROJECT_ID')
|
||||
const apiKey = getEnv('LOKALISE_API_TOKEN')
|
||||
const dirPath = path.resolve(__dirname, '../../assets/translations')
|
||||
console.log('Downloading tldraw project files...')
|
||||
|
||||
const bundleUrl = await getAsyncBundleUrl(
|
||||
projectId,
|
||||
{
|
||||
format: 'json',
|
||||
original_filenames: true,
|
||||
export_empty_as: 'skip',
|
||||
},
|
||||
apiKey
|
||||
)
|
||||
|
||||
await fetchAndSave(
|
||||
bundleUrl,
|
||||
dirPath,
|
||||
false /* sort json */,
|
||||
(fileName) => fileName.split('/')[0].replace('_', '-').toLowerCase() + '.json',
|
||||
'\t',
|
||||
true /* skip english */
|
||||
)
|
||||
|
||||
console.log('Downloaded files successfully!')
|
||||
}
|
||||
|
||||
async function i18nDownloadDotcomStrings() {
|
||||
const projectId = getEnv('LOKALISE_PROJECT_ID')
|
||||
const apiKey = getEnv('LOKALISE_API_TOKEN')
|
||||
const dirPath = path.resolve(__dirname, '../../apps/dotcom/client/public/tla/locales')
|
||||
console.log('Downloading dotcom project files...')
|
||||
|
||||
const bundleUrl = await getAsyncBundleUrl(
|
||||
projectId,
|
||||
{
|
||||
format: 'json_structured',
|
||||
original_filenames: true,
|
||||
include_comments: true,
|
||||
include_description: true,
|
||||
export_empty_as: 'skip',
|
||||
plural_format: 'icu',
|
||||
placeholder_format: 'icu',
|
||||
},
|
||||
apiKey
|
||||
)
|
||||
|
||||
await fetchAndSave(
|
||||
bundleUrl,
|
||||
dirPath,
|
||||
true /* sort json */,
|
||||
(fileName) => fileName.split('/')[1].replace('_', '-').toLowerCase(),
|
||||
2
|
||||
)
|
||||
|
||||
console.log('Downloaded files successfully!')
|
||||
}
|
||||
|
||||
i18nDownloadStrings().catch((error) => {
|
||||
console.error('Failed to download i18n strings:')
|
||||
console.error(formatError(error))
|
||||
process.exit(1)
|
||||
})
|
||||
1480
internal/scripts/i18n-fix-strings.ts
Normal file
1480
internal/scripts/i18n-fix-strings.ts
Normal file
File diff suppressed because it is too large
Load Diff
253
internal/scripts/i18n-upload-strings.ts
Normal file
253
internal/scripts/i18n-upload-strings.ts
Normal file
@@ -0,0 +1,253 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { LokaliseApi, QueuedProcess } from '@lokalise/node-api'
|
||||
|
||||
// Pushes the English source strings to Lokalise and orders machine translations
|
||||
// for anything still untranslated (see i18n-download-strings.ts for the other
|
||||
// half of the pipeline).
|
||||
//
|
||||
// Two Lokalise projects are in play:
|
||||
// - the dotcom project (LOKALISE_PROJECT_ID), source apps/dotcom/client/public/tla/locales/en.json
|
||||
// - the tldraw SDK project (LOKALISE_TLDRAW_PROJECT_ID), source assets/translations/main.json
|
||||
// Only the dotcom project orders translations automatically; SDK orders are
|
||||
// still placed by hand.
|
||||
|
||||
const UPLOAD_POLL_INTERVAL_MS = 2000
|
||||
const UPLOAD_MAX_POLLS = 120
|
||||
/** Items per page when listing keys and languages, within Lokalise's per-endpoint maximums. */
|
||||
const PAGE_LIMIT = 500
|
||||
/** Anything pricier than this wants a human to look at it first. */
|
||||
const MAX_ORDER_TOTAL = 10
|
||||
|
||||
function formatError(error: unknown) {
|
||||
if (error instanceof Error) {
|
||||
return error.stack ?? error.message
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.stringify(error, null, 2)
|
||||
} catch {
|
||||
return String(error)
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
function getEnv(name: string) {
|
||||
const value = process.env[name]
|
||||
if (!value) throw new Error(`Missing required env var: ${name}`)
|
||||
return value
|
||||
}
|
||||
|
||||
/**
|
||||
* Lokalise's file upload is asynchronous: it returns a queued process and
|
||||
* creates the keys in the background. Poll until it's done — otherwise anything
|
||||
* we ask about the project next still describes the state from before the
|
||||
* upload, and freshly added strings look like they don't exist.
|
||||
*/
|
||||
async function waitForUpload(lokaliseApi: LokaliseApi, projectId: string, processId: string) {
|
||||
for (let i = 0; i < UPLOAD_MAX_POLLS; i++) {
|
||||
const queuedProcess = await lokaliseApi
|
||||
.queuedProcesses()
|
||||
.get(processId, { project_id: projectId })
|
||||
|
||||
if (queuedProcess.status === 'finished') return queuedProcess
|
||||
if (queuedProcess.status === 'failed' || queuedProcess.status === 'cancelled') {
|
||||
throw new Error(
|
||||
`Upload ${queuedProcess.status}: ${queuedProcess.message || 'No message'}\n${JSON.stringify(queuedProcess, null, 2)}`
|
||||
)
|
||||
}
|
||||
|
||||
await sleep(UPLOAD_POLL_INTERVAL_MS)
|
||||
}
|
||||
|
||||
throw new Error(`Upload timed out after ${UPLOAD_MAX_POLLS} polls for project ${projectId}`)
|
||||
}
|
||||
|
||||
function logUploadResult(queuedProcess: QueuedProcess) {
|
||||
const files = queuedProcess.details?.files ?? []
|
||||
if (!files.length) {
|
||||
console.log(' Upload finished, but Lokalise reported no file details.')
|
||||
return
|
||||
}
|
||||
for (const file of files) {
|
||||
console.log(
|
||||
` ${file.name_original ?? 'file'}: ${file.key_count_inserted ?? 0} inserted, ` +
|
||||
`${file.key_count_updated ?? 0} updated, ${file.key_count_skipped ?? 0} skipped ` +
|
||||
`(${file.key_count_total ?? 0} total).`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function uploadFile(
|
||||
lokaliseApi: LokaliseApi,
|
||||
options: { projectId: string; filePath: string; filename: string; cleanupMode: boolean }
|
||||
) {
|
||||
const { projectId, filePath, filename, cleanupMode } = options
|
||||
const file = fs.readFileSync(filePath, 'utf8')
|
||||
console.log(`Uploading ${filename}...`)
|
||||
|
||||
const uploadResult = await lokaliseApi.files().upload(projectId, {
|
||||
data: Buffer.from(file).toString('base64'),
|
||||
filename,
|
||||
lang_iso: 'en',
|
||||
detect_icu_plurals: true,
|
||||
cleanup_mode: cleanupMode,
|
||||
})
|
||||
|
||||
if (!uploadResult.process_id) {
|
||||
throw new Error(`Upload did not return a process_id: ${JSON.stringify(uploadResult, null, 2)}`)
|
||||
}
|
||||
|
||||
const queuedProcess = await waitForUpload(lokaliseApi, projectId, uploadResult.process_id)
|
||||
console.log(`Uploaded ${filename} successfully!`)
|
||||
logUploadResult(queuedProcess)
|
||||
}
|
||||
|
||||
/** Every target language in the project, English excluded. Paginated: there are 50+ of them. */
|
||||
async function listTargetLanguageIsos(lokaliseApi: LokaliseApi, projectId: string) {
|
||||
const isos: string[] = []
|
||||
for (let page = 1; ; page++) {
|
||||
const result = await lokaliseApi
|
||||
.languages()
|
||||
.list({ project_id: projectId, limit: PAGE_LIMIT, page })
|
||||
isos.push(...result.items.map((item) => item.lang_iso))
|
||||
if (result.items.length < PAGE_LIMIT) break
|
||||
}
|
||||
return isos.filter((lang) => lang !== 'en')
|
||||
}
|
||||
|
||||
/** Every key with at least one missing translation. Paginated: a big batch would otherwise truncate. */
|
||||
async function listUntranslatedKeyIds(lokaliseApi: LokaliseApi, projectId: string) {
|
||||
const keyIds: number[] = []
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const result = await lokaliseApi.keys().list({
|
||||
project_id: projectId,
|
||||
filter_untranslated: 1,
|
||||
pagination: 'cursor',
|
||||
limit: PAGE_LIMIT,
|
||||
...(cursor ? { cursor } : {}),
|
||||
})
|
||||
keyIds.push(...result.items.map((item) => item.key_id))
|
||||
cursor = result.hasNextCursor() ? (result.nextCursor ?? undefined) : undefined
|
||||
} while (cursor)
|
||||
return keyIds
|
||||
}
|
||||
|
||||
async function orderDotcomTranslations(lokaliseApi: LokaliseApi, projectId: string) {
|
||||
const teamId = getEnv('LOKALISE_TEAM_ID')
|
||||
const cardId = getEnv('LOKALISE_CARD_ID')
|
||||
|
||||
const targetLanguageIsos = await listTargetLanguageIsos(lokaliseApi, projectId)
|
||||
const allProjectUntranslatedKeys = await listUntranslatedKeyIds(lokaliseApi, projectId)
|
||||
|
||||
if (!allProjectUntranslatedKeys.length) {
|
||||
console.log('No new strings to translate.')
|
||||
return
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Found ${allProjectUntranslatedKeys.length} untranslated key(s) across ${targetLanguageIsos.length} language(s).`
|
||||
)
|
||||
|
||||
const orderDetails = {
|
||||
project_id: projectId,
|
||||
payment_method: 'credit_card' as const,
|
||||
card_id: String(cardId),
|
||||
briefing: 'Please translate the following keys',
|
||||
source_language_iso: 'en',
|
||||
target_language_isos: targetLanguageIsos,
|
||||
keys: allProjectUntranslatedKeys,
|
||||
provider_slug: 'google',
|
||||
translation_tier: 1,
|
||||
}
|
||||
|
||||
console.log('Placing test order for new strings...')
|
||||
const placeTranslationOrderDryRun = await lokaliseApi.orders().create(
|
||||
{
|
||||
...orderDetails,
|
||||
dry_run: true,
|
||||
},
|
||||
{
|
||||
team_id: teamId,
|
||||
}
|
||||
)
|
||||
|
||||
// Lokalise won't accept an order that prices out at 0, so there's nothing to
|
||||
// place here. Log enough to diagnose it: the usual cause is that the keys are
|
||||
// untranslated but not billable (already covered by translation memory, or
|
||||
// sitting in an open order from a previous run).
|
||||
if (placeTranslationOrderDryRun.total === 0) {
|
||||
console.log(
|
||||
`There are strings to translate but the dry run priced them at 0, so Lokalise won't take the order.`
|
||||
)
|
||||
console.log(
|
||||
` Untranslated keys (${allProjectUntranslatedKeys.length}): ${allProjectUntranslatedKeys.join(', ')}`
|
||||
)
|
||||
console.log(
|
||||
` Target languages (${targetLanguageIsos.length}): ${targetLanguageIsos.join(', ')}`
|
||||
)
|
||||
console.log(` Dry run response: ${JSON.stringify(placeTranslationOrderDryRun, null, 2)}`)
|
||||
console.log('Place the order manually in Lokalise if these strings need translating.')
|
||||
return
|
||||
}
|
||||
|
||||
if (placeTranslationOrderDryRun.total > MAX_ORDER_TOTAL) {
|
||||
console.error(
|
||||
`Cost of translations (${placeTranslationOrderDryRun.total}) is exceeding expectations (${MAX_ORDER_TOTAL}). Place a manual order.`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Placing actual order for new strings (total: ${placeTranslationOrderDryRun.total})...`
|
||||
)
|
||||
await lokaliseApi.orders().create(
|
||||
{
|
||||
...orderDetails,
|
||||
},
|
||||
{
|
||||
team_id: teamId,
|
||||
}
|
||||
)
|
||||
console.log('Finished placing order for new strings.')
|
||||
}
|
||||
|
||||
async function i18nUploadStrings() {
|
||||
const lokaliseApi = new LokaliseApi({ apiKey: getEnv('LOKALISE_API_TOKEN') })
|
||||
const dotcomProjectId = getEnv('LOKALISE_PROJECT_ID')
|
||||
const tldrawProjectId = getEnv('LOKALISE_TLDRAW_PROJECT_ID')
|
||||
|
||||
await uploadFile(lokaliseApi, {
|
||||
projectId: dotcomProjectId,
|
||||
filePath: path.resolve(__dirname, '../../apps/dotcom/client/public/tla/locales/en.json'),
|
||||
filename: 'en.json',
|
||||
cleanupMode: true,
|
||||
})
|
||||
await orderDotcomTranslations(lokaliseApi, dotcomProjectId)
|
||||
|
||||
// The SDK project has no automated ordering — uploading keeps its source
|
||||
// strings current so a manual order can pick them up. It runs last so a
|
||||
// problem here can't stop the dotcom half of the pipeline.
|
||||
// Cleanup mode deletes keys missing from the uploaded file, which is what
|
||||
// keeps the project matching main.json. Both projects export with
|
||||
// original_filenames, and i18n-download-strings.ts maps every file in a
|
||||
// locale's folder to the same <locale>.json, so a stray file here doesn't
|
||||
// show up as extra keys — it silently overwrites a locale's real strings
|
||||
// depending on export order.
|
||||
await uploadFile(lokaliseApi, {
|
||||
projectId: tldrawProjectId,
|
||||
filePath: path.resolve(__dirname, '../../assets/translations/main.json'),
|
||||
filename: 'main.json',
|
||||
cleanupMode: true,
|
||||
})
|
||||
}
|
||||
|
||||
i18nUploadStrings().catch((error) => {
|
||||
console.error('Failed to upload i18n strings:')
|
||||
console.error(formatError(error))
|
||||
process.exit(1)
|
||||
})
|
||||
77
internal/scripts/lib/add-extensions.ts
Normal file
77
internal/scripts/lib/add-extensions.ts
Normal file
@@ -0,0 +1,77 @@
|
||||
import { existsSync, readFileSync, statSync, writeFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { glob } from 'glob'
|
||||
import { parse, print, visit } from 'recast'
|
||||
|
||||
const extensions = ['.js', '.mjs', '.cjs']
|
||||
function resolveRelativePath(importingFile: string, relativePath: string) {
|
||||
if (!relativePath.startsWith('.')) {
|
||||
return relativePath
|
||||
}
|
||||
|
||||
const containingDir = path.dirname(importingFile)
|
||||
|
||||
if (
|
||||
existsSync(path.join(containingDir, relativePath)) &&
|
||||
!statSync(path.join(containingDir, relativePath)).isDirectory()
|
||||
) {
|
||||
// if the file already exists, e.g. .css files, just use it
|
||||
return relativePath
|
||||
}
|
||||
|
||||
// strip the file extension if applicable
|
||||
relativePath.replace(/\.(m|c)?js$/, '')
|
||||
|
||||
for (const extension of extensions) {
|
||||
if (relativePath.endsWith(extension)) {
|
||||
return relativePath
|
||||
} else {
|
||||
let candidate = `${relativePath}${extension}`
|
||||
if (existsSync(path.join(containingDir, candidate))) {
|
||||
return candidate
|
||||
}
|
||||
|
||||
candidate = `${relativePath}/index${extension}`
|
||||
|
||||
if (existsSync(path.join(containingDir, candidate))) {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Could not resolve relative path ${relativePath} from ${importingFile}`)
|
||||
}
|
||||
|
||||
export function addJsExtensions(distDir: string) {
|
||||
for (const file of glob.sync(path.join(distDir, '**/*.{mjs,cjs,js}'))) {
|
||||
const code = parse(readFileSync(file, 'utf8'), { parser: require('recast/parsers/typescript') })
|
||||
|
||||
visit(code, {
|
||||
visitImportDeclaration(path) {
|
||||
path.value.source.value = resolveRelativePath(file, path.value.source.value)
|
||||
return false
|
||||
},
|
||||
visitExportAllDeclaration(path) {
|
||||
path.value.source.value = resolveRelativePath(file, path.value.source.value)
|
||||
return false
|
||||
},
|
||||
visitExportNamedDeclaration(path) {
|
||||
if (path.value.source) {
|
||||
path.value.source.value = resolveRelativePath(file, path.value.source.value)
|
||||
}
|
||||
this.traverse(path)
|
||||
},
|
||||
visitCallExpression(path) {
|
||||
if (
|
||||
path.value.callee.type === 'Import' &&
|
||||
path.value.arguments[0]?.type === 'StringLiteral'
|
||||
) {
|
||||
path.value.arguments[0].value = resolveRelativePath(file, path.value.arguments[0].value)
|
||||
}
|
||||
this.traverse(path)
|
||||
},
|
||||
})
|
||||
|
||||
writeFileSync(file, print(code).code)
|
||||
}
|
||||
}
|
||||
217
internal/scripts/lib/deploy.ts
Normal file
217
internal/scripts/lib/deploy.ts
Normal file
@@ -0,0 +1,217 @@
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import * as github from '@actions/github'
|
||||
import { exec } from './exec'
|
||||
|
||||
export function getDeployInfo() {
|
||||
const githubPrNumber = process.env.GITHUB_REF?.match(/refs\/pull\/(\d+)\/merge/)?.[1]
|
||||
|
||||
let previewId = process.env.TLDRAW_PREVIEW_ID
|
||||
if (!previewId && process.env.TLDRAW_ENV === 'preview') {
|
||||
if (githubPrNumber) {
|
||||
previewId = `pr-${githubPrNumber}`
|
||||
} else {
|
||||
throw new Error(
|
||||
'If running preview deploys from outside of a PR action, TLDRAW_PREVIEW_ID env var must be set'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const sha: string | undefined =
|
||||
// if the event is 'pull_request', github.context.sha is an ephemeral merge commit
|
||||
// while the actual commit we want to create the deployment for is the 'head' of the PR.
|
||||
github.context.eventName === 'pull_request'
|
||||
? github.context.payload.pull_request?.head.sha
|
||||
: github.context.sha
|
||||
|
||||
if (!sha) {
|
||||
throw new Error('Could not determine the SHA of the commit to deploy')
|
||||
}
|
||||
|
||||
return {
|
||||
githubPrNumber,
|
||||
previewId,
|
||||
sha,
|
||||
}
|
||||
}
|
||||
|
||||
// Creates a github 'deployment', which creates a 'View Deployment' button in the PR timeline.
|
||||
export async function createGithubDeployment(
|
||||
env: { GH_TOKEN: string; TLDRAW_ENV: string },
|
||||
{
|
||||
app,
|
||||
deploymentUrl,
|
||||
inspectUrl,
|
||||
sha,
|
||||
}: { app: string; deploymentUrl: string; inspectUrl?: string; sha: string }
|
||||
) {
|
||||
const client = github.getOctokit(env.GH_TOKEN)
|
||||
|
||||
const deployment = await client.rest.repos.createDeployment({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
ref: sha,
|
||||
payload: { web_url: deploymentUrl },
|
||||
environment: `${app}-${env.TLDRAW_ENV}`,
|
||||
transient_environment: env.TLDRAW_ENV === 'preview',
|
||||
production_environment: env.TLDRAW_ENV === 'production',
|
||||
required_contexts: [],
|
||||
auto_merge: false,
|
||||
task: 'deploy',
|
||||
})
|
||||
|
||||
await client.rest.repos.createDeploymentStatus({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
deployment_id: (deployment.data as any).id,
|
||||
state: 'success',
|
||||
environment_url: deploymentUrl,
|
||||
log_url: inspectUrl,
|
||||
})
|
||||
}
|
||||
|
||||
/** Deploy a worker to wrangler, returning the deploy ID */
|
||||
export async function wranglerDeploy({
|
||||
location,
|
||||
dryRun,
|
||||
env,
|
||||
vars,
|
||||
sentry,
|
||||
}: {
|
||||
location: string
|
||||
dryRun: boolean
|
||||
env: string
|
||||
vars: Record<string, string>
|
||||
sentry?: {
|
||||
authToken: string
|
||||
project: string
|
||||
environment?: string
|
||||
}
|
||||
}) {
|
||||
const varsArray = []
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
varsArray.push('--var', `${key}:${value}`)
|
||||
}
|
||||
|
||||
const out = await exec(
|
||||
'yarn',
|
||||
[
|
||||
'wrangler',
|
||||
'deploy',
|
||||
dryRun ? '--dry-run' : null,
|
||||
'--env',
|
||||
env,
|
||||
'--outdir',
|
||||
'.wrangler/dist',
|
||||
...varsArray,
|
||||
],
|
||||
{
|
||||
pwd: location,
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
// wrangler needs CI=1 set to prevent it from trying to do interactive prompts
|
||||
CI: '1',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
if (dryRun) return
|
||||
|
||||
const versionMatch = out.match(/Current Version ID: (.+)/)
|
||||
if (!versionMatch) {
|
||||
throw new Error('Could not find the deploy ID in wrangler output')
|
||||
}
|
||||
|
||||
const workerNameMatch = out.match(/Uploaded ([^ ]+)/)
|
||||
|
||||
if (!workerNameMatch) {
|
||||
throw new Error('Could not find the worker name in wrangler output')
|
||||
}
|
||||
|
||||
if (sentry) {
|
||||
const release = `${sentry.environment ?? workerNameMatch[1]}.${versionMatch[1]}`
|
||||
|
||||
const sentryEnv = {
|
||||
SENTRY_AUTH_TOKEN: sentry.authToken,
|
||||
SENTRY_ORG: 'tldraw',
|
||||
SENTRY_PROJECT: sentry.project,
|
||||
}
|
||||
|
||||
// create a sentry release:
|
||||
await exec('yarn', ['run', '-T', 'sentry-cli', 'releases', 'new', release], {
|
||||
pwd: location,
|
||||
env: sentryEnv,
|
||||
})
|
||||
|
||||
// upload sourcemaps to the release:
|
||||
await exec(
|
||||
'yarn',
|
||||
[
|
||||
'run',
|
||||
'-T',
|
||||
'sentry-cli',
|
||||
'releases',
|
||||
'files',
|
||||
release,
|
||||
'upload-sourcemaps',
|
||||
'.wrangler/dist',
|
||||
],
|
||||
{
|
||||
pwd: location,
|
||||
env: sentryEnv,
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export interface ServiceBinding {
|
||||
binding: string
|
||||
service: string
|
||||
}
|
||||
|
||||
export async function setWranglerPreviewConfig(
|
||||
location: string,
|
||||
{
|
||||
name,
|
||||
routeHostname,
|
||||
serviceBinding,
|
||||
}: { name: string; routeHostname?: string; serviceBinding?: ServiceBinding },
|
||||
queueName?: string
|
||||
) {
|
||||
// A zone route instead of a custom domain: custom domains mint a dedicated
|
||||
// certificate pack per hostname, while routes are covered by the zone's
|
||||
// universal wildcard cert and the existing proxied *.tldraw.xyz DNS record.
|
||||
const zoneName = routeHostname?.split('.').slice(-2).join('.')
|
||||
const additionalProperties = `name = "${name}"
|
||||
${routeHostname ? `routes = [ { pattern = "${routeHostname}/*", zone_name = "${zoneName}" } ]` : ''}
|
||||
${serviceBinding ? `services = [ {binding = "${serviceBinding.binding}", service = "${serviceBinding.service}" } ]` : ''}`
|
||||
|
||||
const envPreviewSection = `\n[env.preview]\n${additionalProperties}\n`
|
||||
|
||||
const path = join(location, 'wrangler.toml')
|
||||
let data = readFileSync(path).toString()
|
||||
|
||||
// Replace the entire [env.preview] section including all its properties
|
||||
// Match from [env.preview] to the next section or end of file
|
||||
const previewSectionRegex = /\n\[env\.preview\]\n(?:[^[](?:[^\n]*\n)*)?/
|
||||
|
||||
if (previewSectionRegex.test(data)) {
|
||||
data = data.replace(previewSectionRegex, envPreviewSection)
|
||||
} else {
|
||||
data += envPreviewSection
|
||||
}
|
||||
|
||||
if (queueName) {
|
||||
const envPreviewQueuesSection = `\n[[env.preview.queues.producers]]
|
||||
queue = "${queueName}"
|
||||
binding = "QUEUE"
|
||||
|
||||
[[env.preview.queues.consumers]]
|
||||
queue = "${queueName}"
|
||||
max_retries =10
|
||||
`
|
||||
data += envPreviewQueuesSection
|
||||
}
|
||||
|
||||
writeFileSync(path, data)
|
||||
}
|
||||
183
internal/scripts/lib/didAnyPackageChange.ts
Normal file
183
internal/scripts/lib/didAnyPackageChange.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import assert from 'assert'
|
||||
import { writeFileSync } from 'fs'
|
||||
import * as diff from 'diff'
|
||||
import kleur from 'kleur'
|
||||
import * as tar from 'tar'
|
||||
import tmp from 'tmp'
|
||||
import { exec } from './exec'
|
||||
import { nicelog } from './nicelog'
|
||||
import { PackageDetails, getAllPackageDetails } from './publishing'
|
||||
|
||||
async function getPackageFirstDiff(
|
||||
pkg: PackageDetails,
|
||||
publishedVersion: string
|
||||
): Promise<Diff | null> {
|
||||
assert(process.env.TLDRAW_BEMO_URL, 'TLDRAW_BEMO_URL env var must be set')
|
||||
const dir = tmp.dirSync({ unsafeCleanup: true })
|
||||
const dirPath = dir.name
|
||||
try {
|
||||
const unscopedName = pkg.name.replace('@tldraw/', '')
|
||||
const url = `https://registry.npmjs.org/${pkg.name}/-/${unscopedName}-${publishedVersion}.tgz`
|
||||
const res = await fetch(url)
|
||||
if (res.status === 404) {
|
||||
// The package is not published at this version. This happens when a
|
||||
// new package was added to the monorepo since the last release, or
|
||||
// when a previous publish was interrupted after the version-bump
|
||||
// commit landed but before npm publish completed. In either case
|
||||
// treat it as an added package so the release proceeds instead of
|
||||
// failing the entire workflow.
|
||||
nicelog(
|
||||
`No published tarball for ${pkg.name}@${publishedVersion} (${url}); treating as added.`
|
||||
)
|
||||
return { type: 'added', packageName: pkg.name, filePath: pkg.name }
|
||||
}
|
||||
if (res.status >= 400) {
|
||||
throw new Error(`Failed to fetch ${url}: ${res.status}`)
|
||||
}
|
||||
const publishedTarballPath = `${dirPath}/published-package.tgz`
|
||||
writeFileSync(publishedTarballPath, Buffer.from(await res.arrayBuffer()))
|
||||
const publishedManifest = getTarballManifestSync(publishedTarballPath)
|
||||
|
||||
const localTarballPath = `${dirPath}/local-package.tgz`
|
||||
await exec('yarn', ['pack', '--out', localTarballPath], { pwd: pkg.dir, env: process.env })
|
||||
|
||||
const localManifest = getTarballManifestSync(localTarballPath)
|
||||
|
||||
return getManifestFirstDiff(pkg.name, publishedManifest, localManifest)
|
||||
} finally {
|
||||
dir.removeCallback()
|
||||
}
|
||||
}
|
||||
|
||||
type Diff =
|
||||
| {
|
||||
type: 'added' | 'removed'
|
||||
packageName: string
|
||||
filePath: string
|
||||
}
|
||||
| {
|
||||
type: 'changed'
|
||||
packageName: string
|
||||
filePath: string
|
||||
diff: string
|
||||
}
|
||||
|
||||
// In the `tldraw` package these files are generated during publishing from the
|
||||
// docs content in `apps/docs/content` (see `generate-tldraw-package-docs.ts`),
|
||||
// not from SDK source. A docs-only change therefore alters them, which would
|
||||
// otherwise make this diff check report a package change and cut a spurious SDK
|
||||
// patch release. Ignore them for the `tldraw` package so that docs-only patches
|
||||
// don't trigger a new SDK version. Other packages ship a hand-authored DOCS.md
|
||||
// as real source, so we only skip these for `tldraw`.
|
||||
//
|
||||
// `generate-tldraw-package-docs.ts` writes these files to the package root, so
|
||||
// in the tarball they live at exactly `package/<name>` (tarball entry paths are
|
||||
// prefixed with `package/`). Match the full path rather than the basename so a
|
||||
// same-named file in some other folder still counts as a real change.
|
||||
const GENERATED_DOCS_FILES = new Set(['package/DOCS.md', 'package/RELEASE_NOTES.md'])
|
||||
const GENERATED_DOCS_PACKAGE = 'tldraw'
|
||||
|
||||
function isGeneratedDocsFile(packageName: string, filePath: string) {
|
||||
if (packageName !== GENERATED_DOCS_PACKAGE) return false
|
||||
return GENERATED_DOCS_FILES.has(filePath)
|
||||
}
|
||||
|
||||
function getManifestFirstDiff(
|
||||
packageName: string,
|
||||
a: Record<string, Buffer>,
|
||||
b: Record<string, Buffer>
|
||||
): Diff | null {
|
||||
const aKeys = Object.keys(a).filter((key) => !isGeneratedDocsFile(packageName, key))
|
||||
const bKeys = Object.keys(b).filter((key) => !isGeneratedDocsFile(packageName, key))
|
||||
for (const key of aKeys) {
|
||||
if (!bKeys.includes(key)) {
|
||||
return { type: 'removed', packageName, filePath: key }
|
||||
}
|
||||
if (!a[key].equals(b[key])) {
|
||||
return {
|
||||
type: 'changed',
|
||||
packageName,
|
||||
filePath: key,
|
||||
diff: diff.createTwoFilesPatch(key, key, a[key].toString(), b[key].toString()),
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of bKeys) {
|
||||
if (!aKeys.includes(key)) {
|
||||
return { type: 'added', packageName, filePath: key }
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function getTarballManifestSync(tarballPath: string) {
|
||||
const manifest: Record<string, Buffer> = {}
|
||||
tar.list({
|
||||
file: tarballPath,
|
||||
onentry: (entry) => {
|
||||
entry.on('data', (data) => {
|
||||
// we could hash these to reduce memory but it's probably fine
|
||||
const existing = manifest[entry.path]
|
||||
if (existing) {
|
||||
manifest[entry.path] = Buffer.concat([existing, data])
|
||||
} else {
|
||||
manifest[entry.path] = data
|
||||
}
|
||||
})
|
||||
},
|
||||
sync: true,
|
||||
})
|
||||
|
||||
return manifest
|
||||
}
|
||||
|
||||
export function formatDiff(diff: Diff) {
|
||||
let message = kleur.cyan().bold(diff.packageName) + ' changed:'
|
||||
switch (diff.type) {
|
||||
case 'added':
|
||||
message += `\n New file added ${kleur.green().bold(diff.filePath)}`
|
||||
break
|
||||
case 'removed':
|
||||
message += `\n File removed ${kleur.red().bold(diff.filePath)}`
|
||||
break
|
||||
case 'changed':
|
||||
message += `\n File changed ${kleur.yellow().bold(diff.filePath)}\n\n`
|
||||
message += diff.diff
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
if (line.startsWith('+')) {
|
||||
return kleur.green(line)
|
||||
} else if (line.startsWith('-')) {
|
||||
return kleur.red(line)
|
||||
} else {
|
||||
return line
|
||||
}
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
break
|
||||
}
|
||||
return message
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare each local package against the published tarball at
|
||||
* `publishedVersion` on npm and return the first detected difference, or null
|
||||
* if every package is identical to its published counterpart.
|
||||
*
|
||||
* Callers should pass the latest version that's actually published on npm
|
||||
* (e.g. from `npm show <pkg> version`) rather than the version in the local
|
||||
* `package.json`. The local version can drift ahead of npm if a previous
|
||||
* publish was interrupted after the version-bump commit was pushed; using the
|
||||
* known-published version keeps the diff check robust to that state.
|
||||
*/
|
||||
export async function getAnyPackageDiff(publishedVersion: string): Promise<Diff | null> {
|
||||
const details = await getAllPackageDetails()
|
||||
for (const pkg of Object.values(details)) {
|
||||
const diff = await getPackageFirstDiff(pkg, publishedVersion)
|
||||
if (diff) {
|
||||
return diff
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
124
internal/scripts/lib/discord.ts
Normal file
124
internal/scripts/lib/discord.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
const MASK_PLACEHOLDER = '\x00MASKED\x00'
|
||||
|
||||
function sanitizeVariables(errorOutput: string, secretValues?: string[]): string {
|
||||
let sanitized = errorOutput
|
||||
|
||||
// Value-based masking: replace any occurrence of known secret values (like GitHub Actions).
|
||||
// Uses an internal placeholder so regex patterns below can't re-match masked output.
|
||||
// Sort longest-first so a longer secret is masked before a shorter substring of it.
|
||||
if (secretValues) {
|
||||
const sorted = secretValues.filter((v) => v.length >= 8).sort((a, b) => b.length - a.length)
|
||||
for (const value of sorted) {
|
||||
sanitized = sanitized.replaceAll(value, MASK_PLACEHOLDER)
|
||||
}
|
||||
}
|
||||
|
||||
// Sanitize wrangler --var KEY:VALUE patterns
|
||||
sanitized = sanitized.replace(/(--var\s+)(\w+):[^ \n]+/g, '$1$2:`***`')
|
||||
|
||||
// Sanitize KEY=VALUE patterns where KEY looks like an env var and value is long enough
|
||||
// to be a secret (e.g. flyctl secrets set). Short values like =debug are left alone.
|
||||
sanitized = sanitized.replace(/\b([A-Z][A-Z_0-9]{2,})=([^ \n]{8,})/g, '$1=`***`')
|
||||
|
||||
// Sanitize --token VALUE and --token=VALUE patterns (e.g. vercel --token xxx)
|
||||
sanitized = sanitized.replace(/(--token[\s=])\S+/g, '$1`***`')
|
||||
|
||||
// Sanitize connection strings (postgres://, redis://, etc.)
|
||||
sanitized = sanitized.replace(
|
||||
/\b(postgres|postgresql|mysql|redis|mongodb|amqp):\/\/[^\s"']+/gi,
|
||||
'$1://`***`'
|
||||
)
|
||||
|
||||
// Replace internal placeholder with final mask
|
||||
sanitized = sanitized.replaceAll(MASK_PLACEHOLDER, '`***`')
|
||||
|
||||
return sanitized
|
||||
}
|
||||
|
||||
export class Discord {
|
||||
static AT_TEAM_MENTION = '<@&1414904550917144586>'
|
||||
|
||||
constructor(opts: {
|
||||
webhookUrl: string
|
||||
shouldNotify: boolean
|
||||
totalSteps?: number
|
||||
messagePrefix?: string
|
||||
secretValues?: string[]
|
||||
}) {
|
||||
this.webhookUrl = opts.webhookUrl
|
||||
this.shouldNotify = opts.shouldNotify
|
||||
this.totalSteps = opts.totalSteps ?? 0
|
||||
this.messagePrefix = opts.messagePrefix ?? ''
|
||||
this.secretValues = opts.secretValues
|
||||
}
|
||||
|
||||
webhookUrl: string
|
||||
shouldNotify: boolean
|
||||
totalSteps: number
|
||||
currentStep = 0
|
||||
messagePrefix: string
|
||||
private secretValues?: string[]
|
||||
|
||||
private async send(method: string, url: string, body: unknown): Promise<any> {
|
||||
try {
|
||||
const response = await fetch(`${this.webhookUrl}${url}`, {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
})
|
||||
if (!response.ok) {
|
||||
console.warn(
|
||||
`Discord webhook request failed: ${method} -> ${response.status} ${response.statusText}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
return response.json()
|
||||
} catch (err) {
|
||||
const errString = err instanceof Error ? (err.stack ?? err.message) : String(err)
|
||||
console.warn(
|
||||
`Discord webhook request failed: ${method}\n${sanitizeVariables(errString, this.secretValues)}`
|
||||
)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async message(content: string, { always = false }: { always?: boolean } = {}) {
|
||||
if (!always && !this.shouldNotify) {
|
||||
return {
|
||||
edit: () => {
|
||||
// noop
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
const prefixedContent = this.messagePrefix ? `${this.messagePrefix} ${content}` : content
|
||||
const message = await this.send('POST', '?wait=true', {
|
||||
content: sanitizeVariables(prefixedContent, this.secretValues),
|
||||
})
|
||||
|
||||
return {
|
||||
edit: async (newContent: string) => {
|
||||
if (!message?.id) return
|
||||
const prefixedNewContent = this.messagePrefix
|
||||
? `${this.messagePrefix} ${newContent}`
|
||||
: newContent
|
||||
await this.send('PATCH', `/messages/${message.id}`, {
|
||||
content: sanitizeVariables(prefixedNewContent, this.secretValues),
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async step<T>(content: string, cb: () => Promise<T>): Promise<T> {
|
||||
this.currentStep++
|
||||
const message = await this.message(`[${this.currentStep}/${this.totalSteps}] ${content}...`)
|
||||
try {
|
||||
const result = await cb()
|
||||
await message.edit(`${content} ✅`)
|
||||
return result
|
||||
} catch (err) {
|
||||
await message.edit(`${content} ❌`)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
76
internal/scripts/lib/exec.ts
Normal file
76
internal/scripts/lib/exec.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import { execFile } from 'child_process'
|
||||
import { nicelog } from './nicelog'
|
||||
|
||||
interface ExecOpts {
|
||||
pwd?: string
|
||||
processStdoutLine?(line: string): void
|
||||
processStderrLine?(line: string): void
|
||||
env?: Partial<NodeJS.ProcessEnv>
|
||||
}
|
||||
|
||||
export function prefixOutput(prefix: string) {
|
||||
return {
|
||||
processStdoutLine: (line: string) => process.stdout.write(`${prefix}${line}\n`),
|
||||
processStderrLine: (line: string) => process.stderr.write(`${prefix}${line}\n`),
|
||||
}
|
||||
}
|
||||
|
||||
export async function exec(
|
||||
command: string,
|
||||
args: (string | null)[],
|
||||
{
|
||||
pwd = process.cwd(),
|
||||
processStdoutLine = (line) => process.stdout.write(`${line}\n`),
|
||||
processStderrLine = (line) => process.stderr.write(`${line}\n`),
|
||||
env,
|
||||
}: ExecOpts = {}
|
||||
): Promise<string> {
|
||||
nicelog(`> $ ${command} ${args.join(' ')} (in ${pwd}))`)
|
||||
return new Promise((resolve, reject) => {
|
||||
const data: string[] = []
|
||||
|
||||
const childProcess = execFile(
|
||||
command,
|
||||
args.filter((arg): arg is string => !!arg),
|
||||
{ cwd: pwd, env: { ...process.env, ...env } },
|
||||
(err) => {
|
||||
if (err) reject(err)
|
||||
else resolve(data.join(''))
|
||||
}
|
||||
)
|
||||
|
||||
let pendingStdoutLine = ''
|
||||
childProcess.stdout!.on('data', (chunk) => {
|
||||
const chunkString: string = chunk.toString('utf-8')
|
||||
data.push(chunkString)
|
||||
|
||||
const lines = chunkString.split('\n')
|
||||
lines[0] = pendingStdoutLine + lines[0]
|
||||
pendingStdoutLine = lines.pop() ?? ''
|
||||
|
||||
for (const line of lines) {
|
||||
processStdoutLine(line)
|
||||
}
|
||||
})
|
||||
childProcess.stdout!.on('close', () => {
|
||||
processStdoutLine(pendingStdoutLine)
|
||||
})
|
||||
|
||||
let pendingStderrLine = ''
|
||||
childProcess.stderr!.on('data', (chunk) => {
|
||||
const chunkString: string = chunk.toString('utf-8')
|
||||
data.push(chunkString)
|
||||
|
||||
const lines = chunkString.split('\n')
|
||||
lines[0] = pendingStderrLine + lines[0]
|
||||
pendingStderrLine = lines.pop() ?? ''
|
||||
|
||||
for (const line of lines) {
|
||||
processStderrLine(line)
|
||||
}
|
||||
})
|
||||
childProcess.stderr!.on('close', () => {
|
||||
processStderrLine(pendingStderrLine)
|
||||
})
|
||||
})
|
||||
}
|
||||
90
internal/scripts/lib/file.ts
Normal file
90
internal/scripts/lib/file.ts
Normal file
@@ -0,0 +1,90 @@
|
||||
import { existsSync, readFileSync } from 'fs'
|
||||
import { readFile, writeFile as writeFileUnchecked } from 'fs/promises'
|
||||
import { dirname, join, relative } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import json5 from 'json5'
|
||||
import { format } from 'oxfmt'
|
||||
import { nicelog } from './nicelog'
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url)
|
||||
const __dirname = dirname(__filename)
|
||||
export const REPO_ROOT = join(__dirname, '../../..')
|
||||
|
||||
const _rootPackageJsonPath = join(REPO_ROOT, 'package.json')
|
||||
if (!existsSync(_rootPackageJsonPath)) {
|
||||
throw new Error('expected to find package.json in REPO_ROOT')
|
||||
}
|
||||
const _rootPackageJson = JSON.parse(readFileSync(_rootPackageJsonPath, 'utf8'))
|
||||
if (_rootPackageJson['name'] !== '@tldraw/monorepo') {
|
||||
throw new Error('expected to find @tldraw/monorepo in REPO_ROOT package.json')
|
||||
}
|
||||
|
||||
export async function readJsonIfExists(file: string) {
|
||||
const fileContents = await readFileIfExists(file)
|
||||
if (fileContents === null) {
|
||||
return null
|
||||
}
|
||||
return json5.parse(fileContents)
|
||||
}
|
||||
|
||||
export async function readFileIfExists(file: string) {
|
||||
try {
|
||||
return await readFile(file, 'utf8')
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const oxfmtConfig = json5.parse(readFileSync(join(REPO_ROOT, '.oxfmtrc.json'), 'utf8'))
|
||||
export async function writeCodeFile(
|
||||
generator: string | null,
|
||||
language: 'typescript' | 'javascript',
|
||||
filePath: string,
|
||||
code: string
|
||||
) {
|
||||
const source = generator
|
||||
? `
|
||||
// This file is automatically generated by ${generator}.
|
||||
// Do not edit manually. Or do, I'm a comment, not a cop.
|
||||
|
||||
${code}
|
||||
`
|
||||
: code
|
||||
const ext = language === 'typescript' ? 'ts' : 'js'
|
||||
const result = await format(`file.${ext}`, source, oxfmtConfig)
|
||||
await writeStringFile(filePath, result.code)
|
||||
}
|
||||
|
||||
export async function writeStringFile(filePath: string, contents: string) {
|
||||
await writeFile(filePath, Buffer.from(contents, 'utf-8'))
|
||||
}
|
||||
|
||||
export async function writeFile(filePath: string, contents: Buffer) {
|
||||
if (process.env.CI && !process.env.ALLOW_REFRESH_ASSETS_CHANGES) {
|
||||
let existingContents: Buffer | null = null
|
||||
try {
|
||||
existingContents = await readFile(filePath)
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
if (existingContents && !existingContents.equals(contents)) {
|
||||
nicelog(
|
||||
`Asset file ${relative(
|
||||
REPO_ROOT,
|
||||
filePath
|
||||
)} has changed. Please run this script again and commit the changes.`
|
||||
)
|
||||
nicelog('Contents before:')
|
||||
nicelog(existingContents.toString('utf-8'))
|
||||
nicelog('\nContents after:')
|
||||
nicelog(contents.toString('utf-8'))
|
||||
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
await writeFileUnchecked(filePath, contents, 'utf-8')
|
||||
}
|
||||
|
||||
export async function writeJsonFile(filePath: string, contents: unknown) {
|
||||
await writeStringFile(filePath, JSON.stringify(contents, null, '\t') + '\n')
|
||||
}
|
||||
40
internal/scripts/lib/kill-tree.ts
Normal file
40
internal/scripts/lib/kill-tree.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { spawnSync } from 'child_process'
|
||||
|
||||
/**
|
||||
* Collect every descendant PID of `pid` by walking the process tree with `pgrep -P`.
|
||||
*
|
||||
* Returned breadth-first (direct children first, deepest descendants last). Callers that SIGKILL
|
||||
* the tree should reverse this so they kill deepest-first.
|
||||
*/
|
||||
export function descendantPids(pid: number): number[] {
|
||||
const result = spawnSync('pgrep', ['-P', String(pid)], { encoding: 'utf8' })
|
||||
const childPids = (result.stdout ?? '')
|
||||
.split('\n')
|
||||
.map((line) => Number(line.trim()))
|
||||
.filter((n) => Number.isInteger(n) && n > 0)
|
||||
return childPids.flatMap((childPid) => [childPid, ...descendantPids(childPid)])
|
||||
}
|
||||
|
||||
/**
|
||||
* SIGKILL `pid` and its entire descendant tree.
|
||||
*
|
||||
* Our dev orchestrators spawn children through wrapper layers (`yarn run -T tsx …`) and processes
|
||||
* that fork their own grandchildren (vite → esbuild, wrangler → workerd, zero-cache → worker
|
||||
* subprocesses). `child.kill()` only signals the direct child, and SIGKILL is uncatchable and does
|
||||
* not cascade — so killing a wrapper or parent just orphans everything beneath it, leaving processes
|
||||
* that keep holding the fixed dev ports. Walking by PID crosses those wrapper and process-group
|
||||
* boundaries.
|
||||
*
|
||||
* Collect the PIDs up front so the tree does not reparent out from under us while we kill, then
|
||||
* SIGKILL deepest-first. The root `pid` itself is left alone so the caller can finish its own
|
||||
* shutdown and exit. Safe to call from signal and synchronous `exit` handlers.
|
||||
*/
|
||||
export function killProcessTree(pid: number) {
|
||||
for (const target of descendantPids(pid).reverse()) {
|
||||
try {
|
||||
process.kill(target, 'SIGKILL')
|
||||
} catch {
|
||||
// already gone
|
||||
}
|
||||
}
|
||||
}
|
||||
44
internal/scripts/lib/labels.ts
Normal file
44
internal/scripts/lib/labels.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
interface Label {
|
||||
// this is what the label is 'called' on github
|
||||
name: string
|
||||
// this is how we describe the label in our pull request template
|
||||
description: string
|
||||
// this is the section title for the label in our changelogs
|
||||
changelogTitle: string
|
||||
}
|
||||
|
||||
const TYPE_LABELS = [
|
||||
{ name: `bugfix`, description: `Bug fix`, changelogTitle: '🐛 Bug Fixes' },
|
||||
{
|
||||
name: `improvement`,
|
||||
description: `Product improvement`,
|
||||
changelogTitle: '💄 Product Improvements',
|
||||
},
|
||||
{
|
||||
name: `feature`,
|
||||
description: `New feature`,
|
||||
changelogTitle: '🎉 New Features',
|
||||
},
|
||||
{
|
||||
name: `api`,
|
||||
description: `API change`,
|
||||
changelogTitle: '🛠️ API Changes',
|
||||
},
|
||||
{
|
||||
name: `other`,
|
||||
description: `Changes that don't affect SDK users, e.g. internal or .com changes`,
|
||||
changelogTitle: '🤷 Other',
|
||||
},
|
||||
] as const satisfies Label[]
|
||||
|
||||
export function getLabelNames() {
|
||||
return [...TYPE_LABELS].map((label) => label.name)
|
||||
}
|
||||
|
||||
function formatTemplateOption(label: Label) {
|
||||
return `- [ ] \`${label.name}\``
|
||||
}
|
||||
|
||||
export function formatLabelOptionsForPRTemplate() {
|
||||
return TYPE_LABELS.map(formatTemplateOption).join('\n')
|
||||
}
|
||||
18
internal/scripts/lib/makeEnv.ts
Normal file
18
internal/scripts/lib/makeEnv.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
export function makeEnv<const Keys extends readonly string[]>(
|
||||
keys: Keys
|
||||
): Record<Keys[number], string> {
|
||||
const env = {} as Record<string, string>
|
||||
const missingVars = []
|
||||
for (const key of keys) {
|
||||
const value = process.env[key]
|
||||
if (value === undefined) {
|
||||
missingVars.push(key)
|
||||
continue
|
||||
}
|
||||
env[key] = value
|
||||
}
|
||||
if (missingVars.length > 0) {
|
||||
throw new Error(`Missing environment variables: ${missingVars.join(', ')}`)
|
||||
}
|
||||
return env as Record<Keys[number], string>
|
||||
}
|
||||
3
internal/scripts/lib/nicelog.ts
Normal file
3
internal/scripts/lib/nicelog.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export function nicelog(...args: any[]) {
|
||||
console.log(...args)
|
||||
}
|
||||
57
internal/scripts/lib/posthog.ts
Normal file
57
internal/scripts/lib/posthog.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import { nicelog } from './nicelog'
|
||||
|
||||
/**
|
||||
* The PostHog project we send CI engineering metrics to. `phc_` keys are public, write-only
|
||||
* ingest keys, so keeping this in the repo is fine — it's the same project the e2e performance
|
||||
* reporter writes to. See https://eu.posthog.com/project/45972
|
||||
*/
|
||||
const CI_METRICS_API_KEY = 'phc_i8oKgMzgV38sn3GfjswW9mevQ3gFlo7bJXekZFeDN6'
|
||||
const DEFAULT_API_HOST = 'https://analytics.tldraw.com/ingest'
|
||||
|
||||
export interface CiMetricEvent {
|
||||
event: string
|
||||
/**
|
||||
* Groups a series together in PostHog. Use a stable string per metric (e.g.
|
||||
* `dotcom-bundle-size`) rather than a per-run id, so a trend over time is one person's events
|
||||
* instead of thousands of one-off people.
|
||||
*/
|
||||
distinctId: string
|
||||
properties: Record<string, unknown>
|
||||
}
|
||||
|
||||
/**
|
||||
* Send events to the CI metrics PostHog project. Everything goes in a single request to the
|
||||
* `/batch/` endpoint: a build reports a few dozen events, and sending them one at a time would
|
||||
* mean a few dozen round trips at the tail of a deploy, where a failure partway through leaves a
|
||||
* build with some of its events recorded and the rest missing.
|
||||
*/
|
||||
export async function captureCiMetricEvents(events: CiMetricEvent[]) {
|
||||
if (events.length === 0) return
|
||||
|
||||
const apiHost = process.env.POSTHOG_API_HOST || DEFAULT_API_HOST
|
||||
const timestamp = new Date().toISOString()
|
||||
|
||||
const response = await fetch(`${apiHost}/batch/`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
api_key: CI_METRICS_API_KEY,
|
||||
batch: events.map(({ event, distinctId, properties }) => ({
|
||||
event,
|
||||
properties: {
|
||||
...properties,
|
||||
distinct_id: distinctId,
|
||||
$lib: 'tldraw-ci-metrics',
|
||||
$lib_version: '1.0.0',
|
||||
},
|
||||
timestamp,
|
||||
})),
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`PostHog API error: ${response.status} ${response.statusText}`)
|
||||
}
|
||||
|
||||
nicelog(`Sent ${events.length} event(s) to PostHog`)
|
||||
}
|
||||
64
internal/scripts/lib/pr-info.ts
Normal file
64
internal/scripts/lib/pr-info.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import { Octokit } from 'octokit'
|
||||
import { exec } from './exec'
|
||||
import { nicelog } from './nicelog'
|
||||
|
||||
export async function getPrDetailsByNumber(octokit: Octokit, prNumber: number) {
|
||||
return await octokit.rest.pulls
|
||||
.get({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
pull_number: prNumber,
|
||||
})
|
||||
.then((res) => res.data)
|
||||
}
|
||||
|
||||
export async function getPrDetails(octokit: Octokit) {
|
||||
const lastCommitMessage = (await exec('git', ['log', '-1', '--oneline'])).trim()
|
||||
const lastPRNumber = lastCommitMessage.match(/\(#(\d+)\)$/)?.[1]
|
||||
if (!lastPRNumber) {
|
||||
nicelog('No PR number found in last commit message. Exiting...')
|
||||
return null
|
||||
}
|
||||
|
||||
return await getPrDetailsByNumber(octokit, parseInt(lastPRNumber))
|
||||
}
|
||||
|
||||
export async function getPrDetailsAndCommitSha(octokit: Octokit) {
|
||||
const prNumber = process.env.PR_NUMBER
|
||||
nicelog(`PR_NUMBER environment variable: ${prNumber}`)
|
||||
let pr: PullRequest | null
|
||||
let commitSha: string | null = null
|
||||
|
||||
if (prNumber) {
|
||||
// Post-merge mode: get PR details by number
|
||||
nicelog(`Post-merge mode: getting PR details for #${prNumber}`)
|
||||
pr = await getPrDetailsByNumber(octokit, parseInt(prNumber))
|
||||
if (!pr) {
|
||||
nicelog('Could not retrieve PR details. Exiting...')
|
||||
return null
|
||||
}
|
||||
commitSha = pr.merge_commit_sha
|
||||
} else {
|
||||
// Merge-time mode: get PR details from commit message
|
||||
nicelog('Merge-time mode: getting PR details from commit message')
|
||||
pr = await getPrDetails(octokit)
|
||||
if (!pr) {
|
||||
nicelog('Could not retrieve PR details. Exiting...')
|
||||
return null
|
||||
}
|
||||
commitSha = (await exec('git', ['rev-parse', 'HEAD'])).trim()
|
||||
}
|
||||
|
||||
if (!commitSha) {
|
||||
nicelog('PR is not merged or merge commit SHA not available. Exiting...')
|
||||
return null
|
||||
}
|
||||
|
||||
return { pr, commitSha }
|
||||
}
|
||||
|
||||
export type PullRequest = NonNullable<Awaited<ReturnType<typeof getPrDetailsByNumber>>>
|
||||
|
||||
export function labelPresent(pr: PullRequest, labelName: string): boolean {
|
||||
return pr.labels.some((label) => label.name === labelName)
|
||||
}
|
||||
295
internal/scripts/lib/publishing.ts
Normal file
295
internal/scripts/lib/publishing.ts
Normal file
@@ -0,0 +1,295 @@
|
||||
import { execSync } from 'child_process'
|
||||
import { existsSync, readFileSync, readdirSync, writeFileSync } from 'fs'
|
||||
import path, { join } from 'path'
|
||||
import { Octokit } from '@octokit/rest'
|
||||
import { fetch } from 'cross-fetch'
|
||||
import { glob } from 'glob'
|
||||
import { parse } from 'semver'
|
||||
import { exec } from './exec'
|
||||
import { REPO_ROOT } from './file'
|
||||
import { nicelog } from './nicelog'
|
||||
import { getAllWorkspacePackages } from './workspace'
|
||||
|
||||
export interface PackageDetails {
|
||||
name: string
|
||||
dir: string
|
||||
localDeps: string[]
|
||||
version: string
|
||||
}
|
||||
|
||||
async function getPackageDetails(dir: string): Promise<PackageDetails | null> {
|
||||
const packageJsonPath = path.join(dir, 'package.json')
|
||||
if (!existsSync(packageJsonPath)) {
|
||||
return null
|
||||
}
|
||||
const packageJson = JSON.parse(readFileSync(path.join(dir, 'package.json'), 'utf8'))
|
||||
if (packageJson.private) {
|
||||
return null
|
||||
}
|
||||
|
||||
const workspacePackages = await getAllWorkspacePackages()
|
||||
return {
|
||||
name: packageJson.name,
|
||||
dir,
|
||||
version: packageJson.version,
|
||||
localDeps: Object.keys(packageJson.dependencies ?? {}).filter((dep) =>
|
||||
workspacePackages.some((p) => p.name === dep)
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAllPackageDetails(): Promise<Record<string, PackageDetails>> {
|
||||
const dirs = readdirSync(join(REPO_ROOT, 'packages'))
|
||||
const details = await Promise.all(
|
||||
dirs.map((dir) => getPackageDetails(path.join(REPO_ROOT, 'packages', dir)))
|
||||
)
|
||||
const results = details.filter((x): x is PackageDetails => Boolean(x))
|
||||
|
||||
return Object.fromEntries(results.map((result) => [result.name, result]))
|
||||
}
|
||||
|
||||
export async function setAllVersions(version: string, options?: { stageChanges?: boolean }) {
|
||||
const packages = await getAllPackageDetails()
|
||||
for (const packageDetails of Object.values(packages)) {
|
||||
const manifest = JSON.parse(readFileSync(path.join(packageDetails.dir, 'package.json'), 'utf8'))
|
||||
manifest.version = version
|
||||
writeFileSync(
|
||||
path.join(packageDetails.dir, 'package.json'),
|
||||
JSON.stringify(manifest, null, '\t') + '\n'
|
||||
)
|
||||
}
|
||||
|
||||
await exec('yarn', ['refresh-assets', '--force'], { env: { ALLOW_REFRESH_ASSETS_CHANGES: '1' } })
|
||||
|
||||
const lernaJson = JSON.parse(readFileSync('lerna.json', 'utf8'))
|
||||
lernaJson.version = version
|
||||
writeFileSync('lerna.json', JSON.stringify(lernaJson, null, '\t') + '\n')
|
||||
|
||||
execSync('yarn')
|
||||
|
||||
if (options?.stageChanges) {
|
||||
await stageAllPackageJsonChanges()
|
||||
}
|
||||
}
|
||||
|
||||
async function stageAllPackageJsonChanges() {
|
||||
// stage the changes
|
||||
const packageJsonFilesToAdd = []
|
||||
for (const workspace of await getAllWorkspacePackages()) {
|
||||
if (workspace.relativePath.startsWith('packages/')) {
|
||||
packageJsonFilesToAdd.push(`${workspace.relativePath}/package.json`)
|
||||
}
|
||||
}
|
||||
const versionFilesToAdd = glob.sync('**/*/version.ts', {
|
||||
ignore: ['node_modules/**'],
|
||||
follow: false,
|
||||
})
|
||||
console.log('versionFilesToAdd', versionFilesToAdd)
|
||||
await exec('git', [
|
||||
'add',
|
||||
'--update',
|
||||
'lerna.json',
|
||||
...packageJsonFilesToAdd,
|
||||
...versionFilesToAdd,
|
||||
])
|
||||
}
|
||||
|
||||
function assertExists<T>(v: T | null | undefined): T {
|
||||
if (v === null || v === undefined) throw new Error('Expected value to exist')
|
||||
return v
|
||||
}
|
||||
|
||||
export async function getLatestTldrawVersionFromNpm({
|
||||
versionPrefix,
|
||||
}: { versionPrefix?: string } = {}) {
|
||||
if (!versionPrefix)
|
||||
return assertExists(parse((await exec('npm', ['show', 'tldraw', 'version'])).trim()))
|
||||
|
||||
const versions = (await exec('npm', ['show', 'tldraw@~' + versionPrefix, 'version'])).trim()
|
||||
if (versions.startsWith('tldraw')) {
|
||||
return assertExists(parse(versions.split('\n').pop()?.split(' ')[1].replaceAll("'", '')))
|
||||
}
|
||||
return assertExists(parse(versions))
|
||||
}
|
||||
|
||||
function topologicalSortPackages(packages: Record<string, PackageDetails>) {
|
||||
const sorted: PackageDetails[] = []
|
||||
const visited = new Set<string>()
|
||||
|
||||
function visit(packageName: string, path: string[]) {
|
||||
if (visited.has(packageName)) {
|
||||
return
|
||||
}
|
||||
visited.add(packageName)
|
||||
const packageDetails = packages[packageName]
|
||||
if (!packageDetails) {
|
||||
throw new Error(`Could not find package ${packageName}. path: ${path.join(' -> ')}`)
|
||||
}
|
||||
packageDetails.localDeps.forEach((dep) => visit(dep, [...path, dep]))
|
||||
sorted.push(packageDetails)
|
||||
}
|
||||
|
||||
Object.keys(packages).forEach((packageName) => visit(packageName, [packageName]))
|
||||
|
||||
return sorted
|
||||
}
|
||||
|
||||
export async function publish(distTag?: string) {
|
||||
// Authentication uses npm's trusted publisher OIDC flow. The publish job in
|
||||
// CI must grant `permissions: id-token: write` so yarn (>= 4.10, which we are
|
||||
// on via `packageManager`) can exchange the GitHub-issued OIDC token for a
|
||||
// short-lived publish token automatically.
|
||||
//
|
||||
// We invoke `yarn npm publish` rather than `npm publish` directly so that
|
||||
// yarn rewrites `workspace:*` dependency specifiers in the published
|
||||
// tarball into the concrete sibling versions. `npm publish` has no concept
|
||||
// of yarn's workspace protocol and would ship `"workspace:*"` literally,
|
||||
// breaking installs for any consumer outside this monorepo.
|
||||
// See https://docs.npmjs.com/trusted-publishers
|
||||
const packages = await getAllPackageDetails()
|
||||
|
||||
const publishOrder = topologicalSortPackages(packages)
|
||||
|
||||
// npm can take several minutes before a published version is readable. Waiting only
|
||||
// on a package's own dependencies keeps a newly visible package installable, without
|
||||
// making the run wait for every package's delay in turn.
|
||||
const readable = new Map<string, Promise<void>>()
|
||||
|
||||
for (const packageDetails of publishOrder) {
|
||||
await Promise.all(packageDetails.localDeps.map((dep) => readable.get(dep)))
|
||||
|
||||
const tag = distTag ?? parse(packageDetails.version)?.prerelease[0] ?? 'latest'
|
||||
nicelog(
|
||||
`Publishing ${packageDetails.name} with version ${packageDetails.version} under tag @${tag}`
|
||||
)
|
||||
|
||||
await retry(
|
||||
async () => {
|
||||
let output = ''
|
||||
const publishStart = Date.now()
|
||||
nicelog(
|
||||
`[publish] ${packageDetails.name}@${packageDetails.version} starting npm publish...`
|
||||
)
|
||||
try {
|
||||
await exec(
|
||||
`yarn`,
|
||||
[
|
||||
'npm',
|
||||
'publish',
|
||||
'--tag',
|
||||
String(tag),
|
||||
'--tolerate-republish',
|
||||
'--provenance',
|
||||
'--access',
|
||||
'public',
|
||||
],
|
||||
{
|
||||
pwd: packageDetails.dir,
|
||||
processStdoutLine: (line) => {
|
||||
output += line + '\n'
|
||||
nicelog(line)
|
||||
},
|
||||
processStderrLine: (line) => {
|
||||
output += line + '\n'
|
||||
nicelog(line)
|
||||
},
|
||||
}
|
||||
)
|
||||
} catch (e) {
|
||||
// A retry after a publish that actually landed is rejected as "published",
|
||||
// or as "staged" (409) while npm is still processing the first attempt.
|
||||
const lowerOutput = output.toLowerCase()
|
||||
if (
|
||||
lowerOutput.includes('cannot publish over the previously published versions') ||
|
||||
lowerOutput.includes('cannot publish over previously staged version')
|
||||
) {
|
||||
nicelog(
|
||||
`[publish] ${packageDetails.name}@${packageDetails.version} already published or staged, skipping`
|
||||
)
|
||||
return
|
||||
}
|
||||
throw e
|
||||
}
|
||||
const elapsed = ((Date.now() - publishStart) / 1000).toFixed(1)
|
||||
nicelog(
|
||||
`[publish] ${packageDetails.name}@${packageDetails.version} npm publish done (${elapsed}s)`
|
||||
)
|
||||
},
|
||||
{
|
||||
delay: 10_000,
|
||||
numAttempts: 5,
|
||||
}
|
||||
)
|
||||
|
||||
const whenReadable = waitUntilReadable(packageDetails)
|
||||
// Rejections surface when awaited below; without this, one arriving mid-loop would
|
||||
// kill the process in the middle of another package's publish.
|
||||
whenReadable.catch(() => {})
|
||||
readable.set(packageDetails.name, whenReadable)
|
||||
}
|
||||
|
||||
await Promise.all(readable.values())
|
||||
}
|
||||
|
||||
function waitUntilReadable(packageDetails: PackageDetails) {
|
||||
const url = `https://registry.npmjs.org/${packageDetails.name}/${packageDetails.version}`
|
||||
return retry(
|
||||
async ({ attempt, total }) => {
|
||||
const res = await fetch(url, { method: 'HEAD' })
|
||||
if (res.status >= 400) {
|
||||
nicelog(
|
||||
`[verify] ${packageDetails.name}@${packageDetails.version} not readable yet (${res.status}), attempt ${attempt + 1} of ${total}`
|
||||
)
|
||||
throw new Error(`Package not found: ${url} (${res.status})`)
|
||||
}
|
||||
nicelog(`[verify] ${packageDetails.name}@${packageDetails.version} is readable`)
|
||||
},
|
||||
{
|
||||
delay: 10_000,
|
||||
// 15 minutes; the slowest package took over 8 minutes in September 2026.
|
||||
numAttempts: 90,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function retry(
|
||||
fn: (args: { attempt: number; remaining: number; total: number }) => Promise<void>,
|
||||
opts: {
|
||||
numAttempts: number
|
||||
delay: number
|
||||
}
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let attempts = 0
|
||||
function attempt() {
|
||||
fn({ attempt: attempts, remaining: opts.numAttempts - attempts, total: opts.numAttempts })
|
||||
.then(resolve)
|
||||
.catch((err) => {
|
||||
attempts++
|
||||
if (attempts >= opts.numAttempts) {
|
||||
reject(err)
|
||||
} else {
|
||||
setTimeout(attempt, opts.delay)
|
||||
}
|
||||
})
|
||||
}
|
||||
attempt()
|
||||
})
|
||||
}
|
||||
|
||||
export async function publishProductionDocsAndExamplesAndBemo({
|
||||
gitRef = 'HEAD',
|
||||
}: { gitRef?: string } = {}) {
|
||||
await exec('git', ['push', 'origin', `${gitRef}:docs-production`, `--force`])
|
||||
await exec('git', ['push', 'origin', `${gitRef}:bemo-production`, `--force`])
|
||||
}
|
||||
|
||||
export async function triggerBumpVersionsWorkflow(ghToken: string) {
|
||||
const octokit = new Octokit({ auth: ghToken })
|
||||
await octokit.rest.actions.createWorkflowDispatch({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
workflow_id: 'bump-versions.yml',
|
||||
ref: 'main',
|
||||
})
|
||||
}
|
||||
19
internal/scripts/lib/skip-ci.ts
Normal file
19
internal/scripts/lib/skip-ci.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
// GitHub skips both push- and pull_request-triggered workflows when one of these markers appears
|
||||
// in a commit message, in the positions documented at
|
||||
// https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs
|
||||
// `[^\S\r\n]` rather than `\s`: eating the newlines around a marker on its own body line would
|
||||
// close the blank line separating subject from body, and `git commit --amend -m` would then fold
|
||||
// the whole first paragraph into the subject.
|
||||
const SKIP_CI_MARKER = /[^\S\r\n]*\[(?:skip ci|ci skip|no ci|skip actions|actions skip)\]/gi
|
||||
|
||||
/**
|
||||
* Remove GitHub's skip-ci markers from a commit message or PR title.
|
||||
*
|
||||
* Release-notes PRs carry `[skip ci]` in their title on purpose, so their squash-merge onto `main`
|
||||
* doesn't run the push workflows. The hotfix scripts cherry-pick that same commit onto a branch
|
||||
* whose push *must* trigger a workflow (the release branch's publish, or the hotfixes branch's
|
||||
* production build), so the marker has to go before the push.
|
||||
*/
|
||||
export function stripSkipCiMarkers(message: string): string {
|
||||
return message.replace(SKIP_CI_MARKER, '').trim()
|
||||
}
|
||||
42
internal/scripts/lib/sort-unions.ts
Normal file
42
internal/scripts/lib/sort-unions.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { namedTypes } from 'ast-types'
|
||||
import { glob } from 'glob'
|
||||
import { parse, print, visit } from 'recast'
|
||||
|
||||
export function sortUnions(tsbuildDir: string) {
|
||||
for (const file of glob.sync(path.join(tsbuildDir, '**/*.d.ts'))) {
|
||||
try {
|
||||
const code = parse(readFileSync(file, 'utf8'), {
|
||||
parser: require('recast/parsers/typescript'),
|
||||
})
|
||||
|
||||
visit(code, {
|
||||
visitTSUnionType(path) {
|
||||
this.traverse(path)
|
||||
const val = path.value as namedTypes.TSUnionType
|
||||
val.types = val.types.sort((a, b) => {
|
||||
const aText = print(a).code
|
||||
const bText = print(b).code
|
||||
return aText.localeCompare(bText)
|
||||
})
|
||||
return false
|
||||
},
|
||||
visitTSTypeLiteral(path) {
|
||||
this.traverse(path)
|
||||
const val = path.value as namedTypes.TSTypeLiteral
|
||||
val.members = val.members.sort((a, b) => {
|
||||
const aText = print(a).code
|
||||
const bText = print(b).code
|
||||
return aText.localeCompare(bText)
|
||||
})
|
||||
return false
|
||||
},
|
||||
})
|
||||
|
||||
writeFileSync(file, print(code).code)
|
||||
} catch (e) {
|
||||
throw new Error(`Error sorting unions in ${file}`, { cause: e })
|
||||
}
|
||||
}
|
||||
}
|
||||
80
internal/scripts/lib/types.ts
Normal file
80
internal/scripts/lib/types.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
import { T } from '@tldraw/validate'
|
||||
|
||||
export const EXPORT_CONFIG_KEY = 'tldraw_template' as const
|
||||
|
||||
export const TemplateDeployConfig = T.object({
|
||||
// Where the live demo is hosted. The vercel-hosted demos deploy through vercel's own git
|
||||
// integration, so only cloudflare templates carry a `deploy` config.
|
||||
target: T.literalEnum('cloudflare'),
|
||||
// The hostname the demo is served from. Must match the custom domain route in wrangler.toml.
|
||||
host: T.string,
|
||||
})
|
||||
export type TemplateDeployConfig = T.TypeOf<typeof TemplateDeployConfig>
|
||||
|
||||
export const TemplateConfig = T.object({
|
||||
repo: T.string,
|
||||
cli: T.object({
|
||||
name: T.string,
|
||||
description: T.string,
|
||||
shortDescription: T.string.optional(),
|
||||
order: T.number.optional(),
|
||||
}).optional(),
|
||||
deploy: TemplateDeployConfig.optional(),
|
||||
scripts: T.dict(T.string, T.nullable(T.string)).optional(),
|
||||
})
|
||||
export type TemplateConfig = T.TypeOf<typeof TemplateConfig>
|
||||
|
||||
export const PRODUCT_CONFIG_KEY = 'tldraw_product' as const
|
||||
|
||||
/**
|
||||
* Commercial product metadata for a published package. The `stableId` identifies the commercial
|
||||
* component a package belongs to; it is referenced by legal in order forms and must never change,
|
||||
* even if the package is renamed or moved. Several packages can share one stable id. See
|
||||
* `internal/docs/product-stable-ids.md` for the full convention.
|
||||
*/
|
||||
export const ProductConfig = T.object({
|
||||
// Immutable commercial identifier, e.g. `tldraw:sdk-core`. Never changes on rename/refactor.
|
||||
stableId: T.string,
|
||||
// Customer-facing label, matching the wording used on the order form, e.g. 'Collaboration
|
||||
// module'. Unlike stableId this may be reworded; the id is what contracts reference.
|
||||
name: T.string,
|
||||
// `product` is a top-level order form line item; `feature` is an optional part of its parent
|
||||
// that can be licensed on its own, listed indented beneath it on the order form.
|
||||
type: T.literalEnum('product', 'feature'),
|
||||
// The component this one belongs to, e.g. commenting's parent is the collaboration module.
|
||||
// Components without a parent are top-level order form line items.
|
||||
parent: T.string.optional(),
|
||||
// Whether this component is separately licensed rather than part of the standard SDK offering.
|
||||
premium: T.boolean,
|
||||
// For premium components: the license key FLAGS bit (in packages/editor's LicenseManager) that
|
||||
// entitles a customer to this component.
|
||||
licenseFlag: T.string.optional(),
|
||||
})
|
||||
export type ProductConfig = T.TypeOf<typeof ProductConfig>
|
||||
|
||||
export const PackageJson = T.object({
|
||||
name: T.string,
|
||||
private: T.boolean.optional(),
|
||||
workspaces: T.arrayOf(T.string).optional(),
|
||||
[EXPORT_CONFIG_KEY]: TemplateConfig.optional(),
|
||||
[PRODUCT_CONFIG_KEY]: ProductConfig.optional(),
|
||||
license: T.string.optional(),
|
||||
description: T.string.optional(),
|
||||
scripts: T.dict(T.string, T.nullable(T.string)).optional(),
|
||||
dependencies: T.dict(T.string, T.string).optional(),
|
||||
devDependencies: T.dict(T.string, T.string).optional(),
|
||||
peerDependencies: T.dict(T.string, T.string).optional(),
|
||||
optionalDependencies: T.dict(T.string, T.string).optional(),
|
||||
engines: T.dict(T.string, T.string).optional(),
|
||||
packageManager: T.string.optional(),
|
||||
}).allowUnknownProperties()
|
||||
export type PackageJson = T.TypeOf<typeof PackageJson>
|
||||
|
||||
export const TsConfigJson = T.object({
|
||||
references: T.arrayOf(
|
||||
T.object({
|
||||
path: T.string,
|
||||
})
|
||||
).optional(),
|
||||
}).allowUnknownProperties()
|
||||
export type TsConfigJson = T.TypeOf<typeof TsConfigJson>
|
||||
98
internal/scripts/lib/upload-static-assets.ts
Normal file
98
internal/scripts/lib/upload-static-assets.ts
Normal file
@@ -0,0 +1,98 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
|
||||
import mime from 'mime'
|
||||
import { exec } from './exec'
|
||||
import { makeEnv } from './makeEnv'
|
||||
|
||||
const env = makeEnv(['R2_ACCESS_KEY_ID', 'R2_ACCESS_KEY_SECRET'])
|
||||
|
||||
const R2_URL = 'https://c34edc4e76350954b63adebde86d5eb1.r2.cloudflarestorage.com'
|
||||
const R2_BUCKET = 'cdn'
|
||||
const ASSETS_FOLDER = './packages/assets'
|
||||
|
||||
// How many files to upload to the CDN at once.
|
||||
const UPLOAD_CONCURRENCY = 16
|
||||
|
||||
const R2 = new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: R2_URL,
|
||||
credentials: {
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_ACCESS_KEY_SECRET,
|
||||
},
|
||||
})
|
||||
|
||||
async function uploadFile(key: string, fullPath: string) {
|
||||
const fileStream = fs.createReadStream(fullPath)
|
||||
const contentType = mime.getType(fullPath) ?? 'application/octet-stream'
|
||||
const uploadParams = {
|
||||
Bucket: R2_BUCKET,
|
||||
Key: key,
|
||||
Body: fileStream,
|
||||
ContentType: contentType,
|
||||
// these assets will never change, so we can cache them forever:
|
||||
CacheControl: 'public, max-age=31536000, immutable',
|
||||
}
|
||||
await R2.send(new PutObjectCommand(uploadParams))
|
||||
process.stdout.write(` • ${key} ✔️\n`)
|
||||
}
|
||||
|
||||
function collectFiles(
|
||||
prefix: string,
|
||||
directoryPath: string
|
||||
): Array<{ key: string; fullPath: string }> {
|
||||
const files: Array<{ key: string; fullPath: string }> = []
|
||||
|
||||
for (const entry of fs.readdirSync(directoryPath, { withFileTypes: true })) {
|
||||
const fullPath = path.join(directoryPath, entry.name)
|
||||
const key = path.join(prefix, entry.name)
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...collectFiles(key, fullPath))
|
||||
} else if (entry.isFile()) {
|
||||
files.push({ key, fullPath })
|
||||
}
|
||||
}
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
export async function uploadStaticAssets(version: string) {
|
||||
try {
|
||||
await exec('yarn', ['refresh-assets'], { env: { ALLOW_REFRESH_ASSETS_CHANGES: '1' } })
|
||||
|
||||
const entries = fs.readdirSync(ASSETS_FOLDER, { withFileTypes: true })
|
||||
|
||||
// Gather every file across all asset folders up front so we can upload
|
||||
// them with bounded concurrency instead of one at a time.
|
||||
const files: Array<{ key: string; fullPath: string }> = []
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith('.')) continue
|
||||
if (!entry.isDirectory()) continue
|
||||
|
||||
const folderName = entry.name
|
||||
files.push(...collectFiles(`${version}/${folderName}`, `${ASSETS_FOLDER}/${folderName}`))
|
||||
}
|
||||
|
||||
console.log(`Uploading ${files.length} static assets to CDN...`)
|
||||
|
||||
// Upload files in parallel, keeping at most UPLOAD_CONCURRENCY requests in
|
||||
// flight at a time. Workers pull from a shared queue until it's empty.
|
||||
let next = 0
|
||||
async function worker() {
|
||||
while (next < files.length) {
|
||||
const { key, fullPath } = files[next++]
|
||||
await uploadFile(key, fullPath)
|
||||
}
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(UPLOAD_CONCURRENCY, files.length) }, () => worker())
|
||||
)
|
||||
|
||||
console.log('Uploaded!')
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
27
internal/scripts/lib/worker-externals.ts
Normal file
27
internal/scripts/lib/worker-externals.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Node builtins (and cloudflare modules) that must stay external when bundling a worker
|
||||
* entrypoint with esbuild. With nodejs_compat these resolve at runtime in workerd; esbuild just
|
||||
* needs to be told not to bundle them. Shared between the dev size reporter
|
||||
* (workers/dev.ts) and the CI bundle-size check (check-worker-bundle.ts) so the lists can't drift.
|
||||
*/
|
||||
export const WORKER_EXTERNAL_DEPS = [
|
||||
'cloudflare:*',
|
||||
'crypto',
|
||||
'tls',
|
||||
'net',
|
||||
'stream',
|
||||
'fs',
|
||||
'os',
|
||||
'perf_hooks',
|
||||
'path',
|
||||
'dns',
|
||||
'util',
|
||||
'util/types',
|
||||
'node:child_process',
|
||||
'node:events',
|
||||
'node:path',
|
||||
'node:process',
|
||||
'node:os',
|
||||
'node:timers',
|
||||
'node:util',
|
||||
]
|
||||
62
internal/scripts/lib/workspace.ts
Normal file
62
internal/scripts/lib/workspace.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import path from 'path'
|
||||
import { glob } from 'glob'
|
||||
import { REPO_ROOT, readJsonIfExists } from './file'
|
||||
import { PackageJson } from './types'
|
||||
|
||||
export interface Package {
|
||||
packageJson: PackageJson
|
||||
relativePath: string
|
||||
path: string
|
||||
name: string
|
||||
}
|
||||
|
||||
async function readPackage(packageJsonFile: string): Promise<Package> {
|
||||
const packageJsonPath = path.resolve(packageJsonFile)
|
||||
const packagePath = path.dirname(packageJsonPath)
|
||||
const relativePath = path.relative(REPO_ROOT, packagePath)
|
||||
|
||||
const packageJsonRaw = await readJsonIfExists(packageJsonFile)
|
||||
if (!packageJsonRaw) {
|
||||
throw new Error(`No package.json found at ${packageJsonPath}`)
|
||||
}
|
||||
|
||||
let packageJson: PackageJson
|
||||
try {
|
||||
packageJson = PackageJson.validate(packageJsonRaw)
|
||||
} catch (e: any) {
|
||||
throw new Error(`Invalid package.json in ${relativePath}: ${e.message}`)
|
||||
}
|
||||
|
||||
return {
|
||||
packageJson,
|
||||
relativePath,
|
||||
path: packagePath,
|
||||
name: packageJson.name,
|
||||
}
|
||||
}
|
||||
|
||||
async function getChildWorkspaces(parent: Package): Promise<Package[]> {
|
||||
if (!parent.packageJson.workspaces) return []
|
||||
|
||||
const foundPackages = []
|
||||
for (const workspace of parent.packageJson.workspaces) {
|
||||
const workspacePath = path.join(parent.path, workspace)
|
||||
for (const packageJsonFilePath of glob.sync(path.join(workspacePath, 'package.json'))) {
|
||||
const child = await readPackage(packageJsonFilePath)
|
||||
foundPackages.push(child)
|
||||
if (child.packageJson.workspaces) {
|
||||
foundPackages.push(...(await getChildWorkspaces(child)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return foundPackages
|
||||
}
|
||||
|
||||
export async function getRootPackage() {
|
||||
return await readPackage(path.join(REPO_ROOT, 'package.json'))
|
||||
}
|
||||
|
||||
export async function getAllWorkspacePackages() {
|
||||
return await getChildWorkspaces(await getRootPackage())
|
||||
}
|
||||
414
internal/scripts/license-report.ts
Normal file
414
internal/scripts/license-report.ts
Normal file
@@ -0,0 +1,414 @@
|
||||
// Reports the third-party licenses in packages/* as html (for printing to PDF), markdown, and — in
|
||||
// `--prod` mode — the docs page at apps/docs/content/community/dependencies.mdx.
|
||||
//
|
||||
// Only published packages/* workspaces are covered: nothing else ships to SDK consumers.
|
||||
|
||||
import { execSync } from 'child_process'
|
||||
import { readFileSync, writeFileSync } from 'fs'
|
||||
import { relative } from 'path'
|
||||
import { glob } from 'glob'
|
||||
import { exec } from './lib/exec'
|
||||
|
||||
// `remoteVersion` is omitted: it's the newest published version matching our range, so it changes
|
||||
// whenever an upstream maintainer publishes, committing a new page on unrelated PRs. The rest is a
|
||||
// function of the lockfile.
|
||||
const FIELDS = [
|
||||
'department',
|
||||
'relatedTo',
|
||||
'name',
|
||||
'licensePeriod',
|
||||
'material',
|
||||
'licenseType',
|
||||
'link',
|
||||
'installedVersion',
|
||||
'definedVersion',
|
||||
'author',
|
||||
] as const
|
||||
|
||||
const COLUMN_LABELS = [
|
||||
'department',
|
||||
'related to',
|
||||
'name',
|
||||
'license period',
|
||||
'material / not material',
|
||||
'license type',
|
||||
'link',
|
||||
'installed version',
|
||||
'defined version',
|
||||
'author',
|
||||
]
|
||||
|
||||
const DOCS_PAGE_PATH = 'apps/docs/content/community/dependencies.mdx'
|
||||
|
||||
interface ReportRow {
|
||||
department: string
|
||||
relatedTo: string
|
||||
name: string
|
||||
licensePeriod: string
|
||||
material: string
|
||||
licenseType: string
|
||||
link: string
|
||||
installedVersion: string
|
||||
definedVersion: string
|
||||
author: string
|
||||
}
|
||||
|
||||
interface InlineLicense {
|
||||
file: string
|
||||
licenseType: string
|
||||
copyright: string
|
||||
licenseUrl: string
|
||||
}
|
||||
|
||||
function parseInlineLicenses(rootDir: string): InlineLicense[] {
|
||||
const licenses: InlineLicense[] = []
|
||||
|
||||
// Find all files with /*! comments
|
||||
const files = glob.sync('**/*.{ts,tsx,js,jsx,css}', {
|
||||
cwd: rootDir,
|
||||
ignore: [
|
||||
'**/node_modules/**',
|
||||
'**/dist/**',
|
||||
'**/build/**',
|
||||
'**/.next/**',
|
||||
'apps/**',
|
||||
'internal/**',
|
||||
'templates/**',
|
||||
],
|
||||
absolute: true,
|
||||
})
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = readFileSync(file, 'utf-8')
|
||||
|
||||
// Find all /*! comment blocks in the file (using global flag)
|
||||
const commentMatches = content.matchAll(/\/\*!([\s\S]*?)\*\//g)
|
||||
|
||||
for (const commentMatch of commentMatches) {
|
||||
const comment = commentMatch[1]
|
||||
|
||||
// Parse license information
|
||||
let licenseType = 'Unknown'
|
||||
let copyright = ''
|
||||
let licenseUrl = ''
|
||||
|
||||
// Extract license type (MIT License, BSD License, Apache License, etc.)
|
||||
const licenseMatch = comment.match(/(MIT|BSD|Apache|ISC)[\s]*License/i)
|
||||
if (licenseMatch) {
|
||||
licenseType = licenseMatch[0].replace('License', '').trim()
|
||||
}
|
||||
|
||||
// Extract copyright
|
||||
const copyrightMatch = comment.match(/Copyright[\s]*(?:\(c\))?[\s]*(.+?)(?:\n|$)/i)
|
||||
if (copyrightMatch) {
|
||||
copyright = copyrightMatch[1].replace(/^[:\-–—\s]+/, '').trim()
|
||||
}
|
||||
|
||||
// Extract license URL
|
||||
const licenseUrlMatch = comment.match(
|
||||
/(?:MIT|BSD|Apache|ISC)[\s]*License:\s*(https?:\/\/[^\s\n]+)/i
|
||||
)
|
||||
if (licenseUrlMatch) {
|
||||
licenseUrl = licenseUrlMatch[1]
|
||||
}
|
||||
|
||||
const relativePath = relative(rootDir, file)
|
||||
licenses.push({
|
||||
file: relativePath,
|
||||
licenseType,
|
||||
copyright,
|
||||
licenseUrl,
|
||||
})
|
||||
}
|
||||
} catch (_e) {
|
||||
// Skip files that can't be read
|
||||
}
|
||||
}
|
||||
|
||||
return licenses.sort((a, b) => a.file.localeCompare(b.file))
|
||||
}
|
||||
|
||||
function getWorkspaces(): string[] {
|
||||
return execSync('yarn workspaces list', { encoding: 'utf-8' })
|
||||
.split('\n')
|
||||
.map((line) => line.split(': ')[1])
|
||||
.filter((location) => location && location.startsWith('packages/'))
|
||||
.filter((location) => {
|
||||
// Private packages (dotcom-shared, worker-shared) never reach npm, so their dependencies
|
||||
// aren't part of what an SDK consumer installs.
|
||||
const manifest = JSON.parse(readFileSync(`${location}/package.json`, 'utf-8'))
|
||||
return !manifest.private
|
||||
})
|
||||
.sort()
|
||||
}
|
||||
|
||||
async function getReportRows(location: string, only: string): Promise<ReportRow[]> {
|
||||
// `yarn exec` rather than `yarn license-report`, which would resolve to the root script that
|
||||
// runs this file.
|
||||
const output = await exec(
|
||||
'yarn',
|
||||
[
|
||||
'exec',
|
||||
'license-report',
|
||||
`--package=${location}/package.json`,
|
||||
'--department.value=tldraw',
|
||||
'--relatedTo.label=Package',
|
||||
`--relatedTo.value=${location}`,
|
||||
'--output=json',
|
||||
`--only=${only}`,
|
||||
...FIELDS.map((field) => `--fields=${field}`),
|
||||
],
|
||||
{ processStdoutLine: () => {}, processStderrLine: () => {} }
|
||||
)
|
||||
|
||||
const start = output.indexOf('[')
|
||||
const end = output.lastIndexOf(']')
|
||||
if (start === -1 || end === -1) {
|
||||
throw new Error(`Could not find JSON in license-report output for ${location}`)
|
||||
}
|
||||
|
||||
const rows: ReportRow[] = JSON.parse(output.slice(start, end + 1))
|
||||
|
||||
// Workspace dependencies are our own packages, covered by the tldraw license rather than a
|
||||
// third party's. license-report can't resolve them either, so they'd read as "n/a".
|
||||
return rows.filter((row) => !row.definedVersion.startsWith('workspace:'))
|
||||
}
|
||||
|
||||
/** Turn the various git remote spellings npm records into something a reader can click. */
|
||||
function toHttpsUrl(link: string): string | null {
|
||||
if (!link || link === 'n/a') return null
|
||||
const https = link
|
||||
.replace(/^git\+ssh:\/\/git@/, 'https://')
|
||||
.replace(/^git\+https:\/\//, 'https://')
|
||||
.replace(/^git\+http:\/\//, 'http://')
|
||||
.replace(/^git:\/\//, 'https://')
|
||||
.replace(/\.git$/, '')
|
||||
return https.startsWith('http') ? https : null
|
||||
}
|
||||
|
||||
function escapeCell(value: string): string {
|
||||
return value.replace(/\|/g, '\\|')
|
||||
}
|
||||
|
||||
/**
|
||||
* MDX reads `<` as the start of a JSX tag and `{` as the start of an expression, so an author string
|
||||
* like `Sindre Sorhus <sindresorhus@gmail.com>` is a parse error that fails the docs build.
|
||||
*/
|
||||
function escapeMdxCell(value: string): string {
|
||||
return escapeCell(value)
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/\{/g, '{')
|
||||
.replace(/\}/g, '}')
|
||||
}
|
||||
|
||||
/** Author and copyright strings carry bracketed emails, which a browser would eat as a tag. */
|
||||
function escapeHtml(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
}
|
||||
|
||||
function generateHtml(rows: ReportRow[], inlineLicenses: InlineLicense[]): string {
|
||||
const headerCells = [...COLUMN_LABELS, 'source']
|
||||
.map((label) => `<th class="string">${label}</th>`)
|
||||
.join('')
|
||||
|
||||
const packageRows = rows
|
||||
.map((row) => {
|
||||
const link = toHttpsUrl(row.link)
|
||||
const cells = FIELDS.map((field) =>
|
||||
field === 'link' && link
|
||||
? `<a href="${escapeHtml(link)}">${escapeHtml(link)}</a>`
|
||||
: escapeHtml(row[field])
|
||||
)
|
||||
return `<tr>${cells.map((cell) => `<td>${cell}</td>`).join('')}<td></td></tr>`
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
const inlineRows = inlineLicenses
|
||||
.map((license) => {
|
||||
const licenseLink = license.licenseUrl
|
||||
? `<a href="${escapeHtml(license.licenseUrl)}">License</a>`
|
||||
: ''
|
||||
const cells = [
|
||||
'tldraw',
|
||||
'Inline Code',
|
||||
escapeHtml(license.file),
|
||||
'',
|
||||
'not material',
|
||||
escapeHtml(license.licenseType),
|
||||
licenseLink,
|
||||
'',
|
||||
'',
|
||||
escapeHtml(license.copyright),
|
||||
'',
|
||||
]
|
||||
return `<tr>${cells.map((cell) => `<td>${cell}</td>`).join('')}</tr>`
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
return `
|
||||
<html>
|
||||
<body>
|
||||
<table><thead><tr>${headerCells}</tr></thead><tbody>
|
||||
${packageRows}
|
||||
<tr></tr>
|
||||
${inlineRows}
|
||||
</tbody></table>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
}
|
||||
|
||||
function generateMarkdown(rows: ReportRow[], inlineLicenses: InlineLicense[]): string {
|
||||
const headers = [...COLUMN_LABELS, 'source']
|
||||
|
||||
const packageRows = rows.map((row) => {
|
||||
const link = toHttpsUrl(row.link)
|
||||
return FIELDS.map((field) =>
|
||||
field === 'link' ? (link ? `[${link}](${link})` : row.link) : row[field]
|
||||
).concat('')
|
||||
})
|
||||
|
||||
const inlineRows = inlineLicenses.map((license) => [
|
||||
'tldraw',
|
||||
'Inline Code',
|
||||
license.file,
|
||||
'',
|
||||
'not material',
|
||||
license.licenseType,
|
||||
license.licenseUrl ? `[License](${license.licenseUrl})` : '',
|
||||
'',
|
||||
'',
|
||||
license.copyright,
|
||||
'',
|
||||
])
|
||||
|
||||
return `# License Report
|
||||
|
||||
| ${headers.join(' | ')} |
|
||||
| ${headers.map(() => '---').join(' | ')} |
|
||||
${[...packageRows, ...inlineRows].map((row) => `| ${row.map(escapeCell).join(' | ')} |`).join('\n')}
|
||||
`
|
||||
}
|
||||
|
||||
function generateDocsPage(rows: ReportRow[], inlineLicenses: InlineLicense[]): string {
|
||||
// One row per dependency rather than per declaring package. Keyed by version too: two packages
|
||||
// resolving the same dependency to different versions are two things to report, not one.
|
||||
const byNameAndVersion = new Map<string, { row: ReportRow; usedBy: string[] }>()
|
||||
for (const row of rows) {
|
||||
const key = `${row.name}@${row.installedVersion}`
|
||||
const existing = byNameAndVersion.get(key)
|
||||
if (existing) {
|
||||
existing.usedBy.push(row.relatedTo)
|
||||
} else {
|
||||
byNameAndVersion.set(key, { row, usedBy: [row.relatedTo] })
|
||||
}
|
||||
}
|
||||
|
||||
const dependencyRows = [...byNameAndVersion.values()]
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.row.name.localeCompare(b.row.name) ||
|
||||
a.row.installedVersion.localeCompare(b.row.installedVersion)
|
||||
)
|
||||
.map(({ row, usedBy }) => {
|
||||
const link = toHttpsUrl(row.link)
|
||||
const name = escapeMdxCell(row.name)
|
||||
const packages = usedBy
|
||||
.map((location) => `\`${location.replace(/^packages\//, '')}\``)
|
||||
.join(', ')
|
||||
return `| ${link ? `[${name}](${link})` : name} | ${escapeMdxCell(row.installedVersion)} | ${escapeMdxCell(row.licenseType)} | ${packages} |`
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
const inlineRows = inlineLicenses
|
||||
.map((license) => {
|
||||
const type = escapeMdxCell(license.licenseType)
|
||||
return `| \`${license.file}\` | ${license.licenseUrl ? `[${type}](${license.licenseUrl})` : type} | ${escapeMdxCell(license.copyright)} |`
|
||||
})
|
||||
.join('\n')
|
||||
|
||||
const inlineSection = inlineLicenses.length
|
||||
? `
|
||||
## Inline code
|
||||
|
||||
Some third-party code is included directly in our source rather than as a dependency. Each of these files carries its license in a comment at the top.
|
||||
|
||||
| File | License | Copyright |
|
||||
| --- | --- | --- |
|
||||
${inlineRows}
|
||||
`
|
||||
: ''
|
||||
|
||||
return `---
|
||||
title: Dependencies
|
||||
description: The third-party packages that the tldraw SDK depends on, and the licenses they are published under.
|
||||
status: published
|
||||
order: 3
|
||||
keywords:
|
||||
- license
|
||||
- licenses
|
||||
- dependencies
|
||||
- third party
|
||||
- open source
|
||||
- compliance
|
||||
---
|
||||
|
||||
{/* This page is generated by \`yarn license-report --prod\`. Do not edit it by hand. */}
|
||||
|
||||
The tldraw SDK itself is published under the [tldraw license](/legal/tldraw-license). It also depends on the third-party packages listed below, each under its own license.
|
||||
|
||||
This covers the runtime dependencies that the \`@tldraw/*\` packages declare directly. It does not include their own transitive dependencies, our development and build tooling, or anything in the tldraw.com application. If you need a full transitive list for the exact versions you have installed, generate one from your own lockfile.
|
||||
|
||||
## Dependencies
|
||||
|
||||
| Dependency | Version | License | Used by |
|
||||
| --- | --- | --- | --- |
|
||||
${dependencyRows}
|
||||
${inlineSection}`
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const devOnly = process.argv.includes('--dev')
|
||||
const prodOnly = process.argv.includes('--prod')
|
||||
const only = devOnly ? 'dev' : prodOnly ? 'prod' : 'dev,prod,peer,opt'
|
||||
|
||||
// Not tolerated per-workspace: a registry blip would silently drop that workspace's packages,
|
||||
// and CI would commit the truncated list to the public page.
|
||||
const rows: ReportRow[] = []
|
||||
for (const location of getWorkspaces()) {
|
||||
console.log('running license-report in', location)
|
||||
rows.push(...(await getReportRows(location, only)))
|
||||
}
|
||||
|
||||
console.log('\nSearching for inline license comments (/*!)...')
|
||||
const inlineLicenses = parseInlineLicenses(process.cwd())
|
||||
console.log(`Found ${inlineLicenses.length} files with inline license comments`)
|
||||
|
||||
const suffix = prodOnly ? '-prod' : devOnly ? '-dev' : ''
|
||||
writeFileSync(`license-report${suffix}.html`, generateHtml(rows, inlineLicenses))
|
||||
writeFileSync(`license-report${suffix}.md`, generateMarkdown(rows, inlineLicenses))
|
||||
console.log(`\nGenerated license-report${suffix}.html and license-report${suffix}.md`)
|
||||
|
||||
// The docs page lists what SDK consumers actually ship, so it only tracks the prod report.
|
||||
if (prodOnly) {
|
||||
writeFileSync(DOCS_PAGE_PATH, generateDocsPage(rows, inlineLicenses))
|
||||
// oxfmt pads markdown table columns, so an unformatted page here would be rewritten by the
|
||||
// next `yarn format` and then reverted by the next regeneration.
|
||||
await exec('yarn', ['oxfmt', '--write', DOCS_PAGE_PATH], {
|
||||
processStdoutLine: () => {},
|
||||
processStderrLine: () => {},
|
||||
})
|
||||
console.log(`Generated ${DOCS_PAGE_PATH}`)
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error(e)
|
||||
process.exit(1)
|
||||
})
|
||||
72
internal/scripts/lint-staged.ts
Normal file
72
internal/scripts/lint-staged.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env tsx
|
||||
import { execFile } from 'child_process'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
|
||||
async function main() {
|
||||
const output: string[] = []
|
||||
const errors: string[] = []
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
const childProcess = execFile('yarn', ['lint-staged'], { cwd: REPO_ROOT }, (err) => {
|
||||
if (err) {
|
||||
// Display a clear error summary at the end
|
||||
console.error('\n' + '='.repeat(80))
|
||||
console.error('❌ lint-staged failed')
|
||||
console.error('='.repeat(80) + '\n')
|
||||
|
||||
// Show error output
|
||||
if (errors.length > 0) {
|
||||
console.error('Error output:')
|
||||
console.error(errors.join(''))
|
||||
console.error('')
|
||||
}
|
||||
|
||||
// Show any captured output that might contain useful info
|
||||
const outputStr = output.join('')
|
||||
if (outputStr) {
|
||||
// Filter for lines that look like errors
|
||||
const errorLines = outputStr.split('\n').filter((line) => {
|
||||
const lower = line.toLowerCase()
|
||||
return (
|
||||
lower.includes('error') ||
|
||||
lower.includes('failed') ||
|
||||
lower.includes('✖') ||
|
||||
lower.includes('❌')
|
||||
)
|
||||
})
|
||||
|
||||
if (errorLines.length > 0) {
|
||||
console.error('Relevant error messages:')
|
||||
errorLines.forEach((line) => console.error(line))
|
||||
console.error('')
|
||||
}
|
||||
}
|
||||
|
||||
console.error('Please fix the errors above and try again.')
|
||||
console.error('='.repeat(80) + '\n')
|
||||
process.exit(1)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
|
||||
// Capture and display stdout in real-time
|
||||
childProcess.stdout?.on('data', (chunk) => {
|
||||
const text = chunk.toString()
|
||||
output.push(text)
|
||||
process.stdout.write(text)
|
||||
})
|
||||
|
||||
// Capture and display stderr in real-time
|
||||
childProcess.stderr?.on('data', (chunk) => {
|
||||
const text = chunk.toString()
|
||||
errors.push(text)
|
||||
process.stderr.write(text)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Unexpected error:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
62
internal/scripts/lint.ts
Normal file
62
internal/scripts/lint.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { execSync } from 'child_process'
|
||||
import * as path from 'path'
|
||||
import { exec } from './lib/exec'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
|
||||
const OXFMT_EXTENSIONS = 'js,jsx,ts,tsx,cjs,mjs,css,md,mdx,html,yml,yaml'
|
||||
const CONTEXT_COMPAT_FILE_PATTERN = /^(apps|packages|templates)\/.+\/(?:CLAUDE|AGENTS)\.md$/i
|
||||
|
||||
async function main() {
|
||||
const shouldFix = process.argv.includes('--fix')
|
||||
const currentOnly = process.argv.includes('--current')
|
||||
|
||||
let target: string
|
||||
let fmtGlob: string
|
||||
if (currentOnly) {
|
||||
const changedFiles = execSync('git diff --name-only --diff-filter d', {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
})
|
||||
.trim()
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
|
||||
const extPattern = new RegExp(`\\.(${OXFMT_EXTENSIONS.replace(/,/g, '|')})$`)
|
||||
const matchingFiles = changedFiles.filter((f) => extPattern.test(f))
|
||||
if (matchingFiles.length === 0) {
|
||||
console.log('No files to lint.')
|
||||
return
|
||||
}
|
||||
// Force one TypeScript file through oxlint so repo-level markdown checks
|
||||
// still run when current-only changes only touch CLAUDE.md / AGENTS.md.
|
||||
if (
|
||||
matchingFiles.some((file) => CONTEXT_COMPAT_FILE_PATTERN.test(file)) &&
|
||||
!matchingFiles.includes('internal/scripts/lint.ts')
|
||||
) {
|
||||
matchingFiles.push('internal/scripts/lint.ts')
|
||||
}
|
||||
target = matchingFiles.join(' ')
|
||||
fmtGlob = target
|
||||
} else {
|
||||
const relativeCwd = path.relative(REPO_ROOT, process.cwd())
|
||||
target = relativeCwd || '.'
|
||||
fmtGlob = relativeCwd
|
||||
? `${relativeCwd}/**/*.{${OXFMT_EXTENSIONS}}`
|
||||
: `**/*.{${OXFMT_EXTENSIONS}}`
|
||||
}
|
||||
|
||||
try {
|
||||
await exec(
|
||||
'yarn',
|
||||
['oxfmt', shouldFix ? '--write' : '--check', '--no-error-on-unmatched-pattern', fmtGlob],
|
||||
{ pwd: REPO_ROOT }
|
||||
)
|
||||
if (!shouldFix) {
|
||||
await exec('yarn', ['oxlint', ...target.split(' ')], { pwd: REPO_ROOT })
|
||||
}
|
||||
} catch {
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
1204
internal/scripts/oxlint/tldraw-plugin.mjs
Normal file
1204
internal/scripts/oxlint/tldraw-plugin.mjs
Normal file
File diff suppressed because it is too large
Load Diff
86
internal/scripts/package.json
Normal file
86
internal/scripts/package.json
Normal file
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"name": "@tldraw/scripts",
|
||||
"description": "build scripts",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"bin": {
|
||||
"process-compose": "./process-compose-bin.cjs"
|
||||
},
|
||||
"author": {
|
||||
"name": "tldraw Inc.",
|
||||
"email": "hello@tldraw.com"
|
||||
},
|
||||
"homepage": "https://tldraw.dev",
|
||||
"license": "SEE LICENSE IN LICENSE.md",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/tldraw/tldraw"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/tldraw/tldraw/issues"
|
||||
},
|
||||
"keywords": [
|
||||
"tldraw",
|
||||
"sdk",
|
||||
"drawing",
|
||||
"app",
|
||||
"development",
|
||||
"whiteboard",
|
||||
"canvas",
|
||||
"infinite"
|
||||
],
|
||||
"devDependencies": {
|
||||
"@actions/github": "^6.0.0",
|
||||
"@aws-sdk/client-s3": "^3.735.0",
|
||||
"@aws-sdk/lib-storage": "^3.735.0",
|
||||
"@types/diff": "^7.0.1",
|
||||
"@types/is-ci": "^3.0.4",
|
||||
"@types/node": "^22.15.31",
|
||||
"@types/pg": "^8.11.11",
|
||||
"ast-types": "^0.14.2",
|
||||
"cross-fetch": "^3.2.0",
|
||||
"esbuild": "^0.28.0",
|
||||
"glob": "^13.0.0",
|
||||
"gray-matter": "^4.0.3",
|
||||
"is-ci": "^3.0.1",
|
||||
"kleur": "^4.1.5",
|
||||
"lazyrepo": "0.0.0-alpha.27",
|
||||
"oxfmt": "^0.41.0",
|
||||
"purgecss": "^5.0.0",
|
||||
"recast": "^0.22.0",
|
||||
"rimraf": "^4.4.1",
|
||||
"semver": "^7.6.3",
|
||||
"svgo": "^3.3.3",
|
||||
"typescript": "^7.0.2",
|
||||
"vite": "^8.0.16",
|
||||
"vite-plugin-circular-dependency": "^0.5.0"
|
||||
},
|
||||
"scripts": {
|
||||
"lint": "yarn run -T tsx lint.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@lokalise/node-api": "^12.8.0",
|
||||
"@octokit/rest": "^20.1.1",
|
||||
"@tldraw/dotcom-shared": "workspace:*",
|
||||
"@tldraw/utils": "workspace:*",
|
||||
"@tldraw/validate": "workspace:*",
|
||||
"@types/minimist": "^1.2.5",
|
||||
"@types/proper-lockfile": "^4.1.4",
|
||||
"@types/tmp": "^0.2.6",
|
||||
"diff": "^9.0.0",
|
||||
"docx": "^9.5.1",
|
||||
"ignore": "^5.3.2",
|
||||
"json5": "^2.2.3",
|
||||
"jszip": "^3.10.1",
|
||||
"mime": "^4.0.6",
|
||||
"minimist": "^1.2.8",
|
||||
"nanoid": "^5.1.16",
|
||||
"octokit": "^3.2.1",
|
||||
"pg": "^8.16.3",
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"strip-ansi": "^7.1.0",
|
||||
"tar": "^7.5.11",
|
||||
"tmp": "^0.2.3",
|
||||
"toml": "^3.0.0"
|
||||
}
|
||||
}
|
||||
21
internal/scripts/postpack.sh
Executable file
21
internal/scripts/postpack.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
echo "[postpack] restoring package.json"
|
||||
|
||||
restore_generated_file() {
|
||||
if [ -f "$1.bak" ]; then
|
||||
mv "$1.bak" "$1"
|
||||
elif [ -f "$1.generated" ]; then
|
||||
rm -rf "$1" "$1.generated"
|
||||
fi
|
||||
}
|
||||
|
||||
# our prepack script makes the following changes we need to reverse
|
||||
# - modifies the package.json, saving previous as package.json.bak
|
||||
mv package.json.bak package.json
|
||||
# - generates tldraw package documentation files
|
||||
restore_generated_file DOCS.md
|
||||
restore_generated_file RELEASE_NOTES.md
|
||||
# - generates an index.d.ts file
|
||||
rm -rf index.d.ts
|
||||
97
internal/scripts/prepack.ts
Normal file
97
internal/scripts/prepack.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
import { execSync } from 'child_process'
|
||||
import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { pathToFileURL } from 'url'
|
||||
import { glob } from 'glob'
|
||||
import { generateTldrawPackageDocs } from './generate-tldraw-package-docs'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
function markGeneratedFile(sourcePackageDir: string, fileName: string) {
|
||||
const filePath = path.join(sourcePackageDir, fileName)
|
||||
if (existsSync(filePath)) {
|
||||
copyFileSync(filePath, `${filePath}.bak`)
|
||||
} else {
|
||||
writeFileSync(`${filePath}.generated`, '')
|
||||
}
|
||||
}
|
||||
|
||||
/** Prepares the package for publishing. the tarball in case it will be written to disk. */
|
||||
export async function preparePackage({ sourcePackageDir }: { sourcePackageDir: string }) {
|
||||
if (!existsSync(path.join(sourcePackageDir, 'src/index.ts'))) {
|
||||
throw new Error(`No src/index.ts file found in '${sourcePackageDir}'!`)
|
||||
}
|
||||
|
||||
const manifest = JSON.parse(readFileSync(path.join(sourcePackageDir, 'package.json'), 'utf8'))
|
||||
const packageName = manifest.name ?? path.basename(sourcePackageDir)
|
||||
const startTime = Date.now()
|
||||
nicelog(`[prepack] ${packageName} starting...`)
|
||||
|
||||
execSync('yarn run -T lazy build', { cwd: sourcePackageDir, stdio: 'inherit' })
|
||||
|
||||
// save package.json and reinstate it in postpack
|
||||
copyFileSync(
|
||||
path.join(sourcePackageDir, 'package.json'),
|
||||
path.join(sourcePackageDir, 'package.json.bak')
|
||||
)
|
||||
|
||||
const cssFiles = glob.sync(path.join(sourcePackageDir, '*.css'))
|
||||
|
||||
// Include DOCS.md in the published tarball when present. npm auto-includes
|
||||
// README.md and LICENSE but not DOCS.md, so we have to add it explicitly.
|
||||
const extraFiles = new Set<string>()
|
||||
const docsPath = path.join(sourcePackageDir, 'DOCS.md')
|
||||
if (existsSync(docsPath)) {
|
||||
extraFiles.add('DOCS.md')
|
||||
}
|
||||
if (packageName === 'tldraw') {
|
||||
markGeneratedFile(sourcePackageDir, 'DOCS.md')
|
||||
markGeneratedFile(sourcePackageDir, 'RELEASE_NOTES.md')
|
||||
generateTldrawPackageDocs({ sourcePackageDir })
|
||||
extraFiles.add('DOCS.md')
|
||||
extraFiles.add('RELEASE_NOTES.md')
|
||||
}
|
||||
|
||||
// construct the final package.json
|
||||
// eslint-disable-next-line no-restricted-globals
|
||||
const newManifest = structuredClone({
|
||||
// filter out comments
|
||||
...Object.fromEntries(
|
||||
Object.entries(manifest).filter(([key]) => !key.startsWith('/*') && key !== 'types')
|
||||
),
|
||||
main: 'dist-cjs/index.js',
|
||||
module: 'dist-esm/index.mjs',
|
||||
source: 'src/index.ts',
|
||||
exports: {
|
||||
'.': {
|
||||
import: './dist-esm/index.mjs',
|
||||
require: './dist-cjs/index.js',
|
||||
},
|
||||
...Object.fromEntries(
|
||||
cssFiles.map((file) => [`./${path.basename(file)}`, `./${path.basename(file)}`])
|
||||
),
|
||||
},
|
||||
files: [...(manifest.files ?? []), 'dist-esm', 'dist-cjs', 'src', ...extraFiles],
|
||||
type: undefined,
|
||||
})
|
||||
writeFileSync(
|
||||
path.join(sourcePackageDir, 'package.json'),
|
||||
JSON.stringify(newManifest, null, `\t`)
|
||||
)
|
||||
|
||||
// GOTCHA: Yarn's pack command seems to have a race condition where it doesn't reliably pick up
|
||||
// files, adding a tiny delay seems to fix it, but we make the delay extra long here just to be
|
||||
// safe.
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
|
||||
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1)
|
||||
nicelog(`[prepack] ${packageName} done (${elapsed}s)`)
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
// module was called directly
|
||||
;(async () => {
|
||||
await preparePackage({
|
||||
sourcePackageDir: path.resolve(path.normalize(process.argv[2] ?? process.cwd())),
|
||||
})
|
||||
})()
|
||||
}
|
||||
97
internal/scripts/process-compose-bin.cjs
Executable file
97
internal/scripts/process-compose-bin.cjs
Executable file
@@ -0,0 +1,97 @@
|
||||
#!/usr/bin/env node
|
||||
/* eslint-disable no-console */
|
||||
//
|
||||
// The `process-compose` bin of @tldraw/scripts: self-provisioning process-compose.
|
||||
//
|
||||
// process-compose has no npm package, so installing it is normally a separate brew/curl step. Exposing
|
||||
// it as a workspace bin lets `yarn dev-app` (and anything else) call `process-compose` like any other
|
||||
// tool; on first use this downloads the pinned binary into node_modules/.cache, then execs it. Caching
|
||||
// it there (rather than a top-level dir) treats it like any bundled-binary dep: nuking node_modules
|
||||
// re-fetches it. Only people who actually run the dotcom stack pay the one-time download (Yarn doesn't
|
||||
// run a workspace's postinstall, so the fetch is lazy rather than eager).
|
||||
|
||||
const { spawn, spawnSync } = require('node:child_process')
|
||||
const { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } = require('node:fs')
|
||||
const { arch: osArch, platform: osPlatform } = require('node:os')
|
||||
const { join } = require('node:path')
|
||||
|
||||
// Bump this to upgrade process-compose for everyone; the next `yarn dev-app` re-downloads.
|
||||
const VERSION = 'v1.116.0'
|
||||
|
||||
const repoRoot = join(__dirname, '..', '..')
|
||||
const binDir = join(repoRoot, 'node_modules', '.cache', 'process-compose')
|
||||
const binPath = join(binDir, 'process-compose')
|
||||
const stampPath = join(binDir, '.version')
|
||||
|
||||
function releaseAsset() {
|
||||
const os = osPlatform()
|
||||
if (os !== 'darwin' && os !== 'linux') {
|
||||
throw new Error(
|
||||
`process-compose auto-install only supports macOS and Linux (got ${os}). ` +
|
||||
`Install it manually: https://f1bonacc1.github.io/process-compose/installation/`
|
||||
)
|
||||
}
|
||||
// Map node's arch names to process-compose's release names; throw on anything else rather than
|
||||
// silently downloading a binary that won't run (e.g. 32-bit).
|
||||
const nodeArch = osArch()
|
||||
const arch = nodeArch === 'arm64' ? 'arm64' : nodeArch === 'x64' ? 'amd64' : null
|
||||
if (!arch) {
|
||||
throw new Error(
|
||||
`process-compose auto-install only supports arm64 and x64 (got ${nodeArch}). ` +
|
||||
`Install it manually: https://f1bonacc1.github.io/process-compose/installation/`
|
||||
)
|
||||
}
|
||||
return `process-compose_${os}_${arch}.tar.gz`
|
||||
}
|
||||
|
||||
async function ensureBinary() {
|
||||
const installed =
|
||||
existsSync(binPath) &&
|
||||
existsSync(stampPath) &&
|
||||
readFileSync(stampPath, 'utf8').trim() === VERSION
|
||||
if (installed) return
|
||||
|
||||
const asset = releaseAsset()
|
||||
const url = `https://github.com/F1bonacc1/process-compose/releases/download/${VERSION}/${asset}`
|
||||
console.error(`Fetching process-compose ${VERSION} (${asset})…`)
|
||||
|
||||
const res = await fetch(url, { redirect: 'follow' })
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to download process-compose: ${res.status} ${res.statusText} (${url})`)
|
||||
}
|
||||
|
||||
mkdirSync(binDir, { recursive: true })
|
||||
const tgz = join(binDir, asset)
|
||||
writeFileSync(tgz, Buffer.from(await res.arrayBuffer()))
|
||||
// The tarball contains the `process-compose` binary at its root. tar ships on macOS and Linux.
|
||||
const untar = spawnSync('tar', ['-xzf', tgz, '-C', binDir, 'process-compose'], {
|
||||
stdio: 'inherit',
|
||||
})
|
||||
rmSync(tgz, { force: true })
|
||||
if (untar.status !== 0) throw new Error('Failed to extract the process-compose archive')
|
||||
chmodSync(binPath, 0o755)
|
||||
writeFileSync(stampPath, VERSION)
|
||||
console.error(`process-compose ${VERSION} ready (${binPath})`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await ensureBinary()
|
||||
|
||||
const child = spawn(binPath, process.argv.slice(2), { stdio: 'inherit' })
|
||||
const forward = (signal) => {
|
||||
if (!child.killed) child.kill(signal)
|
||||
}
|
||||
process.on('SIGINT', () => forward('SIGINT'))
|
||||
process.on('SIGTERM', () => forward('SIGTERM'))
|
||||
process.on('SIGHUP', () => forward('SIGHUP'))
|
||||
child.on('error', (error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
child.on('exit', (code, signal) => process.exit(signal ? 1 : (code ?? 0)))
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error)
|
||||
process.exit(1)
|
||||
})
|
||||
69
internal/scripts/product-manifest.ts
Normal file
69
internal/scripts/product-manifest.ts
Normal file
@@ -0,0 +1,69 @@
|
||||
import { join } from 'path'
|
||||
import { REPO_ROOT, readJsonIfExists } from './lib/file'
|
||||
import { PRODUCT_CONFIG_KEY, ProductConfig } from './lib/types'
|
||||
import { getAllWorkspacePackages } from './lib/workspace'
|
||||
|
||||
interface ManifestComponent extends ProductConfig {
|
||||
packages: { name: string; description?: string }[]
|
||||
features: ManifestComponent[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Prints the product manifest: the commercial components customers license, as a tree. Top-level
|
||||
* entries are order form line items; their `features` are the parts that can be licensed
|
||||
* individually, listed indented beneath them on the order form.
|
||||
*
|
||||
* This is the aggregate view of the `tldraw_product` fields in each package's package.json — those
|
||||
* fields are the source of truth, and are validated by `yarn check-packages`. See
|
||||
* internal/docs/product-stable-ids.md.
|
||||
*/
|
||||
async function main() {
|
||||
const lernaJson = (await readJsonIfExists(join(REPO_ROOT, 'lerna.json'))) as {
|
||||
version: string
|
||||
} | null
|
||||
if (!lernaJson) throw new Error('Could not read lerna.json')
|
||||
|
||||
const componentsById = new Map<string, ManifestComponent>()
|
||||
|
||||
for (const { packageJson, relativePath } of await getAllWorkspacePackages()) {
|
||||
if (!relativePath.startsWith('packages/') || packageJson.private) continue
|
||||
const product = packageJson[PRODUCT_CONFIG_KEY]
|
||||
if (!product) continue
|
||||
|
||||
// packages sharing a stable id are required to have identical product metadata; this is
|
||||
// enforced by `yarn check-packages`, so taking the first package's config here is safe
|
||||
const component = componentsById.get(product.stableId) ?? {
|
||||
...product,
|
||||
packages: [],
|
||||
features: [],
|
||||
}
|
||||
component.packages.push({
|
||||
name: packageJson.name,
|
||||
description: packageJson.description,
|
||||
})
|
||||
componentsById.set(product.stableId, component)
|
||||
}
|
||||
|
||||
const byName = (a: { name: string }, b: { name: string }) => a.name.localeCompare(b.name)
|
||||
const all = [...componentsById.values()].sort(byName)
|
||||
for (const component of all) {
|
||||
component.packages.sort(byName)
|
||||
}
|
||||
|
||||
// nest features under the component they belong to. `check-packages` guarantees every parent
|
||||
// resolves, so nothing can be silently dropped here.
|
||||
for (const component of all) {
|
||||
if (component.parent) {
|
||||
componentsById.get(component.parent)!.features.push(component)
|
||||
}
|
||||
}
|
||||
// the standard offering leads, then the premium modules, matching the order form's layout
|
||||
const products = all
|
||||
.filter((component) => !component.parent)
|
||||
.sort((a, b) => Number(a.premium) - Number(b.premium) || byName(a, b))
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(JSON.stringify({ version: lernaJson.version, products }, null, '\t'))
|
||||
}
|
||||
|
||||
main()
|
||||
483
internal/scripts/profile-tsserver.ts
Normal file
483
internal/scripts/profile-tsserver.ts
Normal file
@@ -0,0 +1,483 @@
|
||||
import { execFileSync, spawn } from 'child_process'
|
||||
import { existsSync, mkdirSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
interface TsServerStats {
|
||||
timestamp: string
|
||||
memoryUsage: {
|
||||
heapUsed: number
|
||||
heapTotal: number
|
||||
external: number
|
||||
rss: number
|
||||
}
|
||||
requestCounts: Map<string, number>
|
||||
slowRequests: Array<{
|
||||
command: string
|
||||
file?: string
|
||||
duration: number
|
||||
}>
|
||||
projectInfo?: {
|
||||
configFiles: string[]
|
||||
fileCount: number
|
||||
externalFiles: number
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
nicelog('🔍 Starting TSServer profiling...\n')
|
||||
|
||||
const targetWorkspace = process.argv[2] || 'apps/dotcom/client'
|
||||
const workspacePath = join(REPO_ROOT, targetWorkspace)
|
||||
|
||||
if (!existsSync(join(workspacePath, 'tsconfig.json'))) {
|
||||
console.error(`❌ No tsconfig.json found in ${targetWorkspace}`)
|
||||
console.error(`Usage: yarn profile-tsserver [workspace-path]`)
|
||||
console.error(`Example: yarn profile-tsserver packages/editor`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
nicelog(` Profiling workspace: ${targetWorkspace}\n`)
|
||||
|
||||
const outputDir = join(REPO_ROOT, '.ts-profile')
|
||||
const tsserverLogPath = join(outputDir, 'tsserver.log')
|
||||
const reportPath = join(outputDir, 'tsserver-report.md')
|
||||
|
||||
// Create output directory
|
||||
if (!existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true })
|
||||
}
|
||||
|
||||
const stats: TsServerStats = {
|
||||
timestamp: new Date().toISOString(),
|
||||
memoryUsage: {
|
||||
heapUsed: 0,
|
||||
heapTotal: 0,
|
||||
external: 0,
|
||||
rss: 0,
|
||||
},
|
||||
requestCounts: new Map(),
|
||||
slowRequests: [],
|
||||
}
|
||||
|
||||
nicelog('📊 Collecting TSServer information...\n')
|
||||
|
||||
// Step 1: Get project info
|
||||
nicelog('Step 1/4: Analyzing project structure...')
|
||||
try {
|
||||
const tscPath = join(REPO_ROOT, 'node_modules/.bin/tsc')
|
||||
const listFilesOutput = execFileSync(tscPath, ['--listFiles', '--noEmit'], {
|
||||
cwd: workspacePath,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
})
|
||||
|
||||
const files = listFilesOutput.split('\n').filter((line) => line.trim())
|
||||
const configFiles = files.filter((f) => f.includes('tsconfig.json'))
|
||||
const externalFiles = files.filter((f) => f.includes('node_modules')).length
|
||||
|
||||
stats.projectInfo = {
|
||||
configFiles,
|
||||
fileCount: files.length,
|
||||
externalFiles,
|
||||
}
|
||||
|
||||
nicelog(` Found ${files.length} files (${externalFiles} from node_modules)\n`)
|
||||
} catch {
|
||||
nicelog(' ⚠️ Could not analyze project (non-fatal)\n')
|
||||
}
|
||||
|
||||
// Step 2: Check for existing TSServer logs
|
||||
nicelog('Step 2/4: Checking for IDE TSServer logs...')
|
||||
|
||||
const possibleLogLocations = [
|
||||
// VSCode
|
||||
join(process.env.HOME || '', 'Library/Application Support/Code/logs'),
|
||||
join(process.env.HOME || '', '.config/Code/logs'),
|
||||
// Cursor
|
||||
join(process.env.HOME || '', 'Library/Application Support/Cursor/logs'),
|
||||
join(process.env.HOME || '', '.config/Cursor/logs'),
|
||||
].filter(existsSync)
|
||||
|
||||
if (possibleLogLocations.length > 0) {
|
||||
nicelog(` Found IDE log locations:\n`)
|
||||
possibleLogLocations.forEach((loc) => {
|
||||
nicelog(` - ${loc}\n`)
|
||||
})
|
||||
nicelog(`\n 💡 To enable verbose TSServer logging in VSCode/Cursor:\n`)
|
||||
nicelog(` 1. Open settings (Cmd+,)\n`)
|
||||
nicelog(` 2. Search for "typescript.tsserver.log"\n`)
|
||||
nicelog(` 3. Set to "verbose"\n`)
|
||||
nicelog(` 4. Reload window and reproduce the slowness\n`)
|
||||
nicelog(` 5. Run: TypeScript: Open TS Server log\n\n`)
|
||||
} else {
|
||||
nicelog(' No IDE logs found\n')
|
||||
}
|
||||
|
||||
// Step 3: Start a TSServer instance for profiling
|
||||
nicelog('Step 3/4: Starting TSServer with profiling...')
|
||||
|
||||
const tsserverPath = join(REPO_ROOT, 'node_modules/typescript/lib/tsserver.js')
|
||||
|
||||
nicelog(` Starting TSServer for ${targetWorkspace}...\n`)
|
||||
nicelog(` This will simulate language server requests...\n\n`)
|
||||
|
||||
// Create a test scenario by opening files and requesting completions
|
||||
const testFiles = [
|
||||
join(workspacePath, 'src/main.tsx'),
|
||||
join(workspacePath, 'src/index.tsx'),
|
||||
join(workspacePath, 'src/App.tsx'),
|
||||
].filter(existsSync)
|
||||
|
||||
if (testFiles.length === 0) {
|
||||
nicelog(' ⚠️ No test files found in src/ directory\n')
|
||||
nicelog(' Skipping interactive profiling\n\n')
|
||||
} else {
|
||||
nicelog(` Will profile ${testFiles.length} files...\n\n`)
|
||||
|
||||
// Run TSServer with tracing
|
||||
const tsserver = spawn(
|
||||
'node',
|
||||
[
|
||||
'--trace-warnings',
|
||||
'--max-old-space-size=4096',
|
||||
tsserverPath,
|
||||
'--logVerbosity',
|
||||
'verbose',
|
||||
'--logFile',
|
||||
tsserverLogPath,
|
||||
],
|
||||
{
|
||||
cwd: workspacePath,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
}
|
||||
)
|
||||
|
||||
let requestId = 0
|
||||
const responses: any[] = []
|
||||
|
||||
// Helper to send TSServer requests
|
||||
const sendRequest = (command: string, args: any) => {
|
||||
const req = {
|
||||
seq: ++requestId,
|
||||
type: 'request',
|
||||
command,
|
||||
arguments: args,
|
||||
}
|
||||
tsserver.stdin.write(JSON.stringify(req) + '\n')
|
||||
return requestId
|
||||
}
|
||||
|
||||
// Collect responses
|
||||
let buffer = ''
|
||||
tsserver.stdout.on('data', (data) => {
|
||||
buffer += data.toString()
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.trim().startsWith('{')) {
|
||||
try {
|
||||
const msg = JSON.parse(line)
|
||||
if (msg.type === 'response') {
|
||||
responses.push(msg)
|
||||
}
|
||||
} catch {
|
||||
// Ignore parse errors
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Wait for TSServer to start
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000))
|
||||
|
||||
// Run test scenarios
|
||||
for (const file of testFiles) {
|
||||
nicelog(` Testing ${file.replace(REPO_ROOT, '.')}...\n`)
|
||||
|
||||
// Open file
|
||||
sendRequest('open', { file })
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
// Request completions at various positions
|
||||
sendRequest('completions', { file, line: 1, offset: 1 })
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
|
||||
// Request quick info
|
||||
sendRequest('quickinfo', { file, line: 1, offset: 1 })
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
|
||||
// Request definition
|
||||
sendRequest('definition', { file, line: 1, offset: 1 })
|
||||
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||
}
|
||||
|
||||
// Request project info
|
||||
sendRequest('projectInfo', { file: testFiles[0], needFileNameList: true })
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||
|
||||
// Analyze responses
|
||||
for (const response of responses) {
|
||||
const command = response.command || 'unknown'
|
||||
stats.requestCounts.set(command, (stats.requestCounts.get(command) || 0) + 1)
|
||||
|
||||
// Track slow requests (using performanceData if available)
|
||||
if (response.performanceData) {
|
||||
const duration = response.performanceData.updateGraphDurationMs || 0
|
||||
if (duration > 100) {
|
||||
stats.slowRequests.push({
|
||||
command,
|
||||
duration,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get memory usage
|
||||
sendRequest('status', {})
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
// Cleanup
|
||||
tsserver.kill()
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
nicelog(' ✅ Profiling complete\n\n')
|
||||
}
|
||||
|
||||
// Step 4: Generate report
|
||||
nicelog('Step 4/4: Generating report...')
|
||||
|
||||
const report = generateReport(stats, outputDir, tsserverLogPath)
|
||||
writeFileSync(reportPath, report)
|
||||
nicelog(` ✅ Report saved to ${reportPath}\n\n`)
|
||||
|
||||
// Print summary
|
||||
console.log('='.repeat(80))
|
||||
console.log('📊 TSServer Profile Summary')
|
||||
console.log('='.repeat(80))
|
||||
|
||||
if (stats.projectInfo) {
|
||||
console.log(`Files in project: ${stats.projectInfo.fileCount.toLocaleString()}`)
|
||||
console.log(`External files: ${stats.projectInfo.externalFiles.toLocaleString()}`)
|
||||
console.log(`Config files: ${stats.projectInfo.configFiles.length}`)
|
||||
}
|
||||
|
||||
if (stats.requestCounts.size > 0) {
|
||||
console.log('\nRequest counts:')
|
||||
Array.from(stats.requestCounts.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.forEach(([cmd, count]) => {
|
||||
console.log(` ${cmd}: ${count}`)
|
||||
})
|
||||
}
|
||||
|
||||
if (stats.slowRequests.length > 0) {
|
||||
console.log('\nSlow requests (>100ms):')
|
||||
stats.slowRequests
|
||||
.sort((a, b) => b.duration - a.duration)
|
||||
.slice(0, 5)
|
||||
.forEach((req) => {
|
||||
console.log(` ${req.command}: ${req.duration.toFixed(0)}ms`)
|
||||
})
|
||||
}
|
||||
|
||||
console.log('\n📄 Full report: ' + reportPath)
|
||||
if (existsSync(tsserverLogPath)) {
|
||||
console.log('📋 TSServer log: ' + tsserverLogPath)
|
||||
}
|
||||
console.log('='.repeat(80) + '\n')
|
||||
}
|
||||
|
||||
function generateReport(
|
||||
stats: TsServerStats,
|
||||
_outputDir: string,
|
||||
_tsserverLogPath: string
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
|
||||
lines.push('# TSServer profiling report')
|
||||
lines.push('')
|
||||
lines.push(`**Generated:** ${new Date(stats.timestamp).toLocaleString()}`)
|
||||
lines.push('')
|
||||
|
||||
lines.push('## Summary')
|
||||
lines.push('')
|
||||
|
||||
if (stats.projectInfo) {
|
||||
lines.push('### Project structure')
|
||||
lines.push('')
|
||||
lines.push('| Metric | Value |')
|
||||
lines.push('|--------|-------|')
|
||||
lines.push(`| Total files | ${stats.projectInfo.fileCount.toLocaleString()} |`)
|
||||
lines.push(`| External files | ${stats.projectInfo.externalFiles.toLocaleString()} |`)
|
||||
lines.push(
|
||||
`| % External | ${((stats.projectInfo.externalFiles / stats.projectInfo.fileCount) * 100).toFixed(1)}% |`
|
||||
)
|
||||
lines.push(`| Config files | ${stats.projectInfo.configFiles.length} |`)
|
||||
lines.push('')
|
||||
|
||||
if (stats.projectInfo.configFiles.length > 0) {
|
||||
lines.push('**Config files:**')
|
||||
lines.push('')
|
||||
stats.projectInfo.configFiles.forEach((f) => {
|
||||
lines.push(`- \`${f.replace(process.env.HOME || '', '~')}\``)
|
||||
})
|
||||
lines.push('')
|
||||
}
|
||||
}
|
||||
|
||||
if (stats.requestCounts.size > 0) {
|
||||
lines.push('### Request counts')
|
||||
lines.push('')
|
||||
lines.push('| Command | Count |')
|
||||
lines.push('|---------|-------|')
|
||||
Array.from(stats.requestCounts.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.forEach(([cmd, count]) => {
|
||||
lines.push(`| ${cmd} | ${count} |`)
|
||||
})
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
if (stats.slowRequests.length > 0) {
|
||||
lines.push('### Slow requests (>100ms)')
|
||||
lines.push('')
|
||||
lines.push('| Command | Duration |')
|
||||
lines.push('|---------|----------|')
|
||||
stats.slowRequests
|
||||
.sort((a, b) => b.duration - a.duration)
|
||||
.forEach((req) => {
|
||||
lines.push(`| ${req.command} | ${req.duration.toFixed(0)}ms |`)
|
||||
})
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
lines.push('## How to diagnose performance issues')
|
||||
lines.push('')
|
||||
|
||||
lines.push('### 1. Enable verbose TSServer logging in your IDE')
|
||||
lines.push('')
|
||||
lines.push('**VSCode/Cursor:**')
|
||||
lines.push('')
|
||||
lines.push('1. Open settings (Cmd+,)')
|
||||
lines.push('2. Search for `typescript.tsserver.log`')
|
||||
lines.push('3. Set to `verbose`')
|
||||
lines.push('4. Search for `typescript.tsserver.trace`')
|
||||
lines.push('5. Set to `verbose`')
|
||||
lines.push('6. Reload window (Cmd+Shift+P → "Reload Window")')
|
||||
lines.push('7. Reproduce the performance issue')
|
||||
lines.push('8. Run command: "TypeScript: Open TS Server log"')
|
||||
lines.push('')
|
||||
|
||||
lines.push('### 2. Look for these patterns in logs')
|
||||
lines.push('')
|
||||
lines.push('- **Slow response times:** Look for requests taking >500ms')
|
||||
lines.push('- **Memory warnings:** Messages about heap size or GC')
|
||||
lines.push('- **File watching:** Excessive file change notifications')
|
||||
lines.push('- **Type checking:** Long `checkSourceFile` or `getSemanticDiagnostics` calls')
|
||||
lines.push('')
|
||||
|
||||
lines.push('### 3. Common causes and fixes')
|
||||
lines.push('')
|
||||
|
||||
lines.push('#### Too many files in project')
|
||||
lines.push('')
|
||||
lines.push('**Symptom:** High file count, especially from node_modules')
|
||||
lines.push('')
|
||||
lines.push('**Fix:** Add to `tsconfig.json`:')
|
||||
lines.push('```json')
|
||||
lines.push('{')
|
||||
lines.push(' "exclude": [')
|
||||
lines.push(' "node_modules",')
|
||||
lines.push(' "**/*.test.ts",')
|
||||
lines.push(' "**/*.spec.ts",')
|
||||
lines.push(' "e2e/**/*"')
|
||||
lines.push(' ]')
|
||||
lines.push('}')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
|
||||
lines.push('#### Slow module resolution')
|
||||
lines.push('')
|
||||
lines.push('**Symptom:** Slow `completions` or `definition` requests')
|
||||
lines.push('')
|
||||
lines.push('**Fix:** Simplify path mappings in `tsconfig.json`:')
|
||||
lines.push('```json')
|
||||
lines.push('{')
|
||||
lines.push(' "compilerOptions": {')
|
||||
lines.push(' "moduleResolution": "bundler",')
|
||||
lines.push(' "baseUrl": ".",')
|
||||
lines.push(' "paths": {')
|
||||
lines.push(' // Minimize wildcards and keep patterns simple')
|
||||
lines.push(' "@/*": ["./src/*"]')
|
||||
lines.push(' }')
|
||||
lines.push(' }')
|
||||
lines.push('}')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
|
||||
lines.push('#### Memory issues')
|
||||
lines.push('')
|
||||
lines.push('**Symptom:** Slowdowns after editing for a while, or high RSS memory')
|
||||
lines.push('')
|
||||
lines.push('**Fix:** Increase TSServer memory in VSCode settings:')
|
||||
lines.push('```json')
|
||||
lines.push('{')
|
||||
lines.push(' "typescript.tsserver.maxTsServerMemory": 8192')
|
||||
lines.push('}')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
|
||||
lines.push('#### Project references issues')
|
||||
lines.push('')
|
||||
lines.push('**Symptom:** Multiple config files, slow cross-package navigation')
|
||||
lines.push('')
|
||||
lines.push('**Fix:** Ensure project references are built:')
|
||||
lines.push('```bash')
|
||||
lines.push('yarn typecheck # or yarn build')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
|
||||
lines.push('### 4. Quick performance wins')
|
||||
lines.push('')
|
||||
lines.push('- **Restart TSServer:** Cmd+Shift+P → "TypeScript: Restart TS Server"')
|
||||
lines.push('- **Clear build info:** `find . -name "*.tsbuildinfo" -delete`')
|
||||
lines.push('- **Update TypeScript:** Check if newer version has fixes')
|
||||
lines.push(
|
||||
'- **Disable auto-imports:** If completions are slow, temporarily disable auto-import suggestions'
|
||||
)
|
||||
lines.push('')
|
||||
|
||||
lines.push('## Profiling tools')
|
||||
lines.push('')
|
||||
lines.push('### Using the compilation profiler')
|
||||
lines.push('')
|
||||
lines.push(
|
||||
'For **batch compilation** performance (not language server), use the other profiling script:'
|
||||
)
|
||||
lines.push('```bash')
|
||||
lines.push('yarn profile-typescript apps/dotcom/client')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
|
||||
lines.push('### Using Chrome DevTools')
|
||||
lines.push('')
|
||||
lines.push('To profile TSServer with Chrome DevTools:')
|
||||
lines.push('```bash')
|
||||
lines.push('node --inspect node_modules/typescript/lib/tsserver.js \\')
|
||||
lines.push(' --logVerbosity verbose \\')
|
||||
lines.push(' --logFile tsserver.log')
|
||||
lines.push('```')
|
||||
lines.push('')
|
||||
lines.push('Then open `chrome://inspect` and attach to the process.')
|
||||
lines.push('')
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('❌ Unexpected error:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
292
internal/scripts/profile-typescript.ts
Normal file
292
internal/scripts/profile-typescript.ts
Normal file
@@ -0,0 +1,292 @@
|
||||
import { execFileSync } from 'child_process'
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
interface ProfileResults {
|
||||
timestamp: string
|
||||
duration: number
|
||||
fileCount: number
|
||||
errorCount: number
|
||||
tsbuildInfoSize: number
|
||||
traceStats?: {
|
||||
totalTime: number
|
||||
longestFiles: Array<{ file: string; duration: number }>
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
nicelog('🔍 Starting TypeScript profiling...\n')
|
||||
|
||||
// Allow targeting a specific workspace, default to apps/dotcom/client
|
||||
const targetWorkspace = process.argv[2] || 'apps/dotcom/client'
|
||||
const workspacePath = join(REPO_ROOT, targetWorkspace)
|
||||
|
||||
if (!existsSync(join(workspacePath, 'tsconfig.json'))) {
|
||||
console.error(`❌ No tsconfig.json found in ${targetWorkspace}`)
|
||||
console.error(`Usage: yarn profile-typescript [workspace-path]`)
|
||||
console.error(`Example: yarn profile-typescript packages/editor`)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
nicelog(` Profiling workspace: ${targetWorkspace}\n`)
|
||||
|
||||
const outputDir = join(REPO_ROOT, '.ts-profile')
|
||||
const traceDir = join(outputDir, 'trace')
|
||||
const reportPath = join(outputDir, 'profile-report.md')
|
||||
|
||||
// Create output directory
|
||||
if (!existsSync(outputDir)) {
|
||||
mkdirSync(outputDir, { recursive: true })
|
||||
}
|
||||
|
||||
const results: ProfileResults = {
|
||||
timestamp: new Date().toISOString(),
|
||||
duration: 0,
|
||||
fileCount: 0,
|
||||
errorCount: 0,
|
||||
tsbuildInfoSize: 0,
|
||||
}
|
||||
|
||||
// Step 1: Count files being processed
|
||||
nicelog('📊 Step 1/5: Counting TypeScript files...')
|
||||
try {
|
||||
const listFilesOutput = execFileSync(
|
||||
join(REPO_ROOT, 'node_modules/.bin/tsc'),
|
||||
['--listFiles', '--noEmit'],
|
||||
{ cwd: workspacePath, encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 }
|
||||
)
|
||||
results.fileCount = listFilesOutput.split('\n').filter((line) => line.trim()).length
|
||||
nicelog(` Found ${results.fileCount} files\n`)
|
||||
} catch {
|
||||
nicelog(' ⚠️ Could not count files (non-fatal)\n')
|
||||
}
|
||||
|
||||
// Step 2: Calculate tsbuildinfo cache size
|
||||
nicelog('📦 Step 2/5: Calculating build cache size...')
|
||||
const findTsbuildInfo = (dir: string): string[] => {
|
||||
const results: string[] = []
|
||||
try {
|
||||
const entries = readdirSync(dir, { withFileTypes: true })
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name !== 'node_modules' && entry.name !== '.git') {
|
||||
results.push(...findTsbuildInfo(fullPath))
|
||||
}
|
||||
} else if (entry.name.endsWith('.tsbuildinfo')) {
|
||||
results.push(fullPath)
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore permission errors
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
const tsbuildInfoFiles = findTsbuildInfo(REPO_ROOT)
|
||||
results.tsbuildInfoSize = tsbuildInfoFiles.reduce((total, file) => {
|
||||
try {
|
||||
const stats = readFileSync(file)
|
||||
return total + stats.length
|
||||
} catch {
|
||||
return total
|
||||
}
|
||||
}, 0)
|
||||
nicelog(
|
||||
` Found ${tsbuildInfoFiles.length} cache files (${(results.tsbuildInfoSize / 1024 / 1024).toFixed(2)} MB)\n`
|
||||
)
|
||||
|
||||
// Step 3: Run extended diagnostics
|
||||
nicelog('⏱️ Step 3/5: Running extended diagnostics...')
|
||||
const startTime = Date.now()
|
||||
try {
|
||||
const diagnosticsOutput = execFileSync(
|
||||
join(REPO_ROOT, 'node_modules/.bin/tsc'),
|
||||
['--extendedDiagnostics', '--noEmit'],
|
||||
{ cwd: workspacePath, encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 }
|
||||
)
|
||||
results.duration = Date.now() - startTime
|
||||
|
||||
// Save full diagnostics output
|
||||
writeFileSync(join(outputDir, 'extended-diagnostics.txt'), diagnosticsOutput)
|
||||
nicelog(` Completed in ${(results.duration / 1000).toFixed(2)}s\n`)
|
||||
} catch (err: any) {
|
||||
results.duration = Date.now() - startTime
|
||||
if (err.stdout) {
|
||||
writeFileSync(join(outputDir, 'extended-diagnostics.txt'), err.stdout)
|
||||
}
|
||||
// Count errors from stderr
|
||||
if (err.stderr) {
|
||||
const errorLines = err.stderr.split('\n').filter((line: string) => line.includes('error TS'))
|
||||
results.errorCount = errorLines.length
|
||||
}
|
||||
nicelog(
|
||||
` Completed in ${(results.duration / 1000).toFixed(2)}s (with ${results.errorCount} errors)\n`
|
||||
)
|
||||
}
|
||||
|
||||
// Step 4: Generate trace file
|
||||
nicelog('🔬 Step 4/5: Generating trace file...')
|
||||
try {
|
||||
if (existsSync(traceDir)) {
|
||||
rmSync(traceDir, { recursive: true, force: true })
|
||||
}
|
||||
mkdirSync(traceDir, { recursive: true })
|
||||
|
||||
const tscPath = join(REPO_ROOT, 'node_modules/.bin/tsc')
|
||||
execFileSync(tscPath, ['--generateTrace', traceDir, '--noEmit'], {
|
||||
cwd: workspacePath,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 50 * 1024 * 1024,
|
||||
stdio: 'pipe',
|
||||
})
|
||||
|
||||
const traceFile = join(traceDir, 'trace.json')
|
||||
if (existsSync(traceFile)) {
|
||||
nicelog(` ✅ Trace generated at ${traceFile}\n`)
|
||||
nicelog(` Open chrome://tracing and load this file\n`)
|
||||
|
||||
// Try to parse trace for basic stats
|
||||
try {
|
||||
const trace = JSON.parse(readFileSync(traceFile, 'utf-8'))
|
||||
if (Array.isArray(trace)) {
|
||||
const checkEvents = trace.filter((e: any) => e.name === 'checkSourceFile')
|
||||
const sorted = checkEvents
|
||||
.sort((a: any, b: any) => (b.dur || 0) - (a.dur || 0))
|
||||
.slice(0, 10)
|
||||
|
||||
results.traceStats = {
|
||||
totalTime: checkEvents.reduce((sum: number, e: any) => sum + (e.dur || 0), 0) / 1000,
|
||||
longestFiles: sorted.map((e: any) => ({
|
||||
file: e.args?.path || 'unknown',
|
||||
duration: (e.dur || 0) / 1000,
|
||||
})),
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Trace parsing is optional
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
nicelog(` ⚠️ Could not generate trace: ${err.message}\n`)
|
||||
if (err.stderr) {
|
||||
nicelog(` Error output: ${err.stderr.slice(0, 200)}\n`)
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Generate markdown report
|
||||
nicelog('📝 Step 5/5: Generating report...')
|
||||
const report = generateReport(results, outputDir, traceDir)
|
||||
writeFileSync(reportPath, report)
|
||||
nicelog(` ✅ Report saved to ${reportPath}\n`)
|
||||
|
||||
// Print summary
|
||||
console.log('\n' + '='.repeat(80))
|
||||
console.log('📊 TypeScript Profile Summary')
|
||||
console.log('='.repeat(80))
|
||||
console.log(`Files processed: ${results.fileCount.toLocaleString()}`)
|
||||
console.log(`Type check duration: ${(results.duration / 1000).toFixed(2)}s`)
|
||||
console.log(`Type errors: ${results.errorCount}`)
|
||||
console.log(`Build cache size: ${(results.tsbuildInfoSize / 1024 / 1024).toFixed(2)} MB`)
|
||||
if (results.traceStats) {
|
||||
console.log(`Total check time: ${results.traceStats.totalTime.toFixed(2)}ms (from trace)`)
|
||||
}
|
||||
console.log('\n📄 Full report: ' + reportPath)
|
||||
console.log('🔬 Chrome trace: ' + join(traceDir, 'trace.json'))
|
||||
console.log('='.repeat(80) + '\n')
|
||||
}
|
||||
|
||||
function generateReport(results: ProfileResults, outputDir: string, traceDir: string): string {
|
||||
const lines: string[] = []
|
||||
|
||||
lines.push('# TypeScript profiling report')
|
||||
lines.push('')
|
||||
lines.push(`**Generated:** ${new Date(results.timestamp).toLocaleString()}`)
|
||||
lines.push('')
|
||||
|
||||
lines.push('## Summary')
|
||||
lines.push('')
|
||||
lines.push('| Metric | Value |')
|
||||
lines.push('|--------|-------|')
|
||||
lines.push(`| Files processed | ${results.fileCount.toLocaleString()} |`)
|
||||
lines.push(`| Type check duration | ${(results.duration / 1000).toFixed(2)}s |`)
|
||||
lines.push(`| Type errors | ${results.errorCount} |`)
|
||||
lines.push(`| Build cache size | ${(results.tsbuildInfoSize / 1024 / 1024).toFixed(2)} MB |`)
|
||||
if (results.traceStats) {
|
||||
lines.push(
|
||||
`| Total type checking time | ${results.traceStats.totalTime.toFixed(2)}ms (from trace) |`
|
||||
)
|
||||
}
|
||||
lines.push('')
|
||||
|
||||
if (results.traceStats && results.traceStats.longestFiles.length > 0) {
|
||||
lines.push('## Slowest files to type check')
|
||||
lines.push('')
|
||||
lines.push('| File | Duration |')
|
||||
lines.push('|------|----------|')
|
||||
for (const { file, duration } of results.traceStats.longestFiles) {
|
||||
const shortPath = file.replace(REPO_ROOT, '.')
|
||||
lines.push(`| \`${shortPath}\` | ${duration.toFixed(2)}ms |`)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
if (results.errorCount > 0) {
|
||||
lines.push('## Type errors')
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Found ${results.errorCount} type errors. See \`extended-diagnostics.txt\` for details.`
|
||||
)
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
lines.push('## How to analyze')
|
||||
lines.push('')
|
||||
lines.push('### Chrome tracing')
|
||||
lines.push('')
|
||||
lines.push('1. Open Chrome/Edge browser')
|
||||
lines.push('2. Navigate to `chrome://tracing`')
|
||||
lines.push(`3. Load file: \`${join(traceDir, 'trace.json')}\``)
|
||||
lines.push('4. Use WASD keys to navigate, search for specific files')
|
||||
lines.push('')
|
||||
lines.push('### Extended diagnostics')
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Full TypeScript diagnostics output saved to \`${join(outputDir, 'extended-diagnostics.txt')}\``
|
||||
)
|
||||
lines.push('')
|
||||
|
||||
lines.push('## Recommendations')
|
||||
lines.push('')
|
||||
lines.push('### If performance is slow:')
|
||||
lines.push('')
|
||||
lines.push('1. **Clear build caches:**')
|
||||
lines.push(' ```bash')
|
||||
lines.push(' find . -name "*.tsbuildinfo" -not -path "*/node_modules/*" -delete')
|
||||
lines.push(' ```')
|
||||
lines.push('')
|
||||
lines.push('2. **Restart TypeScript server in your IDE**')
|
||||
lines.push('')
|
||||
lines.push('3. **Check for files with long check times in the Chrome trace**')
|
||||
lines.push('')
|
||||
lines.push('4. **Exclude test files from main compilation:**')
|
||||
lines.push(' Add to `tsconfig.json`:')
|
||||
lines.push(' ```json')
|
||||
lines.push(' "exclude": ["**/__test__/**", "**/*.test.ts"]')
|
||||
lines.push(' ```')
|
||||
lines.push('')
|
||||
lines.push('5. **Add to compiler options for faster dependency checking:**')
|
||||
lines.push(' ```json')
|
||||
lines.push(' "assumeChangesOnlyAffectDirectDependencies": true')
|
||||
lines.push(' ```')
|
||||
lines.push('')
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('❌ Unexpected error:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
458
internal/scripts/prune-preview-deploys.ts
Normal file
458
internal/scripts/prune-preview-deploys.ts
Normal file
@@ -0,0 +1,458 @@
|
||||
import * as github from '@actions/github'
|
||||
import { DeleteObjectsCommand, ListObjectsV2Command, S3Client } from '@aws-sdk/client-s3'
|
||||
import { exec } from './lib/exec'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
// Do not use `process.env` directly in this script. Add your variable to `makeEnv` and use it via
|
||||
// `env` instead. This makes sure that all required env vars are present.
|
||||
const env = makeEnv([
|
||||
'CLOUDFLARE_ACCOUNT_ID',
|
||||
'CLOUDFLARE_API_TOKEN',
|
||||
'GH_TOKEN',
|
||||
'SUPABASE_ACCESS_TOKEN',
|
||||
'SUPABASE_PREVIEW_PROJECT_ID',
|
||||
'ZERO_R2_ENDPOINT',
|
||||
'ZERO_R2_BUCKET_NAME',
|
||||
'ZERO_R2_ACCESS_KEY_ID',
|
||||
'ZERO_R2_SECRET_ACCESS_KEY',
|
||||
'R2_ACCESS_KEY_ID',
|
||||
'R2_ACCESS_KEY_SECRET',
|
||||
])
|
||||
|
||||
interface ListWorkersResult {
|
||||
success: boolean
|
||||
result: { id: string }[]
|
||||
}
|
||||
|
||||
const _isPrClosedCache = new Map<number, boolean>()
|
||||
async function isPrClosed(prNumber: number) {
|
||||
if (_isPrClosedCache.has(prNumber)) {
|
||||
return _isPrClosedCache.get(prNumber)!
|
||||
}
|
||||
|
||||
let prResult
|
||||
try {
|
||||
prResult = await github.getOctokit(env.GH_TOKEN).rest.pulls.get({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
pull_number: prNumber,
|
||||
})
|
||||
} catch (err: any) {
|
||||
if (err.status === 404) {
|
||||
_isPrClosedCache.set(prNumber, true)
|
||||
return true
|
||||
}
|
||||
throw err
|
||||
}
|
||||
const result = prResult.data.state === 'closed'
|
||||
_isPrClosedCache.set(prNumber, result)
|
||||
return result
|
||||
}
|
||||
|
||||
const CLOUDFLARE_WORKER_REGEX = /^pr-(\d+)-/
|
||||
const CLOUDFLARE_SYNC_WORKER_REGEX = /^pr-\d+-tldraw-multiplayer$/
|
||||
|
||||
async function cloudflareV4Api(endpoint: string, options: RequestInit = {}): Promise<Response> {
|
||||
const url = `https://api.cloudflare.com/client/v4${endpoint}`
|
||||
return fetch(url, {
|
||||
...options,
|
||||
headers: {
|
||||
Authorization: `Bearer ${env.CLOUDFLARE_API_TOKEN}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async function cloudflareApi(endpoint: string, options: RequestInit = {}): Promise<Response> {
|
||||
return cloudflareV4Api(`/accounts/${env.CLOUDFLARE_ACCOUNT_ID}${endpoint}`, options)
|
||||
}
|
||||
|
||||
async function listPreviewWorkerDeployments() {
|
||||
const res = await cloudflareApi('/workers/scripts')
|
||||
const data = (await res.json()) as ListWorkersResult
|
||||
if (!data.success) {
|
||||
throw new Error('Failed to list workers ' + JSON.stringify(data))
|
||||
}
|
||||
return (
|
||||
data.result
|
||||
.map((r) => r.id)
|
||||
.filter((id) => id.match(CLOUDFLARE_WORKER_REGEX))
|
||||
// Delete workers with service bindings to other workers first (image-optimizer and tldrawusercontent both bind to the sync worker)
|
||||
.sort((a, b) => {
|
||||
const aHasBinding = a.includes('image-optimizer') || a.includes('tldrawusercontent')
|
||||
const bHasBinding = b.includes('image-optimizer') || b.includes('tldrawusercontent')
|
||||
if (aHasBinding && !bHasBinding) return -1
|
||||
if (!aHasBinding && bHasBinding) return 1
|
||||
return 0
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Preview routes and cert packs live on the preview zone rather than the account.
|
||||
const CLOUDFLARE_PREVIEW_ZONE = 'tldraw.xyz'
|
||||
let _previewZoneId: string | undefined
|
||||
async function getPreviewZoneId() {
|
||||
if (_previewZoneId) return _previewZoneId
|
||||
const res = await cloudflareV4Api(`/zones?name=${CLOUDFLARE_PREVIEW_ZONE}`)
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`Failed to look up zone ${CLOUDFLARE_PREVIEW_ZONE}: ${res.status} ${res.statusText}`
|
||||
)
|
||||
}
|
||||
const data = (await res.json()) as { success: boolean; result: { id: string }[] }
|
||||
if (!data.success || !data.result.length) {
|
||||
// an empty result also happens when the token lacks zone-scoped "Zone: Read"
|
||||
throw new Error(`Failed to find zone ${CLOUDFLARE_PREVIEW_ZONE}: ${JSON.stringify(data)}`)
|
||||
}
|
||||
_previewZoneId = data.result[0].id
|
||||
return _previewZoneId
|
||||
}
|
||||
|
||||
// Preview workers are reachable via zone routes ("pr-NNNN-<app>.tldraw.xyz/*").
|
||||
// Deleting a worker does not delete its routes, so prune them separately.
|
||||
// Only routes matching this exact preview shape may ever be deleted — anything
|
||||
// else on the zone (or anything a future refactor feeds in) must not qualify.
|
||||
const PREVIEW_ROUTE_PATTERN_REGEX = /^pr-\d+-[a-z0-9-]+\.tldraw\.xyz\/\*$/
|
||||
const _workerRouteIdCache = new Map<string, string>()
|
||||
async function listPreviewWorkerRoutes() {
|
||||
const zoneId = await getPreviewZoneId()
|
||||
const res = await cloudflareV4Api(`/zones/${zoneId}/workers/routes`)
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to list worker routes: ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const data = (await res.json()) as {
|
||||
success: boolean
|
||||
result: { id: string; pattern: string }[]
|
||||
}
|
||||
if (!data.success) {
|
||||
throw new Error('Failed to list worker routes ' + JSON.stringify(data))
|
||||
}
|
||||
const previewRoutes = data.result.filter((r) => PREVIEW_ROUTE_PATTERN_REGEX.test(r.pattern))
|
||||
for (const r of previewRoutes) {
|
||||
_workerRouteIdCache.set(r.pattern, r.id)
|
||||
}
|
||||
return previewRoutes.map((r) => r.pattern)
|
||||
}
|
||||
|
||||
async function deletePreviewWorkerRoute(pattern: string) {
|
||||
if (!PREVIEW_ROUTE_PATTERN_REGEX.test(pattern)) {
|
||||
throw new Error(`Refusing to delete non-preview route ${pattern}`)
|
||||
}
|
||||
const id = _workerRouteIdCache.get(pattern)
|
||||
if (!id) {
|
||||
nicelog(`Route ${pattern} did not exist, skipping`)
|
||||
return
|
||||
}
|
||||
nicelog('Deleting worker route:', pattern)
|
||||
const zoneId = await getPreviewZoneId()
|
||||
const res = await cloudflareV4Api(`/zones/${zoneId}/workers/routes/${id}`, { method: 'DELETE' })
|
||||
if (res.status === 404) {
|
||||
nicelog(`Route ${pattern} did not exist, skipping`)
|
||||
return
|
||||
}
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to delete worker route ${pattern}: ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const data = (await res.json()) as { success: boolean }
|
||||
if (!data.success) {
|
||||
throw new Error(`Failed to delete worker route ${pattern}: ${JSON.stringify(data)}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteQueue(queueName: string) {
|
||||
nicelog('Deleting queue:', queueName)
|
||||
await exec('npx', ['wrangler', 'queues', 'delete', queueName], {
|
||||
env: { CI: '1' },
|
||||
})
|
||||
}
|
||||
|
||||
async function deleteQueueConsumer(queueName: string, scriptName: string) {
|
||||
nicelog('Deleting queue consumer:', scriptName, 'from queue:', queueName)
|
||||
await exec('npx', ['wrangler', 'queues', 'consumer', 'worker', 'remove', queueName, scriptName], {
|
||||
env: { CI: '1' },
|
||||
})
|
||||
}
|
||||
|
||||
async function deletePreviewWorker(workerName: string) {
|
||||
nicelog('Deleting worker:', workerName)
|
||||
await exec('npx', ['wrangler', 'delete', '--name', workerName], {
|
||||
env: { CI: '1' },
|
||||
})
|
||||
}
|
||||
|
||||
async function deletePreviewWorkerDeployment(id: string) {
|
||||
// We want to delete the queue consumer and the queue only once. We'll do it just before we delete the worker
|
||||
if (id.match(CLOUDFLARE_SYNC_WORKER_REGEX)) {
|
||||
const prNumber = Number(id.match(CLOUDFLARE_WORKER_REGEX)?.[1])
|
||||
const queueName = `tldraw-multiplayer-queue-pr-${prNumber}`
|
||||
|
||||
try {
|
||||
await deleteQueueConsumer(queueName, id)
|
||||
} catch (err) {
|
||||
nicelog(`Failed to delete consumer ${id}: ${err}`)
|
||||
}
|
||||
await deletePreviewWorker(id)
|
||||
try {
|
||||
await deleteQueue(queueName)
|
||||
} catch (err) {
|
||||
nicelog(`Failed to delete queue ${queueName}: ${err}`)
|
||||
}
|
||||
} else {
|
||||
await deletePreviewWorker(id)
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a worker with a custom domain leaves its edge certificate behind
|
||||
// (https://github.com/cloudflare/workers-sdk/issues/5139), so prune per-PR
|
||||
// advanced cert packs on the preview zone separately.
|
||||
const CERT_PACK_HOST_REGEX = /^pr-\d+-/
|
||||
|
||||
// `status=all` also returns packs that are already gone (`deleted` /
|
||||
// `pending_deletion`); a host can therefore appear on several packs. Skip the
|
||||
// dead ones and keep every live pack id per host — caching just the last-seen
|
||||
// id would let a dead pack shadow a live one, which would then never be pruned.
|
||||
const _certPackCache = new Map<string, string[]>()
|
||||
const CERT_PACKS_PER_PAGE = 50
|
||||
const CERT_PACK_GONE_STATUSES = new Set(['deleted', 'pending_deletion'])
|
||||
async function listPreviewCertPacks() {
|
||||
const zoneId = await getPreviewZoneId()
|
||||
for (let page = 1; ; page++) {
|
||||
const res = await cloudflareV4Api(
|
||||
`/zones/${zoneId}/ssl/certificate_packs?status=all&per_page=${CERT_PACKS_PER_PAGE}&page=${page}`
|
||||
)
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to list certificate packs: ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const data = (await res.json()) as {
|
||||
success: boolean
|
||||
result: { id: string; type: string; status: string; hosts: string[] }[]
|
||||
}
|
||||
if (!data.success) {
|
||||
throw new Error('Failed to list certificate packs ' + JSON.stringify(data))
|
||||
}
|
||||
for (const pack of data.result) {
|
||||
if (pack.type !== 'advanced') continue
|
||||
if (CERT_PACK_GONE_STATUSES.has(pack.status)) continue
|
||||
const prHost = pack.hosts.find((h) => CERT_PACK_HOST_REGEX.test(h))
|
||||
if (!prHost) continue
|
||||
const ids = _certPackCache.get(prHost) ?? []
|
||||
ids.push(pack.id)
|
||||
_certPackCache.set(prHost, ids)
|
||||
}
|
||||
if (data.result.length < CERT_PACKS_PER_PAGE) break
|
||||
}
|
||||
return [..._certPackCache.keys()]
|
||||
}
|
||||
|
||||
async function deletePreviewCertPack(host: string) {
|
||||
const packIds = _certPackCache.get(host)
|
||||
if (!packIds?.length) {
|
||||
throw new Error(`Certificate pack for ${host} not found in cache`)
|
||||
}
|
||||
const zoneId = await getPreviewZoneId()
|
||||
for (const packId of packIds) {
|
||||
nicelog('Deleting certificate pack:', packId, 'for', host)
|
||||
const res = await cloudflareV4Api(`/zones/${zoneId}/ssl/certificate_packs/${packId}`, {
|
||||
method: 'DELETE',
|
||||
})
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`Failed to delete certificate pack ${packId}: ${res.status} ${res.statusText}`
|
||||
)
|
||||
}
|
||||
const data = (await res.json()) as { success: boolean }
|
||||
if (!data.success) {
|
||||
throw new Error(`Failed to delete certificate pack ${packId}: ${JSON.stringify(data)}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const supabaseHeaders = {
|
||||
Authorization: `Bearer ${env.SUPABASE_ACCESS_TOKEN}`,
|
||||
}
|
||||
|
||||
async function deletePreviewDatabase(branchName: string) {
|
||||
const branchId = _supabaseBranchCache.get(branchName)
|
||||
if (!branchId) {
|
||||
nicelog(`Branch ${branchName} not found in cache`)
|
||||
return
|
||||
}
|
||||
const url = `https://api.supabase.com/v1/branches/${branchId}`
|
||||
nicelog('DELETE', url)
|
||||
const res = await fetch(url, { method: 'DELETE', headers: supabaseHeaders })
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`Failed to delete Supabase branch ${branchName}: ${res.status} ${res.statusText}`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFlyioPreviewApp(appName: string) {
|
||||
const result = await exec('flyctl', ['apps', 'list', '-o', 'tldraw-gb-ltd'])
|
||||
if (result.indexOf(appName) >= 0) {
|
||||
await exec('flyctl', ['apps', 'destroy', appName, '-y'])
|
||||
}
|
||||
}
|
||||
|
||||
const PREVIEW_DB_REGEX = /^pr-\d+$/
|
||||
const _supabaseBranchCache = new Map<string, string>()
|
||||
async function listPreviewDatabases() {
|
||||
const url = `https://api.supabase.com/v1/projects/${env.SUPABASE_PREVIEW_PROJECT_ID}/branches`
|
||||
const res = await fetch(url, { headers: supabaseHeaders })
|
||||
if (!res.ok) {
|
||||
throw new Error(`Failed to list Supabase branches: ${res.status} ${res.statusText}`)
|
||||
}
|
||||
const branches = (await res.json()) as { id: string; name: string }[]
|
||||
const preview = branches.filter((b) => PREVIEW_DB_REGEX.test(b.name))
|
||||
for (const b of preview) {
|
||||
_supabaseBranchCache.set(b.name, b.id)
|
||||
}
|
||||
return preview.map((b) => b.name)
|
||||
}
|
||||
const ZERO_CACHE_APP_REGEX = /^pr-\d+-zero-(cache|rm|vs)$/
|
||||
async function listFlyioPreviewApps() {
|
||||
// This is the kind of output this returns.
|
||||
// We'll skip the first line then get the first column of each line.
|
||||
// NAME OWNER STATUS LATEST DEPLOY
|
||||
// pr-5795-zero-cache tldraw-gb-ltd deployed 39m37s ago
|
||||
const result = await exec('flyctl', ['apps', 'list', '-o', 'tldraw-gb-ltd'])
|
||||
const lines = result.trim().split('\n')
|
||||
if (lines.length <= 1) return []
|
||||
|
||||
const appNames = lines.slice(1).map((line) => {
|
||||
const [name] = line.trim().split(/\s+/)
|
||||
return name
|
||||
})
|
||||
|
||||
return appNames.filter((name) => ZERO_CACHE_APP_REGEX.test(name))
|
||||
}
|
||||
|
||||
interface R2BucketRef {
|
||||
client: S3Client
|
||||
bucket: string
|
||||
label: string
|
||||
}
|
||||
|
||||
async function listR2PrPrefixes({ client, bucket }: R2BucketRef): Promise<string[]> {
|
||||
const prefixes: string[] = []
|
||||
let continuationToken: string | undefined
|
||||
do {
|
||||
const res = await client.send(
|
||||
new ListObjectsV2Command({
|
||||
Bucket: bucket,
|
||||
Prefix: 'pr-',
|
||||
Delimiter: '/',
|
||||
ContinuationToken: continuationToken,
|
||||
})
|
||||
)
|
||||
for (const prefix of res.CommonPrefixes ?? []) {
|
||||
if (prefix.Prefix) prefixes.push(prefix.Prefix)
|
||||
}
|
||||
continuationToken = res.IsTruncated ? res.NextContinuationToken : undefined
|
||||
} while (continuationToken)
|
||||
return prefixes
|
||||
}
|
||||
|
||||
async function deleteR2Prefix({ client, bucket, label }: R2BucketRef, prefix: string) {
|
||||
nicelog(`Deleting ${label}:`, prefix)
|
||||
while (true) {
|
||||
const list = await client.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix }))
|
||||
const objects = list.Contents
|
||||
if (!objects || objects.length === 0) break
|
||||
const result = await client.send(
|
||||
new DeleteObjectsCommand({
|
||||
Bucket: bucket,
|
||||
Delete: { Objects: objects.map((o) => ({ Key: o.Key })) },
|
||||
})
|
||||
)
|
||||
if (result.Errors && result.Errors.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to delete ${result.Errors.length} objects: ${JSON.stringify(result.Errors)}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const zeroBackups: R2BucketRef = {
|
||||
client: new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: env.ZERO_R2_ENDPOINT,
|
||||
credentials: {
|
||||
accessKeyId: env.ZERO_R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.ZERO_R2_SECRET_ACCESS_KEY,
|
||||
},
|
||||
}),
|
||||
bucket: env.ZERO_R2_BUCKET_NAME,
|
||||
label: 'Zero litestream backup',
|
||||
}
|
||||
|
||||
// Matches the bucket / endpoint used by `coalesceWithPreviousAssets` in deploy-dotcom.ts.
|
||||
const dotcomAssetsCache: R2BucketRef = {
|
||||
client: new S3Client({
|
||||
region: 'auto',
|
||||
endpoint: 'https://c34edc4e76350954b63adebde86d5eb1.r2.cloudflarestorage.com',
|
||||
credentials: {
|
||||
accessKeyId: env.R2_ACCESS_KEY_ID,
|
||||
secretAccessKey: env.R2_ACCESS_KEY_SECRET,
|
||||
},
|
||||
}),
|
||||
bucket: 'dotcom-deploy-assets-cache',
|
||||
label: 'dotcom deploy assets cache',
|
||||
}
|
||||
|
||||
const deletionErrors: string[] = []
|
||||
|
||||
async function main() {
|
||||
nicelog('Pruning preview worker deployments')
|
||||
await processItems(listPreviewWorkerDeployments, deletePreviewWorkerDeployment)
|
||||
nicelog('\nPruning preview worker routes')
|
||||
await processItems(listPreviewWorkerRoutes, deletePreviewWorkerRoute)
|
||||
nicelog('\nPruning preview certificate packs')
|
||||
await processItems(listPreviewCertPacks, deletePreviewCertPack)
|
||||
nicelog('\nPruning Supabase preview databases')
|
||||
await processItems(listPreviewDatabases, deletePreviewDatabase)
|
||||
nicelog('\nPruning fly.io preview apps')
|
||||
await processItems(listFlyioPreviewApps, deleteFlyioPreviewApp)
|
||||
for (const r2 of [zeroBackups, dotcomAssetsCache]) {
|
||||
nicelog(`\nPruning ${r2.label}`)
|
||||
await processItems(
|
||||
() => listR2PrPrefixes(r2),
|
||||
(prefix) => deleteR2Prefix(r2, prefix)
|
||||
)
|
||||
}
|
||||
nicelog('\nDone')
|
||||
if (deletionErrors.length > 0) {
|
||||
nicelog('\nDeletion errors:')
|
||||
for (const error of deletionErrors) {
|
||||
nicelog(error)
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function processItems(
|
||||
fetchFn: () => Promise<string[]>,
|
||||
deleteFn: (id: string) => Promise<void>
|
||||
) {
|
||||
const items = await fetchFn()
|
||||
for (const item of items) {
|
||||
const number = Number(item.match(/pr-(\d+)/)?.[1])
|
||||
if (!number || isNaN(number)) {
|
||||
nicelog(`Skipping ${item} because it doesn't match the regex`)
|
||||
continue
|
||||
}
|
||||
if (await isPrClosed(number)) {
|
||||
nicelog(`Deleting ${item} because PR is closed`)
|
||||
try {
|
||||
await deleteFn(item)
|
||||
} catch (err) {
|
||||
deletionErrors.push(`${item}: ${err}`)
|
||||
}
|
||||
} else {
|
||||
nicelog(`Skipping ${item} because PR is still open`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
143
internal/scripts/publish-editor-extensions.ts
Normal file
143
internal/scripts/publish-editor-extensions.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { parse } from 'semver'
|
||||
import { exec } from './lib/exec'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
// VSCE_PAT needs to be set. It is used by the vsce publish command.
|
||||
// OVSX_PAT needs to be set for Open VSX publishing (handled by publish script).
|
||||
const env = makeEnv(['VSCE_PAT', 'OVSX_PAT', 'TLDRAW_ENV'])
|
||||
|
||||
const EXTENSION_DIR = 'apps/vscode/extension'
|
||||
const DISTRIBUTION_DIR = 'apps/vscode/extension/release'
|
||||
const MAX_VERSION_CONFLICT_ATTEMPTS = 5
|
||||
const VERSION_CONFLICT_RETRY_DELAY_MS = 30_000
|
||||
|
||||
function isVersionConflictError(err: unknown): boolean {
|
||||
const message = err instanceof Error ? err.message : ''
|
||||
const stderr = typeof (err as any)?.stderr === 'string' ? (err as any).stderr : ''
|
||||
return message.includes('already exists') || stderr.includes('already exists')
|
||||
}
|
||||
|
||||
async function fetchMarketplaceVersion(): Promise<string> {
|
||||
await exec('yarn', ['get-info'], { pwd: EXTENSION_DIR })
|
||||
const extensionInfoJsonPath = path.join(EXTENSION_DIR, 'extension.json')
|
||||
if (!existsSync(extensionInfoJsonPath)) {
|
||||
throw new Error('Published extension info not found.')
|
||||
}
|
||||
const extensionInfoJson = JSON.parse(readFileSync(extensionInfoJsonPath, 'utf8'))
|
||||
const version = extensionInfoJson.versions[0].version
|
||||
if (!version) {
|
||||
throw new Error('Could not get the version of the published extension.')
|
||||
}
|
||||
return version
|
||||
}
|
||||
|
||||
async function bumpVersion(fromVersion?: string): Promise<string> {
|
||||
const currentVersion = fromVersion ?? (await fetchMarketplaceVersion())
|
||||
const semVer = parse(currentVersion)
|
||||
if (!semVer) {
|
||||
throw new Error(`Could not parse the published version: ${currentVersion}`)
|
||||
}
|
||||
const release = env.TLDRAW_ENV === 'production' ? 'minor' : 'patch'
|
||||
const nextVersion = semVer.inc(release).version
|
||||
nicelog(`Bumping extension version from ${currentVersion} to ${nextVersion}`)
|
||||
|
||||
const packageJsonPath = path.join(EXTENSION_DIR, 'package.json')
|
||||
if (!existsSync(packageJsonPath)) {
|
||||
throw new Error("Could not find the extension's package.json file.")
|
||||
}
|
||||
const packageJson = JSON.parse(readFileSync(packageJsonPath, 'utf8'))
|
||||
packageJson.version = nextVersion
|
||||
writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, '\t') + '\n')
|
||||
return nextVersion
|
||||
}
|
||||
|
||||
async function copyExtensionToReleaseFolder(version: string) {
|
||||
if (!existsSync(DISTRIBUTION_DIR)) {
|
||||
mkdirSync(DISTRIBUTION_DIR, { recursive: true })
|
||||
}
|
||||
|
||||
const tempDir = path.join(EXTENSION_DIR, 'temp')
|
||||
if (!existsSync(tempDir)) {
|
||||
throw new Error(
|
||||
'Extension temp directory not found. Make sure packaging completed successfully.'
|
||||
)
|
||||
}
|
||||
|
||||
const files = readdirSync(tempDir)
|
||||
const vsixFile = files.find((file: string) => file.endsWith('.vsix'))
|
||||
if (!vsixFile) {
|
||||
throw new Error('No .vsix file found in temp directory.')
|
||||
}
|
||||
|
||||
const sourcePath = path.join(tempDir, vsixFile)
|
||||
const targetPath = path.join(DISTRIBUTION_DIR, 'tldraw-vscode.vsix')
|
||||
|
||||
nicelog(`Copying extension from ${sourcePath} to ${targetPath}`)
|
||||
copyFileSync(sourcePath, targetPath)
|
||||
|
||||
nicelog('Setting git user identity...')
|
||||
await exec('git', ['config', 'user.name', 'huppy-bot[bot]'])
|
||||
await exec('git', ['config', 'user.email', '128400622+huppy-bot[bot]@users.noreply.github.com'])
|
||||
|
||||
nicelog('Committing extension to git...')
|
||||
await exec('git', ['add', '-f', targetPath])
|
||||
await exec('git', ['commit', '-m', `Add VSCode extension v${version} [skip ci]`])
|
||||
nicelog('Pushing changes to remote repository...')
|
||||
await exec('git', ['push'])
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (env.TLDRAW_ENV !== 'production' && env.TLDRAW_ENV !== 'staging') {
|
||||
throw new Error('Workflow triggered from a branch other than main or production.')
|
||||
}
|
||||
|
||||
await exec('yarn', ['lazy', 'run', 'build', '--filter=packages/*'])
|
||||
|
||||
// When two pushes to main happen in quick succession, the concurrency group serializes
|
||||
// the runs but `vsce show` can lag a fresh publish by 10+ minutes, so both runs compute
|
||||
// the same next version. The "already exists" rejection is authoritative, so on conflict
|
||||
// bump past the rejected version locally instead of re-fetching the stale listing.
|
||||
let conflictedVersion: string | undefined
|
||||
for (let attempt = 1; attempt <= MAX_VERSION_CONFLICT_ATTEMPTS; attempt++) {
|
||||
const version = await bumpVersion(conflictedVersion)
|
||||
|
||||
try {
|
||||
switch (env.TLDRAW_ENV) {
|
||||
case 'production':
|
||||
await exec('yarn', ['package'], { pwd: EXTENSION_DIR })
|
||||
await exec('yarn', ['publish'], { pwd: EXTENSION_DIR })
|
||||
await copyExtensionToReleaseFolder(version)
|
||||
return
|
||||
case 'staging':
|
||||
await exec('yarn', ['package', '--pre-release'], { pwd: EXTENSION_DIR })
|
||||
await exec('yarn', ['publish', '--pre-release'], { pwd: EXTENSION_DIR })
|
||||
return
|
||||
}
|
||||
} catch (err) {
|
||||
// Exit code 75 from the publish script means an earlier upload may have succeeded.
|
||||
if ((err as { code?: number })?.code === 75) {
|
||||
throw new Error(
|
||||
`Publishing version ${version} stopped because an earlier Marketplace upload may have succeeded. Open VSX has not been attempted. Check Marketplace and complete this release at the same version; do not rerun the workflow blindly, as it computes a new version.`,
|
||||
{ cause: err }
|
||||
)
|
||||
}
|
||||
if (isVersionConflictError(err) && attempt < MAX_VERSION_CONFLICT_ATTEMPTS) {
|
||||
conflictedVersion = version
|
||||
nicelog(
|
||||
`Version ${version} already exists (attempt ${attempt}/${MAX_VERSION_CONFLICT_ATTEMPTS}), bumping past it and retrying in ${VERSION_CONFLICT_RETRY_DELAY_MS / 1000}s...`
|
||||
)
|
||||
await new Promise((resolve) => setTimeout(resolve, VERSION_CONFLICT_RETRY_DELAY_MS))
|
||||
continue
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(async (err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
35
internal/scripts/publish-manual.ts
Normal file
35
internal/scripts/publish-manual.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { appendFileSync } from 'fs'
|
||||
import { parse } from 'semver'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import {
|
||||
getLatestTldrawVersionFromNpm,
|
||||
publish,
|
||||
publishProductionDocsAndExamplesAndBemo,
|
||||
setAllVersions,
|
||||
} from './lib/publishing'
|
||||
import { uploadStaticAssets } from './lib/upload-static-assets'
|
||||
|
||||
const env = makeEnv(['TLDRAW_VERSION_STRING'])
|
||||
|
||||
async function main() {
|
||||
const latestVersion = await getLatestTldrawVersionFromNpm()
|
||||
const version = parse(env.TLDRAW_VERSION_STRING)
|
||||
if (!version) throw new Error('Invalid version')
|
||||
|
||||
const isLatestVersion = version.compare(latestVersion) >= 0 && !version.prerelease.length
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `is_latest_version=${isLatestVersion}\n`)
|
||||
}
|
||||
|
||||
await setAllVersions(env.TLDRAW_VERSION_STRING)
|
||||
|
||||
await uploadStaticAssets(env.TLDRAW_VERSION_STRING)
|
||||
|
||||
await publish()
|
||||
|
||||
if (isLatestVersion) {
|
||||
await publishProductionDocsAndExamplesAndBemo()
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
147
internal/scripts/publish-new.ts
Normal file
147
internal/scripts/publish-new.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import assert from 'assert'
|
||||
import { Octokit } from '@octokit/rest'
|
||||
import minimist from 'minimist'
|
||||
import { SemVer, parse } from 'semver'
|
||||
import { exec } from './lib/exec'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import {
|
||||
getLatestTldrawVersionFromNpm,
|
||||
publish,
|
||||
publishProductionDocsAndExamplesAndBemo,
|
||||
setAllVersions,
|
||||
triggerBumpVersionsWorkflow,
|
||||
} from './lib/publishing'
|
||||
import { uploadStaticAssets } from './lib/upload-static-assets'
|
||||
|
||||
const env = makeEnv(['GH_TOKEN'])
|
||||
|
||||
type ReleaseType =
|
||||
| {
|
||||
bump: 'major' | 'minor'
|
||||
}
|
||||
| {
|
||||
bump: 'override'
|
||||
version: SemVer
|
||||
}
|
||||
|
||||
function getReleaseType(): ReleaseType {
|
||||
const arg = minimist(process.argv.slice(2))['bump']
|
||||
if (!arg) {
|
||||
throw new Error('Must provide a --bump argument')
|
||||
}
|
||||
if (arg === 'major' || arg === 'minor') {
|
||||
return { bump: arg }
|
||||
}
|
||||
const parsed = parse(arg)
|
||||
if (parsed) {
|
||||
return { bump: 'override', version: parsed }
|
||||
}
|
||||
throw new Error('Invalid bump argument ' + JSON.stringify(arg))
|
||||
}
|
||||
|
||||
async function getDraftRelease(version: string, octokit: Octokit) {
|
||||
const expectedVersion = `v${version}`
|
||||
|
||||
// Fetch all releases with pagination
|
||||
const releases = await octokit.paginate(octokit.rest.repos.listReleases, {
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
per_page: 100, // Maximum per page to reduce API calls
|
||||
})
|
||||
|
||||
const draftRelease = releases.find((release) => release.draft && release.name === expectedVersion)
|
||||
|
||||
if (!draftRelease) {
|
||||
const availableDrafts = releases
|
||||
.filter((r) => r.draft)
|
||||
.map((r) => r.name)
|
||||
.join(', ')
|
||||
throw new Error(
|
||||
`No draft release found named ${expectedVersion}. Available draft releases: ${availableDrafts || 'none'}`
|
||||
)
|
||||
}
|
||||
|
||||
assert(draftRelease.body, `Draft release ${expectedVersion} has no body/release notes`)
|
||||
return draftRelease
|
||||
}
|
||||
|
||||
async function getNextVersion(releaseType: ReleaseType): Promise<string> {
|
||||
if (releaseType.bump === 'override') {
|
||||
return releaseType.version.format()
|
||||
}
|
||||
|
||||
const latestVersion = await getLatestTldrawVersionFromNpm()
|
||||
|
||||
nicelog('latestVersion', latestVersion)
|
||||
|
||||
const [prereleaseTag, prereleaseNumber] = latestVersion.prerelease
|
||||
if (prereleaseTag && typeof prereleaseNumber !== 'number') {
|
||||
throw new Error(`Invalid prerelease format in version ${latestVersion}, expected e.g. -alpha.1`)
|
||||
}
|
||||
|
||||
// if we're in prerelease mode, don't bump the version, only the prerelease number
|
||||
const nextVersion = prereleaseTag
|
||||
? `${latestVersion.major}.${latestVersion.minor}.${latestVersion.patch}-${prereleaseTag}.${
|
||||
Number(prereleaseNumber) + 1
|
||||
}`
|
||||
: latestVersion.inc(releaseType.bump).format()
|
||||
|
||||
return nextVersion
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// // check we're on the main branch on HEAD
|
||||
const currentBranch = (await exec('git', ['rev-parse', '--abbrev-ref', 'HEAD'])).toString().trim()
|
||||
if (currentBranch !== 'production') {
|
||||
throw new Error('Must be on production branch to publish')
|
||||
}
|
||||
|
||||
const releaseType = getReleaseType()
|
||||
const nextVersion = await getNextVersion(releaseType)
|
||||
|
||||
const isPrerelease = parse(nextVersion)!.prerelease.length > 0
|
||||
assert(!isPrerelease, 'Prerelease versions are no longer supported for new releases')
|
||||
|
||||
console.log('Releasing version', nextVersion)
|
||||
|
||||
// Check that a draft release exists early, before doing any work
|
||||
const octokit = new Octokit({ auth: env.GH_TOKEN })
|
||||
const draftRelease = await getDraftRelease(nextVersion, octokit)
|
||||
nicelog('Found draft release:', draftRelease)
|
||||
|
||||
await setAllVersions(nextVersion, { stageChanges: true })
|
||||
|
||||
const gitTag = `v${nextVersion}`
|
||||
await exec('git', ['commit', '-m', `${gitTag}`])
|
||||
|
||||
// create new 'release' branch called e.g. v2.0.x or v4.3.x, for making patch releases
|
||||
const { major, minor } = parse(nextVersion)!
|
||||
const branchName = `v${major}.${minor}.x`
|
||||
// create and push a new tag to the release branch
|
||||
await exec('git', ['tag', '-a', gitTag, '-m', gitTag, '-f'])
|
||||
await exec('git', ['push', 'origin', `HEAD:refs/heads/${branchName}`, '--follow-tags'])
|
||||
await publishProductionDocsAndExamplesAndBemo()
|
||||
|
||||
// convert draft release to published release
|
||||
await octokit.rest.repos.updateRelease({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
release_id: draftRelease.id,
|
||||
draft: false,
|
||||
tag_name: gitTag,
|
||||
name: gitTag,
|
||||
})
|
||||
|
||||
// upload static assets
|
||||
await uploadStaticAssets(nextVersion)
|
||||
|
||||
// finally, publish the packages [IF THIS STEP FAILS, RUN THE `publish-manual.ts` script locally]
|
||||
await publish()
|
||||
|
||||
// Trigger version bump on main branch to sync all package versions
|
||||
nicelog('Triggering version bump on main branch')
|
||||
await triggerBumpVersionsWorkflow(env.GH_TOKEN)
|
||||
}
|
||||
|
||||
main()
|
||||
138
internal/scripts/publish-patch.ts
Normal file
138
internal/scripts/publish-patch.ts
Normal file
@@ -0,0 +1,138 @@
|
||||
import { appendFileSync, readFileSync } from 'node:fs'
|
||||
import { Octokit } from '@octokit/rest'
|
||||
import { parse } from 'semver'
|
||||
import { extractChangelog } from './extract-draft-changelog'
|
||||
import { getAnyPackageDiff } from './lib/didAnyPackageChange'
|
||||
import { exec } from './lib/exec'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import {
|
||||
getLatestTldrawVersionFromNpm,
|
||||
publish,
|
||||
publishProductionDocsAndExamplesAndBemo,
|
||||
setAllVersions,
|
||||
triggerBumpVersionsWorkflow,
|
||||
} from './lib/publishing'
|
||||
import { uploadStaticAssets } from './lib/upload-static-assets'
|
||||
|
||||
async function main() {
|
||||
const currentBranch = (await exec('git', ['rev-parse', '--abbrev-ref', 'HEAD'])).toString().trim()
|
||||
const match = currentBranch.match(/^v(\d+)\.(\d+)\.x$/)
|
||||
if (!match) {
|
||||
throw new Error('Branch name does not match expected format: v{major}.{minor}.x')
|
||||
}
|
||||
const [major, minor] = match.slice(1).map(Number)
|
||||
const latestVersionInBranch = await getLatestTldrawVersionFromNpm({
|
||||
versionPrefix: `${major}.${minor}`,
|
||||
})
|
||||
|
||||
// Check if this commit is already tagged with the initial release for this major.minor version
|
||||
const expectedInitialTag = `v${latestVersionInBranch.major}.${latestVersionInBranch.minor}.0`
|
||||
const tagsAtThisCommit = (await exec('git', ['tag', '--points-at', 'HEAD']))
|
||||
.toString()
|
||||
.trim()
|
||||
.split('\n')
|
||||
|
||||
if (tagsAtThisCommit.includes(expectedInitialTag)) {
|
||||
// Skip release if this is the initial release commit for this major.minor version
|
||||
nicelog('Initial release commit, skipping patch release')
|
||||
return
|
||||
} else {
|
||||
nicelog('Not an initial release commit, continuing with patch release')
|
||||
}
|
||||
|
||||
const latestVersion = await getLatestTldrawVersionFromNpm()
|
||||
const isLatestVersion = latestVersion.compare(latestVersionInBranch) === 0
|
||||
|
||||
if (isLatestVersion) {
|
||||
await publishProductionDocsAndExamplesAndBemo()
|
||||
}
|
||||
|
||||
// Ensure asset directories exist before comparing package contents.
|
||||
// CI may skip postinstall (and thus refresh-assets) when install-state.gz is cached.
|
||||
await exec('yarn', ['refresh-assets', '--force'])
|
||||
|
||||
// Skip releasing a new version if the package contents are identical.
|
||||
// This may happen when cherry-picking docs-only changes.
|
||||
if (!(await getAnyPackageDiff(latestVersionInBranch.format()))) {
|
||||
nicelog('No packages have changed, skipping release')
|
||||
return
|
||||
}
|
||||
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `is_latest_version=${isLatestVersion}\n`)
|
||||
}
|
||||
|
||||
// Capture the previous tag BEFORE calling .inc(): semver's SemVer.prototype.inc()
|
||||
// mutates the instance in place, so reading latestVersionInBranch.format() afterwards
|
||||
// would return the new version and leave prevTag === tag, producing an empty changelog.
|
||||
const prevTag = `v${latestVersionInBranch.format()}`
|
||||
|
||||
// Determine the version to bump from. Normally this is the latest version
|
||||
// published on npm for this branch. But if a previous patch release was
|
||||
// interrupted after the "v X.Y.Z [skip ci]" version-bump commit was pushed
|
||||
// but before npm publish completed, the local package.json files (and the
|
||||
// matching git tag) will be ahead of npm. In that case we bump from the
|
||||
// local version so we sidestep the orphaned version+tag entirely instead
|
||||
// of trying to re-create them - both `git commit` (no staged changes) and
|
||||
// `git push --follow-tags` (tag already exists at a different commit on
|
||||
// origin) would otherwise fail and permanently wedge the branch.
|
||||
const localTldrawVersion = parse(
|
||||
JSON.parse(readFileSync('packages/tldraw/package.json', 'utf8')).version
|
||||
)
|
||||
const baseVersion =
|
||||
localTldrawVersion && localTldrawVersion.compare(latestVersionInBranch) > 0
|
||||
? localTldrawVersion
|
||||
: latestVersionInBranch
|
||||
if (baseVersion !== latestVersionInBranch) {
|
||||
nicelog(
|
||||
`Local package version ${baseVersion.format()} is ahead of latest npm version ` +
|
||||
`${latestVersionInBranch.format()}; bumping from local to skip past orphaned version.`
|
||||
)
|
||||
}
|
||||
|
||||
// .inc() mutates baseVersion; reparse so callers downstream of us see the
|
||||
// original instance unchanged.
|
||||
const nextVersion = parse(baseVersion.format())!.inc('patch').format()
|
||||
nicelog('Releasing version', nextVersion)
|
||||
|
||||
await setAllVersions(nextVersion, { stageChanges: true })
|
||||
|
||||
const tag = `v${nextVersion}`
|
||||
|
||||
// create and push a new tag
|
||||
await exec('git', ['commit', '-m', `${tag} [skip ci]`])
|
||||
await exec('git', ['tag', '-a', tag, '-m', tag, '-f'])
|
||||
await exec('git', ['push', '--follow-tags'])
|
||||
|
||||
// Generate changelog and create GitHub release
|
||||
nicelog('Generating changelog...')
|
||||
const changelog = await extractChangelog(prevTag, 'HEAD')
|
||||
|
||||
nicelog('Creating GitHub release...')
|
||||
const octokit = new Octokit({ auth: process.env.GH_TOKEN })
|
||||
await octokit.repos.createRelease({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
tag_name: tag,
|
||||
name: tag,
|
||||
body: changelog,
|
||||
draft: false,
|
||||
prerelease: false,
|
||||
})
|
||||
|
||||
await uploadStaticAssets(nextVersion)
|
||||
|
||||
// if we're on the latest version, publish to npm under 'latest' tag.
|
||||
// otherwise we don't want to overwrite the latest tag, so we publish under 'revision'.
|
||||
// semver rules will still be respected because there's no prerelease tag in the version,
|
||||
// so clients will get the updated version if they have a range like ^1.0.0
|
||||
await publish(isLatestVersion ? 'latest' : 'revision')
|
||||
|
||||
// If this is the latest version, trigger version bump on main branch to sync all package versions
|
||||
if (isLatestVersion) {
|
||||
nicelog('This is the latest version, triggering version bump on main branch')
|
||||
await triggerBumpVersionsWorkflow(process.env.GH_TOKEN!)
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
||||
30
internal/scripts/publish-prerelease.ts
Normal file
30
internal/scripts/publish-prerelease.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
import { exec } from './lib/exec'
|
||||
import { getLatestTldrawVersionFromNpm, publish, setAllVersions } from './lib/publishing'
|
||||
import { uploadStaticAssets } from './lib/upload-static-assets'
|
||||
|
||||
async function main(releaseTag: string) {
|
||||
const sha = (await exec('git', ['rev-parse', 'HEAD'])).trim().slice(0, 12)
|
||||
|
||||
const latestVersion = await getLatestTldrawVersionFromNpm()
|
||||
|
||||
const nextVersion = latestVersion.prerelease.length
|
||||
? // if the package is in prerelease mode, we want to release a canary for the current version rather than bumping
|
||||
latestVersion
|
||||
: latestVersion?.inc('minor')
|
||||
|
||||
const versionString = `${nextVersion.major}.${nextVersion.minor}.${nextVersion.patch}-${releaseTag}.${sha}`
|
||||
|
||||
await setAllVersions(versionString)
|
||||
await uploadStaticAssets(versionString)
|
||||
await publish(releaseTag)
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2)
|
||||
if (args.length === 0) {
|
||||
throw new Error('Usage: publish-prerelease.ts <releaseTag>')
|
||||
}
|
||||
const releaseTag = args[0]
|
||||
if (releaseTag !== 'canary' && releaseTag !== 'internal' && releaseTag !== 'next') {
|
||||
throw new Error('releaseTag must be "canary", "internal", or "next"')
|
||||
}
|
||||
main(releaseTag)
|
||||
14
internal/scripts/purge-css.ts
Normal file
14
internal/scripts/purge-css.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { PurgeCSS } from 'purgecss'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
async function main() {
|
||||
const purgeCSSResults = await new PurgeCSS().purge({
|
||||
content: ['packages/tldraw/**/*.tsx', 'packages/editor/**/*.tsx'],
|
||||
css: ['packages/tldraw/src/lib/ui.css'],
|
||||
rejected: true,
|
||||
})
|
||||
|
||||
nicelog(purgeCSSResults)
|
||||
}
|
||||
|
||||
main()
|
||||
772
internal/scripts/refresh-assets.ts
Normal file
772
internal/scripts/refresh-assets.ts
Normal file
@@ -0,0 +1,772 @@
|
||||
import { execFileSync } from 'child_process'
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { SemVer } from 'semver'
|
||||
import { optimize } from 'svgo'
|
||||
import { publishDates, version } from '../../packages/editor/src/version'
|
||||
import {
|
||||
readJsonIfExists,
|
||||
REPO_ROOT,
|
||||
writeCodeFile,
|
||||
writeFile,
|
||||
writeJsonFile,
|
||||
writeStringFile,
|
||||
} from './lib/file'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
|
||||
// We'll need to copy the assets into these folders
|
||||
const PUBLIC_FOLDER_PATHS = [join(REPO_ROOT, 'packages', 'assets')]
|
||||
|
||||
const FONT_MAPPING: Record<string, string> = {
|
||||
'IBMPlexMono-Medium': 'tldraw_mono',
|
||||
'IBMPlexMono-MediumItalic': 'tldraw_mono_italic',
|
||||
'IBMPlexMono-Bold': 'tldraw_mono_bold',
|
||||
'IBMPlexMono-BoldItalic': 'tldraw_mono_italic_bold',
|
||||
'IBMPlexSerif-Medium': 'tldraw_serif',
|
||||
'IBMPlexSerif-MediumItalic': 'tldraw_serif_italic',
|
||||
'IBMPlexSerif-Bold': 'tldraw_serif_bold',
|
||||
'IBMPlexSerif-BoldItalic': 'tldraw_serif_italic_bold',
|
||||
'IBMPlexSans-Medium': 'tldraw_sans',
|
||||
'IBMPlexSans-MediumItalic': 'tldraw_sans_italic',
|
||||
'IBMPlexSans-Bold': 'tldraw_sans_bold',
|
||||
'IBMPlexSans-BoldItalic': 'tldraw_sans_italic_bold',
|
||||
'Shantell_Sans-Informal_Regular': 'tldraw_draw',
|
||||
'Shantell_Sans-Informal_Regular_Italic': 'tldraw_draw_italic',
|
||||
'Shantell_Sans-Informal_Bold': 'tldraw_draw_bold',
|
||||
'Shantell_Sans-Informal_Bold_Italic': 'tldraw_draw_italic_bold',
|
||||
}
|
||||
|
||||
const ASSETS_FOLDER_PATH = join(REPO_ROOT, 'assets')
|
||||
const DOTCOM_FOLDER_PATH = join(REPO_ROOT, 'apps', 'dotcom')
|
||||
|
||||
const collectedAssetUrls: Record<
|
||||
'fonts' | 'icons' | 'translations' | 'embedIcons',
|
||||
Record<string, { file: string; hash?: string }>
|
||||
> = {
|
||||
fonts: {},
|
||||
icons: {},
|
||||
translations: {},
|
||||
embedIcons: {},
|
||||
}
|
||||
|
||||
const mergedIconFooter = '</svg>'
|
||||
|
||||
function shouldSkipRefreshAssets(): boolean {
|
||||
const onlyIfNecessary =
|
||||
process.env.ONLY_REFRESH_ASSETS_IF_NECESSARY === '1' ||
|
||||
process.argv.includes('--only-refresh-assets-if-necessary')
|
||||
if (!onlyIfNecessary) return false
|
||||
|
||||
try {
|
||||
const changedFiles = execFileSync(
|
||||
'git',
|
||||
[
|
||||
'diff',
|
||||
'--cached',
|
||||
'--name-only',
|
||||
'--',
|
||||
'package.json',
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'assets',
|
||||
'apps/dotcom/client/assets',
|
||||
':(glob)packages/*/package.json',
|
||||
],
|
||||
{ cwd: REPO_ROOT, encoding: 'utf8' }
|
||||
).trim()
|
||||
return changedFiles.length === 0
|
||||
} catch {
|
||||
// If git inspection fails, fall back to running the full refresh.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function optimizeAndMergeSvgs(
|
||||
icons: string[],
|
||||
sourceFolderPath: string,
|
||||
mergedIconName: string,
|
||||
mergedIconHeader: string
|
||||
) {
|
||||
// Create the optimized SVGs
|
||||
const optimizedSvgs = icons.map((icon) => {
|
||||
const iconPath = join(sourceFolderPath, icon)
|
||||
const content = readFileSync(iconPath, 'utf8')
|
||||
const svg = optimize(content, { path: iconPath })
|
||||
return { fileName: icon, data: svg.data }
|
||||
})
|
||||
|
||||
// Merge svgs into a single file
|
||||
const mergedSvgParts = [
|
||||
mergedIconHeader,
|
||||
'<style>',
|
||||
// in order to target individual icons with the URL hash, we need to hide all of them by
|
||||
// default...
|
||||
'svg > [id] { display: none; }',
|
||||
// ...then show the one that's targeted
|
||||
'svg > [id]:target { display: inline; }',
|
||||
'</style>',
|
||||
]
|
||||
for (const { fileName, data } of optimizedSvgs) {
|
||||
if (!data.startsWith(mergedIconHeader) || !data.endsWith(mergedIconFooter)) {
|
||||
nicelog('SVG does not match expected format for merging:', fileName)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const id = fileName.replace('.svg', '')
|
||||
const svgContent = data.slice(mergedIconHeader.length, -mergedIconFooter.length)
|
||||
mergedSvgParts.push(`<g id="${id}">${svgContent}</g>`)
|
||||
}
|
||||
mergedSvgParts.push(mergedIconFooter)
|
||||
const mergedSvg = optimize(mergedSvgParts.join('\n')).data
|
||||
optimizedSvgs.push({ fileName: mergedIconName, data: mergedSvg })
|
||||
return optimizedSvgs
|
||||
}
|
||||
|
||||
// 1. ICONS
|
||||
async function copyIcons() {
|
||||
const sourceFolderPath = join(ASSETS_FOLDER_PATH, 'icons', 'icon')
|
||||
const mergedIconName = '0_merged.svg'
|
||||
|
||||
// this is how `svgo` starts each one of our optimized icon SVGs. if they don't all start with this,
|
||||
// we can't merge them as it means they're of different size and are using different fill rules.
|
||||
const mergedIconHeader =
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="30" height="30" fill="none">'
|
||||
|
||||
// Get a list of all icons
|
||||
const icons = readdirSync(sourceFolderPath).filter((icon) => icon.endsWith('.svg'))
|
||||
// Write list of names into icon-names.json (just the name, not extension)
|
||||
const iconNames = icons.map((name) => name.replace('.svg', ''))
|
||||
|
||||
const optimizedSvgs = optimizeAndMergeSvgs(
|
||||
icons,
|
||||
sourceFolderPath,
|
||||
mergedIconName,
|
||||
mergedIconHeader
|
||||
)
|
||||
|
||||
// Optimize all of the svg icons and write them into the new folders
|
||||
for (const folderPath of PUBLIC_FOLDER_PATHS) {
|
||||
const publicIconsRootFolderPath = join(folderPath, 'icons')
|
||||
const publicIconsFolderPath = join(publicIconsRootFolderPath, 'icon')
|
||||
|
||||
if (existsSync(publicIconsRootFolderPath)) {
|
||||
rmSync(publicIconsRootFolderPath, { recursive: true })
|
||||
}
|
||||
|
||||
// Create the folders
|
||||
mkdirSync(publicIconsRootFolderPath, { recursive: true })
|
||||
mkdirSync(publicIconsFolderPath, { recursive: true })
|
||||
|
||||
// Copy each optimized icons into the new folder
|
||||
for (const { fileName, data } of optimizedSvgs) {
|
||||
await writeStringFile(join(publicIconsFolderPath, fileName), data)
|
||||
}
|
||||
|
||||
// Write the JSON file containing all of the names of the icons
|
||||
await writeJsonFile(join(publicIconsFolderPath, 'icon-names.json'), iconNames)
|
||||
}
|
||||
|
||||
// Get the names of all of the svg icons and create a TypeScript file of valid icon names
|
||||
const iconTypeFile = `
|
||||
/** @public */
|
||||
export type TLUiIconType =
|
||||
${icons.map((icon) => JSON.stringify(icon.replace('.svg', ''))).join(' | ')}
|
||||
|
||||
/** @public */
|
||||
export const iconTypes = [
|
||||
${icons.map((icon) => JSON.stringify(icon.replace('.svg', ''))).join(', ')}
|
||||
] as const`
|
||||
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'packages', 'tldraw', 'src', 'lib', 'ui', 'icon-types.ts'),
|
||||
iconTypeFile
|
||||
)
|
||||
|
||||
// add to the asset declaration file
|
||||
for (const icon of icons) {
|
||||
const name = icon.replace('.svg', '')
|
||||
collectedAssetUrls.icons[name] = {
|
||||
file: `icons/icon/${mergedIconName}`,
|
||||
hash: name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function optimizeAndMergeDotcomIcons() {
|
||||
const sourceFolderPath = join(DOTCOM_FOLDER_PATH, 'client', 'assets', 'icons', 'icon')
|
||||
const mergedIconName = '0_merged_tla.svg'
|
||||
|
||||
// this is how `svgo` starts each one of our optimized icon SVGs. if they don't all start with this,
|
||||
// we can't merge them as it means they're of different size and are using different fill rules.
|
||||
const mergedIconHeader =
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="15" height="15" fill="none">'
|
||||
|
||||
// Get a list of all icons
|
||||
const icons = readdirSync(sourceFolderPath).filter((icon) => /icon-.*\.svg$/.test(icon))
|
||||
|
||||
const optimizedSvgs = optimizeAndMergeSvgs(
|
||||
icons,
|
||||
sourceFolderPath,
|
||||
mergedIconName,
|
||||
mergedIconHeader
|
||||
)
|
||||
|
||||
const mergedIconPath = join(DOTCOM_FOLDER_PATH, 'client', 'src', 'assets')
|
||||
|
||||
for (const { fileName, data } of optimizedSvgs) {
|
||||
if (fileName.includes(mergedIconName)) {
|
||||
await writeStringFile(join(mergedIconPath, fileName), data)
|
||||
} else {
|
||||
await writeStringFile(join(sourceFolderPath, fileName), data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. EMBED-ICONS
|
||||
async function copyEmbedIcons() {
|
||||
const folderName = 'embed-icons'
|
||||
const extension = '.png'
|
||||
|
||||
const sourceFolderPath = join(ASSETS_FOLDER_PATH, folderName)
|
||||
const itemsToCopy = readdirSync(sourceFolderPath).filter((icon) => icon.endsWith(extension))
|
||||
|
||||
for (const publicFolderPath of PUBLIC_FOLDER_PATHS) {
|
||||
const destinationFolderPath = join(publicFolderPath, folderName)
|
||||
|
||||
// Delete the folder if it exists
|
||||
if (existsSync(destinationFolderPath)) {
|
||||
rmSync(destinationFolderPath, { recursive: true })
|
||||
}
|
||||
|
||||
// Make the new folder
|
||||
mkdirSync(destinationFolderPath, { recursive: true })
|
||||
|
||||
// Copy all items into the new folder
|
||||
for (const item of itemsToCopy) {
|
||||
await writeFile(join(destinationFolderPath, item), readFileSync(join(sourceFolderPath, item)))
|
||||
}
|
||||
}
|
||||
|
||||
// add to the asset declaration file
|
||||
for (const item of itemsToCopy) {
|
||||
const name = item.replace(extension, '')
|
||||
collectedAssetUrls.embedIcons[name] = { file: `${folderName}/${item}` }
|
||||
}
|
||||
}
|
||||
|
||||
// 3. FONTS
|
||||
async function copyFonts() {
|
||||
const folderName = 'fonts'
|
||||
const extension = '.woff2'
|
||||
|
||||
const sourceFolderPath = join(ASSETS_FOLDER_PATH, folderName)
|
||||
const itemsToCopy = readdirSync(sourceFolderPath).filter((icon) => icon.endsWith(extension))
|
||||
|
||||
for (const publicFolderPath of PUBLIC_FOLDER_PATHS) {
|
||||
const destinationFolderPath = join(publicFolderPath, folderName)
|
||||
|
||||
// Delete the folder if it exists
|
||||
if (existsSync(destinationFolderPath)) {
|
||||
rmSync(destinationFolderPath, { recursive: true })
|
||||
}
|
||||
|
||||
// Make the new folder
|
||||
mkdirSync(destinationFolderPath)
|
||||
|
||||
// Copy all items into the new folder
|
||||
for (const item of itemsToCopy) {
|
||||
await writeFile(join(destinationFolderPath, item), readFileSync(join(sourceFolderPath, item)))
|
||||
}
|
||||
}
|
||||
|
||||
// add to the asset declaration file
|
||||
for (const item of itemsToCopy) {
|
||||
const itemWithoutExtension = item.replace(extension, '')
|
||||
const name = FONT_MAPPING[itemWithoutExtension]
|
||||
if (!name) {
|
||||
nicelog('Font mapping not found for', itemWithoutExtension)
|
||||
process.exit(1)
|
||||
}
|
||||
collectedAssetUrls.fonts[name] = { file: `${folderName}/${item}` }
|
||||
}
|
||||
}
|
||||
|
||||
// 4. TRANSLATIONS
|
||||
async function copyTranslations() {
|
||||
const folderName = 'translations'
|
||||
const extension = '.json'
|
||||
|
||||
const sourceFolderPath = join(ASSETS_FOLDER_PATH, folderName)
|
||||
const itemsToCopy = readdirSync(sourceFolderPath).filter((item) => item.endsWith(extension))
|
||||
|
||||
for (const publicFolderPath of PUBLIC_FOLDER_PATHS) {
|
||||
const destinationFolderPath = join(publicFolderPath, folderName)
|
||||
|
||||
// Delete the folder if it exists
|
||||
if (existsSync(destinationFolderPath)) {
|
||||
rmSync(destinationFolderPath, { recursive: true })
|
||||
}
|
||||
|
||||
// Make the new folder
|
||||
mkdirSync(destinationFolderPath)
|
||||
|
||||
// Copy all items into the new folder
|
||||
for (const item of itemsToCopy) {
|
||||
await writeFile(join(destinationFolderPath, item), readFileSync(join(sourceFolderPath, item)))
|
||||
}
|
||||
}
|
||||
|
||||
// Create hardcoded files
|
||||
const uiPath = join(
|
||||
REPO_ROOT,
|
||||
'packages',
|
||||
'tldraw',
|
||||
'src',
|
||||
'lib',
|
||||
'ui',
|
||||
'hooks',
|
||||
'useTranslation'
|
||||
)
|
||||
|
||||
// languages.ts
|
||||
|
||||
const languagesSource = await readJsonIfExists(join(sourceFolderPath, 'languages.json'))!
|
||||
interface Language {
|
||||
label: string
|
||||
locale: string
|
||||
}
|
||||
const languagesFile = `
|
||||
/** @public */
|
||||
export const LANGUAGES = ${JSON.stringify(
|
||||
languagesSource.sort((a: Language, b: Language) => a.label.localeCompare(b.label, 'en'))
|
||||
)} as const
|
||||
`
|
||||
const schemaPath = join(REPO_ROOT, 'packages', 'tlschema', 'src', 'translations')
|
||||
const schemaLanguagesFilePath = join(schemaPath, 'languages.ts')
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
schemaLanguagesFilePath,
|
||||
languagesFile
|
||||
)
|
||||
|
||||
// main.ts
|
||||
|
||||
const defaultTranslation = await readJsonIfExists(join(sourceFolderPath, 'main.json'))!
|
||||
const defaultTranslationFilePath = join(uiPath, 'defaultTranslation.ts')
|
||||
const defaultTranslationFile = `
|
||||
/** @internal */
|
||||
export const DEFAULT_TRANSLATION = ${JSON.stringify(defaultTranslation)}
|
||||
`
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
defaultTranslationFilePath,
|
||||
defaultTranslationFile
|
||||
)
|
||||
|
||||
// translationKeys.ts
|
||||
|
||||
const translationKeys = Object.keys(defaultTranslation).map((key) => `'${key}'`)
|
||||
const translationKeysFilePath = join(uiPath, 'TLUiTranslationKey.ts')
|
||||
const translationKeysFile = `
|
||||
/** @public */
|
||||
export type TLUiTranslationKey = ${translationKeys.join(' | ')}
|
||||
`
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
translationKeysFilePath,
|
||||
translationKeysFile
|
||||
)
|
||||
|
||||
// add to the asset declaration file
|
||||
for (const item of itemsToCopy) {
|
||||
const name = item.replace(extension, '')
|
||||
collectedAssetUrls.translations[name] = { file: `${folderName}/${item}` }
|
||||
}
|
||||
}
|
||||
|
||||
// 5. WATERMARKS
|
||||
async function copyWatermarks() {
|
||||
const folderName = 'watermarks'
|
||||
const extension = '.svg'
|
||||
|
||||
const sourceFolderPath = join(ASSETS_FOLDER_PATH, folderName)
|
||||
const itemsToCopy = readdirSync(sourceFolderPath).filter((watermark) =>
|
||||
watermark.endsWith(extension)
|
||||
)
|
||||
|
||||
const optimizedItems = itemsToCopy.map((watermark) => {
|
||||
const watermarkPath = join(sourceFolderPath, watermark)
|
||||
const content = readFileSync(watermarkPath, 'utf8')
|
||||
const svg = optimize(content, { path: watermarkPath })
|
||||
return { fileName: watermark, data: svg.data }
|
||||
})
|
||||
|
||||
const assetsDestinationPath = join(REPO_ROOT, 'packages', 'assets', folderName)
|
||||
if (existsSync(assetsDestinationPath)) {
|
||||
rmSync(assetsDestinationPath, { recursive: true })
|
||||
}
|
||||
mkdirSync(assetsDestinationPath)
|
||||
|
||||
const file = new CodeFile()
|
||||
for (const { fileName, data } of optimizedItems) {
|
||||
const varName = file.formatName(fileName)
|
||||
file.append(`export const ${varName} = ${JSON.stringify(data)};`)
|
||||
await writeStringFile(join(assetsDestinationPath, fileName), data)
|
||||
}
|
||||
|
||||
const codeDestinationPath = join(REPO_ROOT, 'packages', 'editor', 'src', 'lib', 'watermarks.ts')
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
codeDestinationPath,
|
||||
file.toString()
|
||||
)
|
||||
}
|
||||
|
||||
// 6. ASSET DECLARATION FILES
|
||||
async function writeUrlBasedAssetDeclarationFile() {
|
||||
const codeFilePath = join(REPO_ROOT, 'packages', 'assets', 'urls.js')
|
||||
const codeFile = new CodeFile(`
|
||||
// eslint-disable-next-line @typescript-eslint/triple-slash-reference
|
||||
/// <reference path="./modules.d.ts" />
|
||||
import { formatAssetUrl } from './utils.js'
|
||||
`)
|
||||
|
||||
const fn = codeFile.appendFn(`
|
||||
/**
|
||||
* @param {AssetUrlOptions} [opts]
|
||||
* @public
|
||||
*/
|
||||
export function getAssetUrlsByMetaUrl(opts) {
|
||||
`)
|
||||
|
||||
fn.append('return {')
|
||||
for (const [type, assets] of Object.entries(collectedAssetUrls)) {
|
||||
fn.append(`${type}: {`)
|
||||
for (const [name, { file, hash }] of Object.entries(assets)) {
|
||||
const href = JSON.stringify(`./${file}`)
|
||||
const fileUrl = fn.memo(file, `formatAssetUrl(new URL(${href}, import.meta.url).href, opts)`)
|
||||
const value = hash ? `${fileUrl} + ${JSON.stringify('#' + hash)}` : fileUrl
|
||||
fn.append(`${JSON.stringify(name)}: ${value},`)
|
||||
}
|
||||
fn.append('},')
|
||||
}
|
||||
fn.append('}')
|
||||
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'javascript',
|
||||
codeFilePath,
|
||||
codeFile.toString()
|
||||
)
|
||||
}
|
||||
|
||||
async function writeImportBasedAssetDeclarationFile(
|
||||
importSuffix: string,
|
||||
fileName: string
|
||||
): Promise<void> {
|
||||
const codeFile = new CodeFile(`
|
||||
// eslint-disable-next-line @typescript-eslint/triple-slash-reference
|
||||
/// <reference path="./modules.d.ts" />
|
||||
import { formatAssetUrl } from './utils.js'
|
||||
`)
|
||||
|
||||
const fn = codeFile.appendFn(`
|
||||
/**
|
||||
* @param {AssetUrlOptions} [opts]
|
||||
* @public
|
||||
*/
|
||||
export function getAssetUrlsByImport(opts) {
|
||||
`)
|
||||
|
||||
fn.append('return {')
|
||||
for (const [type, assets] of Object.entries(collectedAssetUrls)) {
|
||||
fn.append(`${type}: {`)
|
||||
for (const [name, { file, hash }] of Object.entries(assets)) {
|
||||
const variableName = codeFile.import(`./${file}${importSuffix}`)
|
||||
const formattedUrl = fn.memo(file, `formatAssetUrl(${variableName}, opts)`)
|
||||
const value = hash ? `${formattedUrl} + ${JSON.stringify('#' + hash)}` : formattedUrl
|
||||
fn.append(`${JSON.stringify(name)}: ${value},`)
|
||||
}
|
||||
fn.append('},')
|
||||
}
|
||||
fn.append('}')
|
||||
|
||||
const codeFilePath = join(REPO_ROOT, 'packages', 'assets', fileName)
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'javascript',
|
||||
codeFilePath,
|
||||
codeFile.toString()
|
||||
)
|
||||
}
|
||||
|
||||
async function writeSelfHostedAssetDeclarationFile(): Promise<void> {
|
||||
const codeFilePath = join(REPO_ROOT, 'packages', 'assets', 'selfHosted.js')
|
||||
const codeFile = new CodeFile(`
|
||||
// eslint-disable-next-line @typescript-eslint/triple-slash-reference
|
||||
/// <reference path="./modules.d.ts" />
|
||||
import { formatAssetUrl } from './utils.js'
|
||||
`)
|
||||
|
||||
const fn = codeFile.appendFn(`
|
||||
/**
|
||||
* @param {AssetUrlOptions} [opts]
|
||||
* @public
|
||||
*/
|
||||
export function getAssetUrls(opts) {
|
||||
`)
|
||||
|
||||
fn.append('return {')
|
||||
for (const [type, assets] of Object.entries(collectedAssetUrls)) {
|
||||
fn.append(`${type}: {`)
|
||||
for (const [name, { file, hash }] of Object.entries(assets)) {
|
||||
const href = JSON.stringify(`./${file}`)
|
||||
const formattedUrl = fn.memo(file, `formatAssetUrl(${href}, opts)`)
|
||||
const value = hash ? `${formattedUrl} + ${JSON.stringify('#' + hash)}` : formattedUrl
|
||||
fn.append(`${JSON.stringify(name)}: ${value},`)
|
||||
}
|
||||
fn.append('},')
|
||||
}
|
||||
fn.append('}')
|
||||
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'javascript',
|
||||
codeFilePath,
|
||||
codeFile.toString()
|
||||
)
|
||||
}
|
||||
|
||||
async function writeAssetDeclarationDTSFile() {
|
||||
let dts = `
|
||||
export type AssetUrl = string | { src: string }
|
||||
export type AssetUrlOptions = { baseUrl?: string } | ((assetUrl: string) => string)
|
||||
export type AssetUrls = {
|
||||
`
|
||||
|
||||
for (const [type, assets] of Object.entries(collectedAssetUrls)) {
|
||||
dts += `${type}: {\n`
|
||||
for (const name of Object.keys(assets)) {
|
||||
dts += `${JSON.stringify(name)}: string,\n`
|
||||
}
|
||||
dts += '},\n'
|
||||
}
|
||||
|
||||
dts += `
|
||||
}
|
||||
`
|
||||
|
||||
const assetDeclarationFilePath = join(REPO_ROOT, 'packages', 'assets', 'types.d.ts')
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
assetDeclarationFilePath,
|
||||
dts
|
||||
)
|
||||
}
|
||||
|
||||
function getNewPublishDates(packageVersion: string) {
|
||||
const currentVersion = new SemVer(version)
|
||||
const currentPackageVersion = new SemVer(packageVersion)
|
||||
const now = new Date().toISOString()
|
||||
if (currentPackageVersion.major > currentVersion.major) {
|
||||
return {
|
||||
major: now,
|
||||
minor: now,
|
||||
patch: now,
|
||||
}
|
||||
} else if (currentPackageVersion.minor > currentVersion.minor) {
|
||||
return {
|
||||
major: publishDates.major,
|
||||
minor: now,
|
||||
patch: now,
|
||||
}
|
||||
} else if (currentPackageVersion.patch > currentVersion.patch) {
|
||||
return {
|
||||
major: publishDates.major,
|
||||
minor: publishDates.minor,
|
||||
patch: now,
|
||||
}
|
||||
}
|
||||
return publishDates
|
||||
}
|
||||
|
||||
async function createVersionFiles() {
|
||||
const packageJson = await readJsonIfExists(join(REPO_ROOT, 'packages', 'tldraw', 'package.json'))
|
||||
const packageVersion = packageJson.version
|
||||
const publishDates = getNewPublishDates(packageVersion)
|
||||
const file = `export const version = '${packageVersion}'
|
||||
export const publishDates = {
|
||||
major: '${publishDates.major}',
|
||||
minor: '${publishDates.minor}',
|
||||
patch: '${publishDates.patch}',
|
||||
}`
|
||||
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'apps', 'docs', 'version.ts'),
|
||||
file
|
||||
)
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'apps', 'dotcom', 'client', 'version.ts'),
|
||||
file
|
||||
)
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'packages', 'editor', 'src', 'version.ts'),
|
||||
file
|
||||
)
|
||||
await writeCodeFile(
|
||||
'internal/scripts/refresh-assets.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'packages', 'tldraw', 'src', 'lib', 'ui', 'version.ts'),
|
||||
file
|
||||
)
|
||||
}
|
||||
|
||||
class Code {
|
||||
protected parts: (string | { toString(): string })[] = []
|
||||
|
||||
toString() {
|
||||
return this.parts.map((part) => (typeof part === 'string' ? part : part.toString())).join('')
|
||||
}
|
||||
|
||||
append(...parts: string[]): this {
|
||||
for (const part of parts) {
|
||||
this.parts.push(part + '\n')
|
||||
}
|
||||
return this
|
||||
}
|
||||
}
|
||||
class CodeFile extends Code {
|
||||
private imports = new Map<string, string>()
|
||||
|
||||
constructor(private header: string = '') {
|
||||
super()
|
||||
}
|
||||
|
||||
override toString(): string {
|
||||
return [
|
||||
this.header,
|
||||
Array.from(this.imports, ([file, name]) => {
|
||||
return `import ${name} from ${JSON.stringify(file)};`
|
||||
}).join('\n'),
|
||||
super.toString(),
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
import(file: string) {
|
||||
let name = this.imports.get(file)
|
||||
if (!name) {
|
||||
name = this.getName(file)
|
||||
this.imports.set(file, name)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
formatName(name: string) {
|
||||
return `$_${name.replace(/\W+/g, '_')}`
|
||||
.replace(/^\$_+(\D)/, (_, s) => s.toLowerCase())
|
||||
.replace(/_+(.)/g, (_, s) => s.toUpperCase())
|
||||
}
|
||||
|
||||
getName(name: string, suffix?: number): string {
|
||||
const formatted = this.formatName(`${name}${suffix ?? ''}`)
|
||||
|
||||
if (this.toString().match(formatted)) {
|
||||
return this.getName(name, (suffix ?? 1) + 1)
|
||||
}
|
||||
|
||||
return formatted
|
||||
}
|
||||
|
||||
appendFn(header: string): CodeFunction {
|
||||
const fn = new CodeFunction(this, header)
|
||||
this.parts.push(fn)
|
||||
return fn
|
||||
}
|
||||
}
|
||||
|
||||
class CodeFunction extends Code {
|
||||
constructor(
|
||||
private file: CodeFile,
|
||||
private header: string
|
||||
) {
|
||||
super()
|
||||
}
|
||||
|
||||
private consts = new Map<string, string>()
|
||||
|
||||
override toString(): string {
|
||||
let body = super.toString()
|
||||
|
||||
const lines = [this.header]
|
||||
for (const [name, expr] of this.consts) {
|
||||
const firstFoundIndex = body.indexOf(name)
|
||||
const secondFoundIndex = body.indexOf(name, firstFoundIndex + 1)
|
||||
if (secondFoundIndex === -1) {
|
||||
// only exists once, we don't need to declare it:
|
||||
body = body.replace(name, expr)
|
||||
} else {
|
||||
lines.push(`const ${name} = ${expr};`)
|
||||
}
|
||||
}
|
||||
|
||||
lines.push(body)
|
||||
lines.push('}')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
private memos = new Map<unknown, string>()
|
||||
memo(key: string, expr: string) {
|
||||
const existing = this.memos.get(key)
|
||||
if (existing) return existing
|
||||
|
||||
const varName = this.file.getName(key)
|
||||
this.consts.set(varName, expr)
|
||||
this.memos.set(key, varName)
|
||||
return varName
|
||||
}
|
||||
}
|
||||
|
||||
// --- RUN
|
||||
async function main() {
|
||||
if (shouldSkipRefreshAssets()) {
|
||||
nicelog(
|
||||
"Skipping refresh-assets (no staged changes to refresh-assets inputs, with '--only-refresh-assets-if-necessary')"
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
nicelog('Copying icons...')
|
||||
await copyIcons()
|
||||
nicelog('Copying dotcom icons...')
|
||||
await optimizeAndMergeDotcomIcons()
|
||||
nicelog('Copying embed icons...')
|
||||
await copyEmbedIcons()
|
||||
nicelog('Copying fonts...')
|
||||
await copyFonts()
|
||||
nicelog('Copying translations...')
|
||||
await copyTranslations()
|
||||
nicelog('Copying watermarks...')
|
||||
await copyWatermarks()
|
||||
nicelog('Writing asset declaration file...')
|
||||
await writeAssetDeclarationDTSFile()
|
||||
await writeUrlBasedAssetDeclarationFile()
|
||||
await writeImportBasedAssetDeclarationFile('', 'imports.js')
|
||||
await writeImportBasedAssetDeclarationFile('?url', 'imports.vite.js')
|
||||
await writeSelfHostedAssetDeclarationFile()
|
||||
await createVersionFiles()
|
||||
nicelog('Done!')
|
||||
}
|
||||
|
||||
main()
|
||||
44
internal/scripts/refresh-create-templates.ts
Normal file
44
internal/scripts/refresh-create-templates.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import { join } from 'path'
|
||||
import { compact } from '@tldraw/utils'
|
||||
import { REPO_ROOT, writeCodeFile } from './lib/file'
|
||||
import { getAllWorkspacePackages } from './lib/workspace'
|
||||
|
||||
async function main() {
|
||||
const packages = await getAllWorkspacePackages()
|
||||
const templates = compact(
|
||||
packages.map((pkg) => {
|
||||
if (!pkg.packageJson.tldraw_template?.cli) return null
|
||||
return {
|
||||
...pkg.packageJson.tldraw_template.cli,
|
||||
repo: pkg.packageJson.tldraw_template.repo,
|
||||
}
|
||||
})
|
||||
)
|
||||
const sortedTemplates = templates.sort((a, b) => {
|
||||
// sort by order, undefined values go last; then by name
|
||||
if (a.order === b.order) return a.name.localeCompare(b.name)
|
||||
if (a.order === undefined && b.order !== undefined) return 1
|
||||
if (a.order !== undefined && b.order === undefined) return -1
|
||||
return a.order! - b.order! // sort by order
|
||||
})
|
||||
|
||||
const code = `
|
||||
export interface Template {
|
||||
repo: string
|
||||
name: string
|
||||
description: string
|
||||
shortDescription?: string
|
||||
order?: number
|
||||
}
|
||||
|
||||
export const TEMPLATES: Template[] = ${JSON.stringify(sortedTemplates, null, 2)}
|
||||
`
|
||||
await writeCodeFile(
|
||||
'refresh-create-templates.ts',
|
||||
'typescript',
|
||||
join(REPO_ROOT, 'packages', 'create-tldraw', 'src', 'templates.ts'),
|
||||
code
|
||||
)
|
||||
}
|
||||
|
||||
main()
|
||||
224
internal/scripts/trigger-dotcom-hotfix.ts
Normal file
224
internal/scripts/trigger-dotcom-hotfix.ts
Normal file
@@ -0,0 +1,224 @@
|
||||
import { Octokit } from 'octokit'
|
||||
import { Discord } from './lib/discord'
|
||||
import { exec } from './lib/exec'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { getPrDetailsAndCommitSha, getPrDetailsByNumber, labelPresent } from './lib/pr-info'
|
||||
import { stripSkipCiMarkers } from './lib/skip-ci'
|
||||
|
||||
function getEnv() {
|
||||
return makeEnv(['DISCORD_DEPLOY_WEBHOOK_URL', 'GITHUB_TOKEN'])
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const env = getEnv()
|
||||
const octokit = new Octokit({ auth: env.GITHUB_TOKEN })
|
||||
|
||||
const result = await getPrDetailsAndCommitSha(octokit)
|
||||
if (!result) {
|
||||
nicelog('Could not retrieve PR details. Exiting...')
|
||||
return
|
||||
}
|
||||
|
||||
const { pr, commitSha } = result
|
||||
|
||||
if (!labelPresent(pr, 'dotcom-hotfix-please')) {
|
||||
nicelog('No dotcom-hotfix-please label found. Exiting...')
|
||||
return
|
||||
}
|
||||
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
totalSteps: 4,
|
||||
shouldNotify: true,
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
await discord.message(`🚀 Triggering dotcom hotfix for PR #${pr.number}...`)
|
||||
|
||||
const hotfixBranchName = `hotfix/dotcom-${pr.number}`
|
||||
|
||||
await discord.step(`Creating hotfix branch and cherry-picking changes`, async () => {
|
||||
await exec('git', ['fetch', 'origin', 'hotfixes'])
|
||||
await exec('git', ['fetch', 'origin', 'main'])
|
||||
await exec('git', ['checkout', 'hotfixes'])
|
||||
await exec('git', ['reset', '--hard', 'origin/hotfixes'])
|
||||
await exec('git', ['checkout', '-b', hotfixBranchName])
|
||||
await exec('git', ['cherry-pick', commitSha])
|
||||
|
||||
// release-notes PRs squash-merge with `[skip ci]` in the title. github honours that marker on
|
||||
// the cherry-picked commit too, so the hotfix branch's required checks would never start and
|
||||
// the PR below would sit in `blocked` until the timeout. strip it before pushing.
|
||||
const message = (await exec('git', ['log', '-1', '--format=%B'])).trim()
|
||||
const cleanedMessage = stripSkipCiMarkers(message)
|
||||
if (cleanedMessage !== message) {
|
||||
await exec('git', ['commit', '--amend', '-m', cleanedMessage])
|
||||
}
|
||||
})
|
||||
|
||||
await discord.step('Pushing hotfix branch to remote', async () => {
|
||||
await exec('git', ['push', 'origin', hotfixBranchName])
|
||||
})
|
||||
|
||||
await discord.step('Creating hotfix PR and waiting for checks to pass', async () => {
|
||||
// the squash-merge onto `hotfixes` is what triggers the production build, and github reads
|
||||
// skip-ci markers from the whole commit message. keep them out of the title (used as the
|
||||
// commit subject) and the body (used as the commit body when someone merges by hand).
|
||||
const originalTitle = stripSkipCiMarkers(pr.title)
|
||||
const prTitle = `[HOTFIX] ${originalTitle}`
|
||||
|
||||
// Extract API changes section from original PR if present
|
||||
const apiChangesHeader = '### API changes'
|
||||
let apiChangesSection = ''
|
||||
if (pr.body?.includes(apiChangesHeader)) {
|
||||
const bodyAfterHeader = pr.body.split(apiChangesHeader)[1]
|
||||
// Extract until next ### header or end of body
|
||||
const nextHeaderIndex = bodyAfterHeader.indexOf('\n###')
|
||||
apiChangesSection =
|
||||
nextHeaderIndex > -1 ? bodyAfterHeader.slice(0, nextHeaderIndex) : bodyAfterHeader
|
||||
apiChangesSection = `\n\n${apiChangesHeader}\n${apiChangesSection.trim()}\n`
|
||||
}
|
||||
|
||||
const prBody = `This is an automated hotfix PR for dotcom deployment.
|
||||
|
||||
**Original PR:** [#${pr.number}](https://github.com/tldraw/tldraw/pull/${pr.number})
|
||||
**Original Title:** ${originalTitle}
|
||||
**Original Author:** @${pr.user?.login}
|
||||
|
||||
This PR cherry-picks the changes from the original PR to the hotfixes branch for immediate dotcom deployment.${apiChangesSection}
|
||||
|
||||
/cc @${pr.user?.login}`
|
||||
|
||||
const createdPr = await octokit.rest.pulls.create({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
title: prTitle,
|
||||
body: prBody,
|
||||
head: hotfixBranchName,
|
||||
base: 'hotfixes',
|
||||
})
|
||||
|
||||
nicelog(`Created hotfix PR: ${hotfixBranchName} -> hotfixes`)
|
||||
await discord.message(
|
||||
`📝 Created hotfix PR: <https://github.com/tldraw/tldraw/pull/${createdPr.data.number}>`
|
||||
)
|
||||
nicelog(`Waiting for PR #${createdPr.data.number} to be ready for merge...`)
|
||||
|
||||
async function noteManualMerge(mergedByLogin: string | undefined) {
|
||||
const by = mergedByLogin ? `@${mergedByLogin}` : 'someone'
|
||||
nicelog(`Hotfix PR #${createdPr.data.number} was merged manually by ${by}`)
|
||||
await discord.message(`✅ Hotfix PR #${createdPr.data.number} was merged manually by ${by}`)
|
||||
}
|
||||
|
||||
// Maximum wait time: 15 minutes total (action timeout is 20 mins, we need buffer for Discord notification)
|
||||
const maxWaitTimeMs = 15 * 60 * 1000
|
||||
const startTime = Date.now()
|
||||
|
||||
// Wait for 5 minutes initially, then check every 15 seconds (our checks take at least 5 mins)
|
||||
await new Promise((resolve) => setTimeout(resolve, 5 * 60 * 1000))
|
||||
|
||||
let checkedForMissingChecks = false
|
||||
|
||||
while (true) {
|
||||
// Check if we've exceeded the timeout
|
||||
const elapsedTime = Date.now() - startTime
|
||||
if (elapsedTime >= maxWaitTimeMs) {
|
||||
nicelog(`Timeout: PR #${createdPr.data.number} checks did not complete in time`)
|
||||
throw new Error(
|
||||
`Hotfix PR #${createdPr.data.number} checks timed out after ${Math.round(elapsedTime / 60000)} minutes. Please check the PR manually: https://github.com/tldraw/tldraw/pull/${createdPr.data.number}`
|
||||
)
|
||||
}
|
||||
const prStatus = await getPrDetailsByNumber(octokit, createdPr.data.number)
|
||||
|
||||
// github stops reporting mergeability once a PR is merged or closed by hand, so without
|
||||
// this we'd spin until the timeout and report a false failure for a hotfix that landed.
|
||||
if (prStatus.merged) {
|
||||
await noteManualMerge(prStatus.merged_by?.login)
|
||||
break
|
||||
}
|
||||
if (prStatus.state === 'closed') {
|
||||
throw new Error(`Hotfix PR #${createdPr.data.number} was closed without being merged`)
|
||||
}
|
||||
|
||||
nicelog(`PR #${createdPr.data.number} mergeable_state: ${prStatus.mergeable_state}`)
|
||||
|
||||
if (prStatus.mergeable_state === 'clean') {
|
||||
nicelog(`PR #${createdPr.data.number} is ready for merge`)
|
||||
try {
|
||||
await octokit.rest.pulls.merge({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
pull_number: createdPr.data.number,
|
||||
merge_method: 'squash',
|
||||
commit_title: prTitle,
|
||||
commit_message: `This is an automated hotfix for dotcom deployment.
|
||||
|
||||
Original PR: #${pr.number}
|
||||
Original Author: @${pr.user?.login}`,
|
||||
})
|
||||
} catch (error) {
|
||||
// the merge api rejects an already-merged PR, so re-check before reporting a failure
|
||||
const latest = await getPrDetailsByNumber(octokit, createdPr.data.number)
|
||||
if (!latest.merged) throw error
|
||||
await noteManualMerge(latest.merged_by?.login)
|
||||
break
|
||||
}
|
||||
|
||||
nicelog(`Successfully merged hotfix PR #${createdPr.data.number}`)
|
||||
break
|
||||
} else if (prStatus.mergeable_state === 'unstable') {
|
||||
nicelog(`PR #${createdPr.data.number} is unstable (some checks failed)`)
|
||||
throw new Error(`Hotfix PR #${createdPr.data.number} is unstable`)
|
||||
} else if (prStatus.mergeable_state === 'dirty') {
|
||||
nicelog(`PR #${createdPr.data.number} has conflicts and cannot be merged`)
|
||||
throw new Error(`Hotfix PR #${createdPr.data.number} has conflicts`)
|
||||
} else {
|
||||
// `blocked` with no check runs at all means nothing is going to change: the push didn't
|
||||
// start any workflows (a skip-ci marker that survived, or a required check that isn't
|
||||
// wired up). fail now with a pointer instead of spinning until the timeout.
|
||||
if (prStatus.mergeable_state === 'blocked' && !checkedForMissingChecks) {
|
||||
checkedForMissingChecks = true
|
||||
const checks = await octokit.rest.checks.listForRef({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
ref: prStatus.head.sha,
|
||||
per_page: 1,
|
||||
})
|
||||
if (checks.data.total_count === 0) {
|
||||
throw new Error(
|
||||
`Hotfix PR #${createdPr.data.number} is blocked and no check runs have started on ${hotfixBranchName}. The required checks will never report, so it can't be merged automatically. Look at the commit message on that branch for a skip-ci marker: https://github.com/tldraw/tldraw/pull/${createdPr.data.number}`
|
||||
)
|
||||
}
|
||||
}
|
||||
nicelog(
|
||||
`PR #${createdPr.data.number} merge status: ${prStatus.mergeable_state}, waiting...`
|
||||
)
|
||||
await new Promise((resolve) => setTimeout(resolve, 15 * 1000))
|
||||
continue
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
await discord.step('Checks have passed, deploy will start soon', async () => {
|
||||
// This step just provides user feedback after successful merge
|
||||
})
|
||||
}
|
||||
|
||||
main().catch(async (e: Error) => {
|
||||
console.error(e)
|
||||
|
||||
const env = getEnv()
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
totalSteps: 3,
|
||||
shouldNotify: true,
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
|
||||
await discord
|
||||
.message(
|
||||
`❌ **Error triggering dotcom hotfix**\n\n\`\`\`ansi\n${e.message.slice(0, 2000)}\n\`\`\``
|
||||
)
|
||||
.finally(() => {
|
||||
process.exit(1)
|
||||
})
|
||||
})
|
||||
168
internal/scripts/trigger-sdk-hotfix.ts
Normal file
168
internal/scripts/trigger-sdk-hotfix.ts
Normal file
@@ -0,0 +1,168 @@
|
||||
import kleur from 'kleur'
|
||||
import { Octokit } from 'octokit'
|
||||
import * as semver from 'semver'
|
||||
import { formatDiff, getAnyPackageDiff } from './lib/didAnyPackageChange'
|
||||
import { Discord } from './lib/discord'
|
||||
import { exec } from './lib/exec'
|
||||
import { makeEnv } from './lib/makeEnv'
|
||||
import { nicelog } from './lib/nicelog'
|
||||
import { getPrDetailsAndCommitSha, labelPresent, PullRequest } from './lib/pr-info'
|
||||
import { stripSkipCiMarkers } from './lib/skip-ci'
|
||||
import { getAllWorkspacePackages } from './lib/workspace'
|
||||
|
||||
function getEnv() {
|
||||
return makeEnv([
|
||||
'DISCORD_DEPLOY_WEBHOOK_URL',
|
||||
'GITHUB_TOKEN',
|
||||
// bemo URL is needed when building packages, which in this file
|
||||
// happens during docs-only mode when we need to check for sdk changes.
|
||||
// See the file
|
||||
'TLDRAW_BEMO_URL',
|
||||
])
|
||||
}
|
||||
|
||||
function getTriggerType(pr: PullRequest): 'none' | 'SDK' | 'docs' {
|
||||
nicelog(`Checking PR ${pr.number} labels...`)
|
||||
|
||||
const hasDocsHotfixLabel = labelPresent(pr, 'docs-hotfix-please')
|
||||
const hasSdkHotfixLabel = labelPresent(pr, 'sdk-hotfix-please')
|
||||
|
||||
if (!hasDocsHotfixLabel && !hasSdkHotfixLabel) {
|
||||
nicelog('No "(docs|sdk)-hotfix-please" label found. Exiting...')
|
||||
return 'none'
|
||||
}
|
||||
|
||||
nicelog(
|
||||
`Found "${hasSdkHotfixLabel ? 'sdk-hotfix-please' : 'docs-hotfix-please'}" label. Proceeding...`
|
||||
)
|
||||
|
||||
const isDocsOnly = hasDocsHotfixLabel && !hasSdkHotfixLabel
|
||||
return isDocsOnly ? 'docs' : 'SDK'
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const env = getEnv()
|
||||
const octokit = new Octokit({ auth: env.GITHUB_TOKEN })
|
||||
|
||||
const result = await getPrDetailsAndCommitSha(octokit)
|
||||
if (!result) {
|
||||
nicelog('Could not retrieve PR details. Exiting...')
|
||||
return
|
||||
}
|
||||
|
||||
const { pr, commitSha } = result
|
||||
|
||||
const triggerType = getTriggerType(pr)
|
||||
if (triggerType === 'none') return
|
||||
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
totalSteps: 3,
|
||||
shouldNotify: true,
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
await discord.message(`Triggering ${triggerType} hotfix...`)
|
||||
|
||||
// first cherry-pick HEAD on top of latest release branch
|
||||
const latestFullReleaseVersion = (await exec('npm', ['show', 'tldraw', 'version']))
|
||||
.trim()
|
||||
.split(/\n/g)
|
||||
.pop()
|
||||
|
||||
nicelog('Latest tldraw version on npm is', latestFullReleaseVersion)
|
||||
const version = semver.parse(latestFullReleaseVersion)
|
||||
if (!version) {
|
||||
throw new Error(
|
||||
`Couldn't get latest tldraw package version from npm: '${latestFullReleaseVersion}'`
|
||||
)
|
||||
}
|
||||
|
||||
const latestReleaseBranch = `v${version.major}.${version.minor}.x`
|
||||
nicelog('Latest release branch', latestReleaseBranch)
|
||||
|
||||
await discord.step(
|
||||
`Cherry-picking PR #${pr.number} onto branch ${latestReleaseBranch}`,
|
||||
async () => {
|
||||
await exec('git', ['fetch', 'origin', latestReleaseBranch])
|
||||
await exec('git', ['fetch', 'origin', 'main'])
|
||||
await exec('git', ['checkout', latestReleaseBranch])
|
||||
await exec('git', ['reset', `origin/${latestReleaseBranch}`, '--hard'])
|
||||
await exec('git', ['log', '-1', '--oneline'])
|
||||
await exec('git', ['cherry-pick', commitSha])
|
||||
|
||||
// the push to the release branch below must trigger publish.yml, but some
|
||||
// merge commits (e.g. release-notes updates) carry `[skip ci]`, which would
|
||||
// suppress it. strip skip-ci markers from the cherry-picked commit message.
|
||||
const message = (await exec('git', ['log', '-1', '--format=%B'])).trim()
|
||||
const cleanedMessage = stripSkipCiMarkers(message)
|
||||
if (cleanedMessage !== message) {
|
||||
await exec('git', ['commit', '--amend', '-m', cleanedMessage])
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
if (triggerType === 'docs') {
|
||||
await discord.step(`Ensuring no SDK changes are present`, async () => {
|
||||
// run yarn again before building packages to make sure everything is ready
|
||||
// in case HEAD included dev dependency changes
|
||||
await exec('yarn', ['install'])
|
||||
await exec('yarn', ['refresh-assets', '--force'])
|
||||
|
||||
const diff = await getAnyPackageDiff(version.format())
|
||||
if (diff) {
|
||||
let message = kleur.red().bold(`・ERROR・`)
|
||||
message += `\nCannot cherry-pick docs changes from PR '${kleur.cyan().bold(pr.title)}' https://github.com/tldraw/tldraw/pulls/${pr}`
|
||||
message += '\nThis PR contains changes to the SDK.\n\n'
|
||||
let formattedDiff = formatDiff(diff)
|
||||
// truncate long diffs for discord's sake, it's not that valuable to see the whole thing
|
||||
if (formattedDiff.length > 1000) {
|
||||
formattedDiff = formattedDiff.slice(0, 1000) + '\n...\n'
|
||||
}
|
||||
message += formattedDiff
|
||||
|
||||
message += '\n💡 Please cherry-pick this PR manually and remove the sdk changes.'
|
||||
message +=
|
||||
'\n Or apply the sdk-hotfix-please PR label and rerun this action to publish the sdk changes if appropriate.'
|
||||
|
||||
throw new Error(message)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
await discord.step('Running sdk tests', async () => {
|
||||
await exec('yarn', ['install'])
|
||||
const packages = await getAllWorkspacePackages()
|
||||
|
||||
await exec('yarn', [
|
||||
'test',
|
||||
...packages
|
||||
.filter((p) => !p.packageJson.private)
|
||||
.flatMap(({ name }) => ['--project', name]),
|
||||
])
|
||||
})
|
||||
}
|
||||
|
||||
await discord.step('Pushing to release branch', async () => {
|
||||
await exec('git', ['push', 'origin', `HEAD:${latestReleaseBranch}`])
|
||||
})
|
||||
// return
|
||||
}
|
||||
|
||||
main().catch(async (e: Error) => {
|
||||
console.error(e)
|
||||
|
||||
const env = getEnv()
|
||||
const discord = new Discord({
|
||||
webhookUrl: env.DISCORD_DEPLOY_WEBHOOK_URL,
|
||||
totalSteps: 8,
|
||||
shouldNotify: true,
|
||||
secretValues: Object.values(env),
|
||||
})
|
||||
|
||||
await discord
|
||||
.message(
|
||||
`❌ **Error triggering SDK hotfix on PR ${process.env.PR_NUMBER}**\n\n\`\`\`ansi\n${e.message.slice(0, 2000)}\n\`\`\``
|
||||
)
|
||||
.finally(() => {
|
||||
process.exit(1)
|
||||
})
|
||||
})
|
||||
21
internal/scripts/tsconfig.json
Normal file
21
internal/scripts/tsconfig.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"extends": "../config/tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"composite": false,
|
||||
"esModuleInterop": true,
|
||||
"emitDeclarationOnly": false,
|
||||
"noEmit": true
|
||||
},
|
||||
"include": ["**/*"],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../packages/dotcom-shared"
|
||||
},
|
||||
{
|
||||
"path": "../../packages/utils"
|
||||
},
|
||||
{
|
||||
"path": "../../packages/validate"
|
||||
}
|
||||
]
|
||||
}
|
||||
150
internal/scripts/typecheck.ts
Normal file
150
internal/scripts/typecheck.ts
Normal file
@@ -0,0 +1,150 @@
|
||||
import { execFile, execFileSync } from 'child_process'
|
||||
import path, { join } from 'path'
|
||||
import { REPO_ROOT, readJsonIfExists } from './lib/file'
|
||||
import { getAllWorkspacePackages } from './lib/workspace'
|
||||
|
||||
async function main() {
|
||||
const allWorkspaces = await getAllWorkspacePackages()
|
||||
const tsconfigFiles: string[] = []
|
||||
for (const workspace of allWorkspaces) {
|
||||
const tsconfigFile = path.join(workspace.path, 'tsconfig.json')
|
||||
const tsconfigExists = await readJsonIfExists(tsconfigFile)
|
||||
if (tsconfigExists) tsconfigFiles.push(tsconfigFile)
|
||||
}
|
||||
|
||||
const args = ['--build']
|
||||
const isWatchMode = process.argv.includes('--watch')
|
||||
if (process.argv.includes('--force')) args.push('--force')
|
||||
if (isWatchMode) args.push('--watch')
|
||||
if (process.argv.includes('--preserveWatchOutput')) args.push('--preserveWatchOutput')
|
||||
if (process.argv.includes('--extendedDiagnostics')) args.push('--extendedDiagnostics')
|
||||
|
||||
const compilerPath = join(REPO_ROOT, 'node_modules/.bin/tsc')
|
||||
|
||||
// tsc currently struggles to resolve some workspace package imports on a fresh checkout
|
||||
// when all projects are passed in a single --build invocation. Building package
|
||||
// workspaces first in topological order (leaves before dependents) produces the
|
||||
// declaration outputs needed for the full graph.
|
||||
const packageTsconfigFiles = tsconfigFiles.filter((file) =>
|
||||
file.includes(`${path.sep}packages${path.sep}`)
|
||||
)
|
||||
if (packageTsconfigFiles.length > 0) {
|
||||
const sortedPackageTsconfigFiles = await topoSortTsconfigs(packageTsconfigFiles)
|
||||
const bootstrapArgs = ['--build']
|
||||
if (process.argv.includes('--force')) bootstrapArgs.push('--force')
|
||||
for (const tsconfigFile of sortedPackageTsconfigFiles) {
|
||||
execFileSync(compilerPath, [...bootstrapArgs, tsconfigFile], { stdio: 'inherit' })
|
||||
}
|
||||
}
|
||||
|
||||
// In watch mode, use execFileSync with inherited stdio - it handles everything
|
||||
if (isWatchMode) {
|
||||
execFileSync(compilerPath, [...args, ...tsconfigFiles], { stdio: 'inherit' })
|
||||
return
|
||||
}
|
||||
|
||||
// In non-watch mode, capture output so we can show a clear error summary
|
||||
const output: string[] = []
|
||||
const errors: string[] = []
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
const childProcess = execFile(
|
||||
compilerPath,
|
||||
[...args, ...tsconfigFiles],
|
||||
{ cwd: REPO_ROOT },
|
||||
(err) => {
|
||||
if (err) {
|
||||
// Extract TypeScript errors from the output
|
||||
const allOutput = output.join('') + errors.join('')
|
||||
const tsErrors = allOutput.split('\n').filter((line) => {
|
||||
// Match TypeScript error lines like: "file.ts(123,4): error TS2304: ..."
|
||||
return /\.(ts|tsx|js|jsx)\(\d+,\d+\):\s+error\s+TS\d+:/i.test(line)
|
||||
})
|
||||
|
||||
// Display a clear error summary at the end
|
||||
console.error('\n' + '='.repeat(80))
|
||||
console.error('❌ TypeScript type checking failed')
|
||||
console.error('='.repeat(80) + '\n')
|
||||
|
||||
if (tsErrors.length > 0) {
|
||||
console.error('TypeScript errors:')
|
||||
tsErrors.forEach((error) => console.error(error))
|
||||
console.error('')
|
||||
} else if (errors.length > 0) {
|
||||
// Fallback: show stderr if we couldn't extract specific errors
|
||||
console.error('Error output:')
|
||||
console.error(errors.join(''))
|
||||
console.error('')
|
||||
}
|
||||
|
||||
console.error('Please fix the errors above and try again.')
|
||||
console.error('='.repeat(80) + '\n')
|
||||
process.exit(err.code || 1)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Capture and display stdout in real-time
|
||||
childProcess.stdout?.on('data', (chunk) => {
|
||||
const text = chunk.toString()
|
||||
output.push(text)
|
||||
process.stdout.write(text)
|
||||
})
|
||||
|
||||
// Capture and display stderr in real-time
|
||||
childProcess.stderr?.on('data', (chunk) => {
|
||||
const text = chunk.toString()
|
||||
errors.push(text)
|
||||
process.stderr.write(text)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Sort tsconfig files in topological order based on their `references` fields,
|
||||
* so that leaf packages (no dependencies) come first and dependents come after
|
||||
* all their dependencies. This is needed because tsc --build doesn't always
|
||||
* resolve the reference graph correctly on a fresh checkout.
|
||||
*/
|
||||
async function topoSortTsconfigs(tsconfigFiles: string[]): Promise<string[]> {
|
||||
const dirToFile = new Map<string, string>()
|
||||
const deps = new Map<string, string[]>()
|
||||
|
||||
for (const file of tsconfigFiles) {
|
||||
const dir = path.dirname(file)
|
||||
dirToFile.set(dir, file)
|
||||
const tsconfig = await readJsonIfExists(file)
|
||||
const refs: string[] = []
|
||||
if (tsconfig?.references) {
|
||||
for (const ref of tsconfig.references) {
|
||||
refs.push(path.resolve(dir, ref.path))
|
||||
}
|
||||
}
|
||||
deps.set(dir, refs)
|
||||
}
|
||||
|
||||
const visited = new Set<string>()
|
||||
const sorted: string[] = []
|
||||
|
||||
function visit(dir: string) {
|
||||
if (visited.has(dir)) return
|
||||
visited.add(dir)
|
||||
for (const dep of deps.get(dir) ?? []) {
|
||||
if (dirToFile.has(dep)) visit(dep)
|
||||
}
|
||||
sorted.push(dirToFile.get(dir)!)
|
||||
}
|
||||
|
||||
for (const dir of dirToFile.keys()) {
|
||||
visit(dir)
|
||||
}
|
||||
|
||||
return sorted
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Unexpected error:', err)
|
||||
process.exit(1)
|
||||
})
|
||||
70
internal/scripts/update-pr-template.ts
Normal file
70
internal/scripts/update-pr-template.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { Octokit } from '@octokit/rest'
|
||||
import { REPO_ROOT } from './lib/file'
|
||||
import { formatLabelOptionsForPRTemplate, getLabelNames } from './lib/labels'
|
||||
|
||||
const prTemplatePath = join(REPO_ROOT, '.github', 'pull_request_template.md')
|
||||
|
||||
const octo = process.env.GH_TOKEN ? new Octokit({ auth: process.env.GH_TOKEN }) : new Octokit()
|
||||
|
||||
async function updatePRTemplate(check: boolean) {
|
||||
if (!existsSync(prTemplatePath)) {
|
||||
console.error('❌ Could not find PR template at', prTemplatePath)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
const prTemplate = readFileSync(prTemplatePath).toString()
|
||||
const labelsPart = prTemplate.match(/(### Change type(.|\s)*?\n)###/)?.[1]
|
||||
if (!labelsPart) {
|
||||
console.error(
|
||||
'❌ Could not find the labels section of the pull request template! It should start with "### Change type"'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
const updated = prTemplate.replace(
|
||||
labelsPart,
|
||||
`### Change type\n\n${formatLabelOptionsForPRTemplate()}\n\n`
|
||||
)
|
||||
if (check && updated !== prTemplate) {
|
||||
console.error(
|
||||
'❌ PR template labels section is out of date. Run `yarn update-pr-template` to fix it.'
|
||||
)
|
||||
console.error(
|
||||
'💡 Were you trying to change the labels section manually? Update internal/scripts/lib/labels.ts instead.'
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// make sure all labels exist
|
||||
const repoLabels = new Set(
|
||||
(
|
||||
await octo.issues.listLabelsForRepo({
|
||||
owner: 'tldraw',
|
||||
repo: 'tldraw',
|
||||
per_page: 100,
|
||||
})
|
||||
).data.map((x) => x.name)
|
||||
)
|
||||
|
||||
const missingLabels = getLabelNames().filter((x) => !repoLabels.has(x))
|
||||
if (missingLabels.length > 0) {
|
||||
console.error(
|
||||
'❌ The following labels do not exist in the tldraw repo:',
|
||||
missingLabels.map((l) => JSON.stringify(l)).join(', ')
|
||||
)
|
||||
console.error(
|
||||
`Add them yourself or update internal/scripts/lib/labels.ts and re-run \`yarn update-pr-template\` to remove them.`
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
if (!check) {
|
||||
console.log('Writing template to', prTemplatePath)
|
||||
writeFileSync(prTemplatePath, updated)
|
||||
} else {
|
||||
console.log('All good!')
|
||||
}
|
||||
}
|
||||
|
||||
updatePRTemplate(process.argv.includes('--check'))
|
||||
6
internal/scripts/vercel/build-docs.sh
Executable file
6
internal/scripts/vercel/build-docs.sh
Executable file
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
yarn run -T lazy build-api
|
||||
yarn run -T lazy refresh-everything --filter=apps/docs
|
||||
yarn run -T lazy build --filter=apps/docs
|
||||
4
internal/scripts/vercel/build.sh
Executable file
4
internal/scripts/vercel/build.sh
Executable file
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
yarn run -T lazy build --filter=apps/examples --filter=packages/tldraw
|
||||
4
internal/scripts/vercel/install.sh
Executable file
4
internal/scripts/vercel/install.sh
Executable file
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
yarn
|
||||
20
internal/scripts/vercel/should-build-analytics.sh
Executable file
20
internal/scripts/vercel/should-build-analytics.sh
Executable file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
SCRIPT_DIR=$(dirname "$0")
|
||||
REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if [[ "$VERCEL_ENV" == "production" ]] ; then
|
||||
echo "Always build on production";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## main is not production anymore, but we still always want to build it
|
||||
if [[ "$VERCEL_GIT_COMMIT_REF" == "main" ]] ; then
|
||||
echo "Always build on main";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## on PR builds, only rebuild if the analytics directory changed
|
||||
git diff HEAD^ HEAD --quiet -- "./apps/analytics/"
|
||||
17
internal/scripts/vercel/should-build-docs.sh
Executable file
17
internal/scripts/vercel/should-build-docs.sh
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
if [[ "$VERCEL_ENV" == "production" ]] ; then
|
||||
echo "Always build on production";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## main is not production anymore, but we still always want to build it
|
||||
if [[ "$VERCEL_GIT_COMMIT_REF" == "main" ]] ; then
|
||||
echo "Always build on main";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## on PR builds, only rebuild if the docs directory changed
|
||||
git diff HEAD^ HEAD --quiet ./apps/docs
|
||||
|
||||
21
internal/scripts/vercel/should-build-template.sh
Executable file
21
internal/scripts/vercel/should-build-template.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
SCRIPT_DIR=$(dirname "$0")
|
||||
REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if [[ "$VERCEL_ENV" == "production" ]] ; then
|
||||
echo "Always build on production";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## main is not production anymore, but we still always want to build it
|
||||
if [[ "$VERCEL_GIT_COMMIT_REF" == "main" ]] ; then
|
||||
echo "Always build on main";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## on PR builds, only rebuild if the template directory changed
|
||||
TEMPLATE_NAME="$1"
|
||||
git diff HEAD^ HEAD --quiet "./templates/${TEMPLATE_NAME}/"
|
||||
20
internal/scripts/vercel/should-build-tldraw-new.sh
Executable file
20
internal/scripts/vercel/should-build-tldraw-new.sh
Executable file
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eux
|
||||
|
||||
SCRIPT_DIR=$(dirname "$0")
|
||||
REPO_ROOT=$(realpath "$SCRIPT_DIR/../../..")
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if [[ "$VERCEL_ENV" == "production" ]] ; then
|
||||
echo "Always build on production";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## main is not production anymore, but we still always want to build it
|
||||
if [[ "$VERCEL_GIT_COMMIT_REF" == "main" ]] ; then
|
||||
echo "Always build on main";
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
## on PR builds, only rebuild if the tldraw-new directory changed
|
||||
git diff HEAD^ HEAD --quiet -- "./apps/tldraw-new/"
|
||||
249
internal/scripts/workers/dev.ts
Normal file
249
internal/scripts/workers/dev.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
import { ChildProcessWithoutNullStreams, spawn } from 'child_process'
|
||||
import { readFileSync } from 'fs'
|
||||
import { createServer } from 'net'
|
||||
import { resolve } from 'path'
|
||||
import kleur from 'kleur'
|
||||
import { lock } from 'proper-lockfile'
|
||||
import stripAnsi from 'strip-ansi'
|
||||
import * as toml from 'toml'
|
||||
import { killProcessTree } from '../lib/kill-tree'
|
||||
import { WORKER_EXTERNAL_DEPS } from '../lib/worker-externals'
|
||||
|
||||
const lockfileName = __dirname
|
||||
|
||||
/**
|
||||
* a long time ago, workerd would regularly crash with a segfault but the error is not caught by the
|
||||
* process, so it will just hang. this script wraps the process, tailing the logs and restarting the
|
||||
* process if we encounter the string 'Segmentation fault'. we think this error is probably fixed
|
||||
* now.
|
||||
*
|
||||
* there's a separate issue where spawning multiple workerd instances at once would cause them to
|
||||
* fight and crash. we use a lockfile to start our wrokerd instances one at a time, waiting for the
|
||||
* previous one to be ready before we start the next.
|
||||
*/
|
||||
class MiniflareMonitor {
|
||||
private process: ChildProcessWithoutNullStreams | null = null
|
||||
|
||||
constructor(
|
||||
private command: string,
|
||||
private args: string[] = []
|
||||
) {}
|
||||
|
||||
public async start(): Promise<void> {
|
||||
await this.stop() // Ensure any existing process is stopped
|
||||
await this.lock()
|
||||
await console.log(`Starting wrangler...`)
|
||||
this.process = spawn(this.command, this.args, {
|
||||
env: {
|
||||
NODE_ENV: 'development',
|
||||
...process.env,
|
||||
},
|
||||
})
|
||||
|
||||
this.process.stdout.on('data', (data: Buffer) => {
|
||||
this.handleOutput(stripAnsi(data.toString().replace('\r', '').trim()))
|
||||
})
|
||||
|
||||
this.process.stderr.on('data', (data: Buffer) => {
|
||||
this.handleOutput(stripAnsi(data.toString().replace('\r', '').trim()), true)
|
||||
})
|
||||
}
|
||||
|
||||
private handleOutput(output: string, err = false): void {
|
||||
if (!output) return
|
||||
|
||||
if (output.includes('Ready on') && this.isLocked()) {
|
||||
this.release()
|
||||
}
|
||||
|
||||
if (output.includes('Segmentation fault')) {
|
||||
console.error('Segmentation fault detected. Restarting Miniflare...')
|
||||
this.restart()
|
||||
} else if (!err) {
|
||||
console.log(output.replace('[mf:inf]', '')) // or handle the output differently
|
||||
} else {
|
||||
console.error(output.replace('[mf:err]', '')) // or handle the output differently
|
||||
}
|
||||
}
|
||||
|
||||
private async restart(): Promise<void> {
|
||||
console.log('Restarting wrangler...')
|
||||
await this.stop()
|
||||
setTimeout(() => this.start(), 3000) // Restart after a short delay
|
||||
}
|
||||
|
||||
private async stop(): Promise<void> {
|
||||
if (this.isLocked()) await this.release()
|
||||
if (this.process) {
|
||||
this.process.kill()
|
||||
this.process = null
|
||||
}
|
||||
}
|
||||
|
||||
private _lockPromise?: Promise<() => Promise<void>>
|
||||
private isLocked() {
|
||||
return !!this._lockPromise
|
||||
}
|
||||
private async lock() {
|
||||
if (this.isLocked()) throw new Error('Already locked')
|
||||
console.log('Locking...')
|
||||
this._lockPromise = lock(lockfileName, {
|
||||
retries: {
|
||||
minTimeout: 500,
|
||||
retries: 10,
|
||||
},
|
||||
})
|
||||
await this._lockPromise
|
||||
console.log('Locked')
|
||||
}
|
||||
private async release() {
|
||||
if (!this.isLocked()) throw new Error('Not locked')
|
||||
console.log('Releasing...')
|
||||
const lockPromise = this._lockPromise!
|
||||
this._lockPromise = undefined
|
||||
const release = await lockPromise
|
||||
await release()
|
||||
console.log('Released')
|
||||
}
|
||||
}
|
||||
|
||||
class SizeReporter {
|
||||
lastLineTime = Date.now()
|
||||
nextTick?: NodeJS.Timeout
|
||||
|
||||
size = 0
|
||||
|
||||
start() {
|
||||
console.log('Spawning size reporter...')
|
||||
const proc = spawn('yarn', [
|
||||
'run',
|
||||
'-T',
|
||||
'esbuild',
|
||||
'src/worker.ts',
|
||||
'--bundle',
|
||||
'--minify',
|
||||
'--watch',
|
||||
// need to list out node packages that are used in the worker.
|
||||
// otherwise, if we use platform=node, the bundle size is not reported correctly
|
||||
...WORKER_EXTERNAL_DEPS.map((dep) => '--external:' + dep),
|
||||
'--target=esnext',
|
||||
'--format=esm',
|
||||
])
|
||||
// listen for lines on stdin
|
||||
proc.stdout.on('data', (data) => {
|
||||
this.size += data.length
|
||||
this.lastLineTime = Date.now()
|
||||
clearTimeout(this.nextTick)
|
||||
this.nextTick = setTimeout(() => {
|
||||
console.log(
|
||||
kleur.bold(kleur.yellow('worker')),
|
||||
'is roughly',
|
||||
kleur.bold(kleur.cyan(Math.floor(this.size / 1024) + 'kb')),
|
||||
'(minified)\n'
|
||||
)
|
||||
this.size = 0
|
||||
}, 10)
|
||||
})
|
||||
proc.stderr.on('data', (data) => {
|
||||
console.log(data.toString())
|
||||
})
|
||||
process.on('SIGINT', () => {
|
||||
console.log('Int')
|
||||
proc.kill()
|
||||
})
|
||||
process.on('SIGTERM', () => {
|
||||
console.log('Term')
|
||||
proc.kill()
|
||||
})
|
||||
process.on('exit', () => {
|
||||
console.log('Exiting')
|
||||
proc.kill()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function getDevPort(args: string[]): number | null {
|
||||
const portIdx = args.findIndex((a) => a === '--port')
|
||||
if (portIdx >= 0 && args[portIdx + 1]) {
|
||||
const fromArg = Number(args[portIdx + 1])
|
||||
return Number.isFinite(fromArg) ? fromArg : null
|
||||
}
|
||||
try {
|
||||
const parsed = toml.parse(readFileSync(resolve(process.cwd(), 'wrangler.toml'), 'utf8'))
|
||||
const port = parsed?.dev?.port
|
||||
return typeof port === 'number' ? port : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function probePortFree(port: number): Promise<void> {
|
||||
return new Promise((res, rej) => {
|
||||
const server = createServer()
|
||||
server.unref()
|
||||
server.once('error', rej)
|
||||
server.once('listening', () => server.close(() => res()))
|
||||
server.listen(port, '0.0.0.0')
|
||||
})
|
||||
}
|
||||
|
||||
async function ensurePortFreeOrExit(port: number) {
|
||||
try {
|
||||
await probePortFree(port)
|
||||
} catch (err: any) {
|
||||
if (err?.code === 'EADDRINUSE') {
|
||||
console.error(
|
||||
`\n${kleur.red(kleur.bold(`✖ Port ${port} is already in use.`))}\n\n` +
|
||||
`This worker (${kleur.cyan(process.cwd())}) needs port ${port}, but another\n` +
|
||||
`process is already listening on it. Wrangler would silently fall back to a random\n` +
|
||||
`port, but the rest of the dev stack expects ${port} — so /api requests from the\n` +
|
||||
`client would either 404 or hit the wrong worker.\n\n` +
|
||||
`Find what's holding it:\n` +
|
||||
` ${kleur.bold(`lsof -nP -iTCP:${port} -sTCP:LISTEN`)}\n\n` +
|
||||
`Then stop that process and re-run.\n`
|
||||
)
|
||||
} else {
|
||||
console.error(`Failed to probe port ${port}:`, err)
|
||||
}
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const port = getDevPort(process.argv.slice(2))
|
||||
if (port != null) await ensurePortFreeOrExit(port)
|
||||
|
||||
new MiniflareMonitor('wrangler', [
|
||||
'dev',
|
||||
'--env',
|
||||
'dev',
|
||||
'--test-scheduled',
|
||||
'--log-level',
|
||||
'info',
|
||||
'--var',
|
||||
'IS_LOCAL:true',
|
||||
...process.argv.slice(2),
|
||||
]).start()
|
||||
|
||||
new SizeReporter().start()
|
||||
|
||||
// On shutdown, reap the whole subtree while it is still alive. wrangler spawns workerd as a child
|
||||
// and the size reporter spawns esbuild through a `yarn run -T` wrapper, so just signalling our
|
||||
// direct children would let those grandchildren reparent to launchd and keep holding the dev port.
|
||||
// Walking by PID, deepest-first, crosses those wrapper and process boundaries. We do this in the
|
||||
// signal handler (not on `exit`) so the tree is still enumerable.
|
||||
let shuttingDown = false
|
||||
const shutdown = () => {
|
||||
if (shuttingDown) return
|
||||
shuttingDown = true
|
||||
killProcessTree(process.pid)
|
||||
process.exit(0)
|
||||
}
|
||||
process.on('SIGINT', shutdown)
|
||||
process.on('SIGTERM', shutdown)
|
||||
process.on('SIGHUP', shutdown)
|
||||
// Backstop for any exit path that bypassed the signal handler.
|
||||
process.on('exit', () => killProcessTree(process.pid))
|
||||
}
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user