snapshot of https://github.com/sqlc-dev/sqlc.git (main, 2026-09-16; blobs > 131072B omitted)

internal/cache/action.go

@@ -0,0 +1,209 @@
+package cache
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+// ActionResult records the outputs of a completed action, mirroring Bazel's
+// ActionResult message. Outputs are not stored inline: each named output is a
+// digest pointing into the CAS.
+type ActionResult struct {
+	// Outputs maps an output name (e.g. "analysis.pb", "plugin.wasm") to the
+	// CAS digest of its contents.
+	Outputs map[string]Digest `json:"outputs"`
+}
+
+// ActionCache maps action digests to ActionResults, stored as JSON files at
+// ac/<xx>/<hash> under the cache root. Unlike CAS entries, action cache
+// entries are not self-validating — the value is not derivable from the key —
+// so Get additionally checks that every referenced output still exists in the
+// CAS before reporting a hit, exactly like Bazel's disk cache does.
+//
+// Entry I/O goes through the same os.Root as the CAS, confining every read
+// and write to the cache directory.
+type ActionCache struct {
+	root *os.Root
+	cas  *CAS
+}
+
+func newActionCache(root *os.Root, cas *CAS) *ActionCache {
+	return &ActionCache{root: root, cas: cas}
+}
+
+// path returns an entry's path relative to the cache root.
+func (a *ActionCache) path(d Digest) string {
+	return filepath.Join("ac", d.Hash[:2], d.Hash)
+}
+
+// Get returns the cached result for an action, or ErrNotFound on a miss. An
+// entry whose outputs are missing or corrupt in the CAS is treated as a miss
+// and evicted.
+func (a *ActionCache) Get(action Digest) (*ActionResult, error) {
+	if !action.valid() {
+		return nil, ErrNotFound
+	}
+	path := a.path(action)
+	data, err := a.root.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, ErrNotFound
+		}
+		return nil, fmt.Errorf("cache: %w", err)
+	}
+	var result ActionResult
+	if err := json.Unmarshal(data, &result); err != nil {
+		a.root.Remove(path)
+		return nil, ErrNotFound
+	}
+	for _, d := range result.Outputs {
+		if !a.cas.Contains(d) {
+			a.root.Remove(path)
+			return nil, ErrNotFound
+		}
+	}
+	return &result, nil
+}
+
+// PutTree stores every file under dir in the CAS and records them as the
+// action's outputs, named by their paths relative to dir. Use this for
+// actions whose tool writes an output directory, like WASM compilation.
+func (a *ActionCache) PutTree(action Digest, dir string) error {
+	outputs := map[string]Digest{}
+	err := filepath.WalkDir(dir, func(path string, entry fs.DirEntry, err error) error {
+		if err != nil || entry.IsDir() {
+			return err
+		}
+		rel, err := filepath.Rel(dir, path)
+		if err != nil {
+			return err
+		}
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		d, err := a.cas.Put(data)
+		if err != nil {
+			return err
+		}
+		outputs[filepath.ToSlash(rel)] = d
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	if len(outputs) == 0 {
+		return fmt.Errorf("cache: no outputs found under %s", dir)
+	}
+	return a.Put(action, &ActionResult{Outputs: outputs})
+}
+
+// GetTree materializes a cached action's outputs as files under dir, or
+// returns ErrNotFound on a miss. A file already present is reused only if its
+// contents still hash to the expected digest; otherwise it is rewritten from
+// the CAS. Writes are staged, fsynced, and renamed, so a crash cannot leave a
+// right-sized but torn file that later reads would trust.
+func (a *ActionCache) GetTree(action Digest, dir string) error {
+	result, err := a.Get(action)
+	if err != nil {
+		return err
+	}
+	for rel, d := range result.Outputs {
+		// Output names come from the action cache entry, which — unlike a CAS
+		// blob — is not self-validating, so a tampered entry could carry a
+		// name like "../../etc/x". Reject anything that isn't a relative path
+		// confined to dir; this is the confinement the os.Root gives the rest
+		// of the cache, which GetTree can't use because dir is a caller-owned
+		// path a tool must read by absolute name.
+		if !filepath.IsLocal(rel) {
+			return fmt.Errorf("cache: unsafe output path %q in action %s", rel, action)
+		}
+		path := filepath.Join(dir, filepath.FromSlash(rel))
+		// Trust an existing file only if it still hashes to the digest;
+		// size alone can mask in-place corruption that would make the
+		// consuming tool hard-fail with no way to repair (see wazero).
+		if existing, err := os.ReadFile(path); err == nil && DigestOf(existing) == d {
+			continue
+		}
+		data, err := a.cas.Get(d)
+		if err != nil {
+			return err
+		}
+		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			return fmt.Errorf("cache: %w", err)
+		}
+		if err := writeFileAtomic(path, data); err != nil {
+			return fmt.Errorf("cache: %w", err)
+		}
+	}
+	return nil
+}
+
+// writeFileAtomic writes data to path via a staged temp file in the same
+// directory, fsynced before an atomic rename, so a reader never observes a
+// partial or torn file even across a crash.
+func writeFileAtomic(path string, data []byte) error {
+	f, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+"-*")
+	if err != nil {
+		return err
+	}
+	tmp := f.Name()
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Sync(); err != nil {
+		f.Close()
+		os.Remove(tmp)
+		return err
+	}
+	if err := f.Close(); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		os.Remove(tmp)
+		return err
+	}
+	return nil
+}
+
+// Put records the result of an action. All outputs must already be in the
+// CAS; writes are staged and renamed so concurrent processes never observe a
+// partial entry.
+func (a *ActionCache) Put(action Digest, result *ActionResult) error {
+	for name, d := range result.Outputs {
+		if !a.cas.Contains(d) {
+			return fmt.Errorf("cache: output %q (%s) missing from CAS", name, d)
+		}
+	}
+	data, err := json.Marshal(result)
+	if err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	path := a.path(action)
+	if err := a.root.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	f, name, err := a.cas.createTemp(action.Hash[:8] + "-")
+	if err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	defer a.root.Remove(name)
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		return fmt.Errorf("cache: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	if err := a.root.Rename(name, path); err != nil {
+		return fmt.Errorf("cache: %w", err)
+	}
+	return nil
+}

internal/cache/cache.go

@@ -0,0 +1,103 @@
+// Package cache implements sqlc's on-disk cache, designed after Bazel's
+// local disk cache.
+//
+// It has two halves:
+//
+//   - A content-addressable store (CAS) holding blobs keyed by the SHA-256
+//     hash of their contents, laid out as cas/<xx>/<hash>.
+//   - An action cache (AC) mapping the digest of an action — a description of
+//     cacheable work and all of its inputs — to the digests of the outputs
+//     that work produced, laid out as ac/<xx>/<hash>.
+//
+// Work whose output is not derivable from its inputs, like query analysis,
+// uses both halves, just like Bazel: hash the action, look its digest up in
+// the action cache, then fetch the referenced output blobs from the CAS.
+//
+// Remote fetches with a declared checksum, like WASM plugins, need no action
+// cache entry at all: the declared sha256 is itself a content address, so the
+// blob is stored and loaded directly from the CAS keyed by that checksum.
+package cache
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// Cache bundles the CAS and the action cache that shares it. All storage
+// I/O is confined to the cache directory through an os.Root; callers should
+// Close the cache when finished with it to release the root.
+type Cache struct {
+	root    *os.Root
+	CAS     *CAS
+	Actions *ActionCache
+}
+
+// Dir returns the cache root, defaulting to os.UserCacheDir(). The location
+// can be overridden with the SQLCCACHE environment variable.
+func Dir() (string, error) {
+	cache := os.Getenv("SQLCCACHE")
+	if cache != "" {
+		return cache, nil
+	}
+	cacheHome, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(cacheHome, "sqlc"), nil
+}
+
+// Open returns the cache rooted at Dir().
+func Open() (*Cache, error) {
+	root, err := Dir()
+	if err != nil {
+		return nil, err
+	}
+	return OpenAt(root)
+}
+
+// OpenAt returns the cache rooted at the given directory, creating it if
+// necessary.
+func OpenAt(dir string) (*Cache, error) {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create %s directory: %w", dir, err)
+	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return nil, fmt.Errorf("cache: %w", err)
+	}
+	cas, err := newCAS(root)
+	if err != nil {
+		root.Close()
+		return nil, err
+	}
+	return &Cache{
+		root:    root,
+		CAS:     cas,
+		Actions: newActionCache(root, cas),
+	}, nil
+}
+
+// Close releases the cache's handle on its root directory.
+func (c *Cache) Close() error {
+	return c.root.Close()
+}
+
+// ExecDir creates a fresh, private scratch directory for materializing the
+// output tree of the given action, for tools that need their outputs on disk
+// (like wazero's compilation cache). Each call returns a new directory under
+// exec/, so two concurrent processes never share one — otherwise a tool
+// staging files there (wazero writes <key>.tmp files in place) could be swept
+// into the other's PutTree. The caller must remove it when done; its contents
+// are always reproducible from the CAS, so losing it is harmless.
+func (c *Cache) ExecDir(action Digest) (string, error) {
+	base := filepath.Join(c.root.Name(), "exec")
+	if err := os.MkdirAll(base, 0755); err != nil {
+		return "", fmt.Errorf("failed to create %s directory: %w", base, err)
+	}
+	dir, err := os.MkdirTemp(base, action.Hash+"-")
+	if err != nil {
+		return "", fmt.Errorf("cache: %w", err)
+	}
+	return dir, nil
+}

internal/cache/cas.go

@@ -0,0 +1,138 @@
+package cache
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+// ErrNotFound is returned when a blob or action result is not in the cache.
+var ErrNotFound = errors.New("cache: not found")
+
+// CAS is an on-disk content-addressable store keyed by SHA-256, modeled on
+// Bazel's disk cache. Blobs live at cas/<xx>/<hash> under the cache root,
+// where <xx> is the first two hex characters of the hash. Because a blob's
+// name is derived from its contents, entries never change once written:
+// writers race benignly and readers can detect corruption by re-hashing.
+//
+// Content with an externally declared checksum (remotely fetched plugins)
+// needs no action cache entry: the declared sha256 is the address, so it is
+// stored and loaded directly — see SHA256Digest.
+//
+// All I/O goes through an os.Root, so no entry name — hash-derived or read
+// from an action cache entry — can escape the cache directory.
+type CAS struct {
+	root *os.Root
+}
+
+func newCAS(root *os.Root) (*CAS, error) {
+	if err := root.MkdirAll("tmp", 0755); err != nil {
+		return nil, fmt.Errorf("cache: create tmp: %w", err)
+	}
+	return &CAS{root: root}, nil
+}
+
+// path returns a blob's path relative to the cache root.
+func (c *CAS) path(d Digest) string {
+	return filepath.Join("cas", d.Hash[:2], d.Hash)
+}
+
+// Filename returns the path of a stored blob, for a consumer that needs the
+// file rather than its bytes — SQLite, for one, opens a database by name. A
+// blob is named after the hash of its contents, so the file at this path never
+// changes and any number of processes may read it at once.
+//
+// It reports false when the blob is not stored.
+func (c *CAS) Filename(d Digest) (string, bool) {
+	if !c.Contains(d) {
+		return "", false
+	}
+	return filepath.Join(c.root.Name(), c.path(d)), true
+}
+
+// createTemp creates a staging file under tmp/ in the cache root, returning
+// the open file and its root-relative name.
+func (c *CAS) createTemp(prefix string) (*os.File, string, error) {
+	for range 10000 {
+		name := filepath.Join("tmp", prefix+strconv.FormatUint(rand.Uint64(), 36))
+		f, err := c.root.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0644)
+		if errors.Is(err, fs.ErrExist) {
+			continue
+		}
+		return f, name, err
+	}
+	return nil, "", errors.New("cache: could not create temp file")
+}
+
+// Put stores a blob and returns its digest. Writing is atomic: the blob is
+// staged in a temp file and renamed into place, so concurrent sqlc processes
+// never observe partial entries.
+func (c *CAS) Put(data []byte) (Digest, error) {
+	d := DigestOf(data)
+	path := c.path(d)
+	// Skip the write only when an entry of the right size already exists; a
+	// wrong-sized entry is corrupt and is atomically replaced by the rename
+	// below.
+	if fi, err := c.root.Stat(path); err == nil && fi.Size() == d.SizeBytes {
+		return d, nil
+	}
+	if err := c.root.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return Digest{}, fmt.Errorf("cache: %w", err)
+	}
+	f, name, err := c.createTemp(d.Hash[:8] + "-")
+	if err != nil {
+		return Digest{}, fmt.Errorf("cache: %w", err)
+	}
+	defer c.root.Remove(name)
+	if _, err := f.Write(data); err != nil {
+		f.Close()
+		return Digest{}, fmt.Errorf("cache: %w", err)
+	}
+	if err := f.Close(); err != nil {
+		return Digest{}, fmt.Errorf("cache: %w", err)
+	}
+	if err := c.root.Rename(name, path); err != nil {
+		return Digest{}, fmt.Errorf("cache: %w", err)
+	}
+	return d, nil
+}
+
+// Get returns the blob for a digest. Contents are re-hashed before being
+// returned; a corrupt entry is evicted and reported as ErrNotFound so
+// callers simply redo the work that produced it.
+func (c *CAS) Get(d Digest) ([]byte, error) {
+	if !d.valid() {
+		return nil, ErrNotFound
+	}
+	path := c.path(d)
+	data, err := c.root.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, fs.ErrNotExist) {
+			return nil, ErrNotFound
+		}
+		return nil, fmt.Errorf("cache: %w", err)
+	}
+	if DigestOf(data).Hash != d.Hash {
+		c.root.Remove(path)
+		return nil, ErrNotFound
+	}
+	return data, nil
+}
+
+// Contains reports whether a blob with the given digest is present, checking
+// size when the digest carries one. It does not verify contents; Get performs
+// full verification.
+func (c *CAS) Contains(d Digest) bool {
+	if !d.valid() {
+		return false
+	}
+	fi, err := c.root.Stat(c.path(d))
+	if err != nil {
+		return false
+	}
+	return d.SizeBytes < 0 || fi.Size() == d.SizeBytes
+}

internal/cache/digest.go

@@ -0,0 +1,100 @@
+package cache
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"hash"
+)
+
+// Digest identifies a blob by its SHA-256 hash and size, mirroring the
+// Digest message from Bazel's remote execution API. The size is stored
+// alongside the hash so that entries can be validated without reading blob
+// contents; a negative size means the size is unknown, as with a checksum
+// declared in a configuration file.
+type Digest struct {
+	// Hash is the lowercase hex-encoded SHA-256 hash of the blob.
+	Hash string `json:"hash"`
+	// SizeBytes is the length of the blob in bytes, or negative if unknown.
+	SizeBytes int64 `json:"size_bytes"`
+}
+
+func (d Digest) String() string {
+	return fmt.Sprintf("sha256:%s/%d", d.Hash, d.SizeBytes)
+}
+
+func (d Digest) valid() bool {
+	if len(d.Hash) != 64 {
+		return false
+	}
+	_, err := hex.DecodeString(d.Hash)
+	return err == nil
+}
+
+// DigestOf returns the Digest of a blob.
+func DigestOf(data []byte) Digest {
+	sum := sha256.Sum256(data)
+	return Digest{
+		Hash:      hex.EncodeToString(sum[:]),
+		SizeBytes: int64(len(data)),
+	}
+}
+
+// SHA256Digest returns a Digest referencing a blob by a declared SHA-256
+// checksum whose size is not known, suitable for looking up remotely fetched
+// content in the CAS.
+func SHA256Digest(hexhash string) Digest {
+	return Digest{
+		Hash:      hexhash,
+		SizeBytes: -1,
+	}
+}
+
+// An Action describes a unit of cacheable work, playing the role of Bazel's
+// Action message: a mnemonic naming the kind of work plus the complete set of
+// inputs that determine its outputs. Two actions with the same digest are
+// assumed to produce the same outputs.
+//
+// Inputs are hashed incrementally with length-prefixed framing so that the
+// boundary between inputs is unambiguous ("ab"+"c" hashes differently from
+// "a"+"bc").
+type Action struct {
+	hasher hash.Hash
+}
+
+// NewAction starts building an action key for the given mnemonic, e.g.
+// "QueryAnalysis". The sha256 of the sqlc binary itself is always the first
+// input: the tool that executes an action determines its outputs just as
+// much as the declared inputs do, so a rebuilt sqlc never reuses stale
+// entries. The binary's digest is memoized in the cache — see toolDigest —
+// which is why actions are created through a Cache.
+func (c *Cache) NewAction(mnemonic string) *Action {
+	a := &Action{hasher: sha256.New()}
+	a.write([]byte(mnemonic))
+	a.AddInput("tool", c.toolDigest())
+	return a
+}
+
+// AddInput mixes a named input into the action key. Order matters: callers
+// must add inputs in a deterministic order.
+func (a *Action) AddInput(name string, data []byte) *Action {
+	a.write([]byte(name))
+	a.write(data)
+	return a
+}
+
+// Digest returns the action's digest, used as the action cache key.
+func (a *Action) Digest() Digest {
+	return Digest{
+		Hash:      hex.EncodeToString(a.hasher.Sum(nil)),
+		SizeBytes: 0,
+	}
+}
+
+func (a *Action) write(data []byte) {
+	var frame [8]byte
+	binary.LittleEndian.PutUint64(frame[:], uint64(len(data)))
+	a.hasher.Write(frame[:])
+	a.hasher.Write(data)
+}

internal/cache/tool.go

@@ -0,0 +1,95 @@
+package cache
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"os"
+	"sync"
+
+	"github.com/sqlc-dev/sqlc/internal/info"
+)
+
+// The sha256 of the sqlc binary is an input to every action — a rebuilt sqlc
+// may analyze queries or embed a different wazero than the one that produced
+// a cache entry, even when the version string is unchanged (dev builds).
+//
+// Hashing a ~100MB executable costs tens of milliseconds, too much to pay on
+// every short-lived sqlc process, so the digest is memoized on disk keyed by
+// the executable's path, size, and mtime — the same trick Bazel's file
+// digest cache uses. A warm run costs one stat and a tiny read; only a
+// rebuilt (or moved) binary is re-hashed.
+var tool struct {
+	sync.Mutex
+	digest []byte
+}
+
+type toolMemo struct {
+	Path      string `json:"path"`
+	SizeBytes int64  `json:"size_bytes"`
+	MtimeNS   int64  `json:"mtime_ns"`
+	SHA256    string `json:"sha256"`
+}
+
+const toolMemoPath = "tool"
+
+func (c *Cache) toolDigest() []byte {
+	tool.Lock()
+	defer tool.Unlock()
+	if tool.digest == nil {
+		tool.digest = c.computeToolDigest()
+	}
+	return tool.digest
+}
+
+func (c *Cache) computeToolDigest() []byte {
+	path, err := os.Executable()
+	if err != nil {
+		return []byte(info.Version)
+	}
+	fi, err := os.Stat(path)
+	if err != nil {
+		return []byte(info.Version)
+	}
+
+	var memo toolMemo
+	if data, err := c.root.ReadFile(toolMemoPath); err == nil {
+		if err := json.Unmarshal(data, &memo); err == nil &&
+			memo.Path == path &&
+			memo.SizeBytes == fi.Size() &&
+			memo.MtimeNS == fi.ModTime().UnixNano() &&
+			memo.SHA256 != "" {
+			return []byte(memo.SHA256)
+		}
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return []byte(info.Version)
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return []byte(info.Version)
+	}
+	sum := hex.EncodeToString(h.Sum(nil))
+
+	memo = toolMemo{
+		Path:      path,
+		SizeBytes: fi.Size(),
+		MtimeNS:   fi.ModTime().UnixNano(),
+		SHA256:    sum,
+	}
+	if data, err := json.Marshal(memo); err == nil {
+		if f, name, err := c.CAS.createTemp("tool-"); err == nil {
+			if _, werr := f.Write(data); werr == nil && f.Close() == nil {
+				c.root.Rename(name, toolMemoPath)
+			} else {
+				f.Close()
+			}
+			c.root.Remove(name)
+		}
+	}
+	return []byte(sum)
+}