upstream-sqlc/internal/goldeneye/sqlite/install.go

package sqlite

import (
	"archive/zip"
	"context"
	"crypto/sha3"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"slices"
	"strings"
	"sync"
)

// DefaultVersion is the SQLite release the dialect is generated from. It is
// the release the ncruces/go-sqlite3 driver in the main module embeds, so
// the functions the dialect knows are the ones the tests run against.
// Bumping it is a deliberate change: releases add functions and overloads,
// so regenerate and review the dialect after changing it, and add the new
// release's amalgamation to the table below.
const DefaultVersion = "3.53.4"

// defaultOptions are the compile options the base dialect is built with:
// the ones sqlite.org's own configure turns on by default, and so the ones
// a stock sqlite3 has. JSON is part of the library unless omitted, so only
// the math functions need naming.
var defaultOptions = []string{"SQLITE_ENABLE_MATH_FUNCTIONS"}

// extensions are the compile options that each become a directory under
// the dialect's extensions/, holding the functions a build with the option
// adds over the default build — the way each PostgreSQL contrib extension
// holds what CREATE EXTENSION adds. Each is named after its option as
// pragma compile_options spells it, in lower case, and lists the options
// its build needs: GEOPOLY lives inside the RTREE module, so enabling it
// alone adds nothing. Options that add virtual tables but no functions,
// such as SESSION and DBSTAT, are not listed, since the dialect has nothing
// to say about them, and ENABLE_FTS4 is the same module as ENABLE_FTS3.
var extensions = []build{
	{"soundex", []string{"SQLITE_SOUNDEX"}},
	{"enable_fts3", []string{"SQLITE_ENABLE_FTS3"}},
	{"enable_fts5", []string{"SQLITE_ENABLE_FTS5"}},
	{"enable_geopoly", []string{"SQLITE_ENABLE_RTREE", "SQLITE_ENABLE_GEOPOLY"}},
	{"enable_offset_sql_func", []string{"SQLITE_ENABLE_OFFSET_SQL_FUNC"}},
	{"enable_percentile", []string{"SQLITE_ENABLE_PERCENTILE"}},
	{"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}},
}

// analysisShell is the shell the analyze cases run through, which is no
// dialect build: nothing is generated from it. It is built with column
// metadata, so that `.stats stmt` can say which table column each result
// column of a statement comes from, and with every extension option at
// once, so that whatever a case's schema asks for is there.
var analysisShell = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)}

// extensionOptions is every option an extension build turns on, once each.
func extensionOptions() []string {
	var opts []string
	for _, b := range extensions {
		for _, opt := range b.options {
			if !slices.Contains(opts, opt) {
				opts = append(opts, opt)
			}
		}
	}
	return opts
}

// asset is one downloadable amalgamation of SQLite: the zip published on
// sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c.
type asset struct {
	Version string
	// Path is the download's address under https://sqlite.org/, the
	// release year included, as the download page's index lists it.
	Path string
	// SHA3 is the SHA3-256 of the zip, as the download page lists it.
	SHA3 string
}

// assets lists every amalgamation Install knows how to fetch, with the
// SHA3-256 of the download. A version that is not in this table cannot be
// installed: verifying the download is the point of the table. The
// checksums are the ones in the index at the foot of
// https://sqlite.org/download.html.
var assets = []asset{
	{"3.53.4", "2026/sqlite-amalgamation-3530400.zip", "628a44cfe82c66aed1ccbbe85a562d2e33ebe64b3288981ed76285612227934e"},
}

// sources are the files taken out of the amalgamation.
var sources = []string{"sqlite3.c", "sqlite3.h", "shell.c"}

func releaseAsset(version string) (asset, error) {
	for _, a := range assets {
		if a.Version == version {
			return a, nil
		}
	}
	return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its amalgamation and checksum to install.go", version)
}

func (a asset) url() string {
	return "https://sqlite.org/" + a.Path
}

// cacheDir is where Install puts a version: the sources under src/, and one
// shell per build under default/, under each option's extension name and
// under analysis/.
func cacheDir(version string) (string, error) {
	dir, err := os.UserCacheDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(dir, "sqlc-sqlite", version), nil
}

// build is one compiled shell: its name, which is the directory it is
// under in the version's cache directory and, for an extension, under the
// dialect's extensions/, and the options it is built with beyond the
// default ones.
type build struct {
	name    string
	options []string
}

// builds lists the dialect builds: the default build first, then one per
// extension.
func builds() []build {
	return append([]build{{"default", nil}}, extensions...)
}

// shells lists every build Install makes: the dialect builds and the
// analysis shell.
func shells() []build {
	return append(builds(), analysisShell)
}

// flags are every option a build is compiled with.
func (b build) flags() []string {
	return append(append([]string{}, defaultOptions...), b.options...)
}

func (b build) binary(dir string) string {
	return filepath.Join(dir, b.name, "sqlite3")
}

// Locate finds the directory of shells Install made for DefaultVersion.
func Locate() (string, error) {
	dir, err := cacheDir(DefaultVersion)
	if err != nil {
		return "", err
	}
	for _, b := range shells() {
		if _, err := os.Stat(b.binary(dir)); err != nil {
			return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " "))
		}
	}
	return dir, nil
}

// Install downloads the amalgamation for a version into the cache and
// compiles the shell from it once per build, returning the directory the
// shells are under. Builds already there are kept, so adding an option
// compiles only its shell. The download is checked against the table's
// SHA3-256 before it is unpacked. Compiling takes the compiler CC names, or
// cc, and a few seconds per build without optimisation, which a shell that
// only reads catalogs does not need.
//
// goos and goarch are what every installer is handed; the sources build the
// same everywhere a C compiler is, but no default compiler or link line is
// known for Windows.
func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) {
	if goos == "windows" {
		return "", errors.New("building SQLite from source is not supported on Windows")
	}
	dir, err := cacheDir(version)
	if err != nil {
		return "", err
	}
	var missing []build
	for _, b := range shells() {
		if _, err := os.Stat(b.binary(dir)); err != nil {
			missing = append(missing, b)
		}
	}
	if len(missing) == 0 {
		return dir, nil
	}
	if err := download(ctx, version, dir, progress); err != nil {
		return "", err
	}
	if err := compile(ctx, dir, missing, progress); err != nil {
		return "", err
	}
	return dir, nil
}

// download fetches the amalgamation and unpacks the sources into src/,
// unless they are already there.
func download(ctx context.Context, version, dir string, progress io.Writer) error {
	src := filepath.Join(dir, "src")
	have := true
	for _, name := range sources {
		if _, err := os.Stat(filepath.Join(src, name)); err != nil {
			have = false
		}
	}
	if have {
		return nil
	}
	a, err := releaseAsset(version)
	if err != nil {
		return err
	}
	if err := os.MkdirAll(src, 0o755); err != nil {
		return err
	}

	url := a.url()
	fmt.Fprintf(progress, "downloading %s\n", url)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return err
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("downloading %s: %s", url, resp.Status)
	}

	// The zip has to land on disk before it can be read — its directory is
	// at the end — so download it whole and hash every byte on the way.
	archive, err := os.CreateTemp(dir, "sqlite-amalgamation-*.partial")
	if err != nil {
		return err
	}
	defer os.Remove(archive.Name())
	sum := sha3.New256()
	if _, err := io.Copy(io.MultiWriter(archive, sum), resp.Body); err != nil {
		archive.Close()
		return err
	}
	if err := archive.Close(); err != nil {
		return err
	}
	if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA3 {
		return fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3)
	}

	zr, err := zip.OpenReader(archive.Name())
	if err != nil {
		return err
	}
	defer zr.Close()
	for _, name := range sources {
		if err := extract(zr, name, filepath.Join(src, name)); err != nil {
			return fmt.Errorf("downloading %s: %w", url, err)
		}
	}
	return nil
}

// extract copies the named file out of the zip. The amalgamation's files
// sit in one directory named after the release, so the name is matched on
// its base.
func extract(zr *zip.ReadCloser, name, dest string) error {
	for _, f := range zr.File {
		if filepath.Base(f.Name) != name || f.FileInfo().IsDir() {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer rc.Close()
		tmp, err := os.CreateTemp(filepath.Dir(dest), name+"-*.partial")
		if err != nil {
			return err
		}
		defer os.Remove(tmp.Name())
		if _, err := io.Copy(tmp, rc); err != nil {
			tmp.Close()
			return err
		}
		if err := tmp.Close(); err != nil {
			return err
		}
		return os.Rename(tmp.Name(), dest)
	}
	return fmt.Errorf("zip does not contain %s", name)
}

// compile builds the shells, as many at a time as there are CPUs.
func compile(ctx context.Context, dir string, todo []build, progress io.Writer) error {
	cc := os.Getenv("CC")
	if cc == "" {
		cc = "cc"
	}
	if _, err := exec.LookPath(cc); err != nil {
		return fmt.Errorf("no C compiler found: put cc on PATH or set CC to one")
	}
	var wg sync.WaitGroup
	slots := make(chan struct{}, max(1, runtime.NumCPU()))
	errs := make([]error, len(todo))
	for i, b := range todo {
		wg.Add(1)
		go func() {
			defer wg.Done()
			slots <- struct{}{}
			defer func() { <-slots }()
			errs[i] = b.compile(ctx, cc, dir, progress)
		}()
	}
	wg.Wait()
	return errors.Join(errs...)
}

// compile builds one shell from the sources in src/, writing it beside its
// destination and renaming so a failed build never masquerades as one.
func (b build) compile(ctx context.Context, cc, dir string, progress io.Writer) error {
	dest := b.binary(dir)
	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
		return err
	}
	tmp := dest + ".partial"
	defer os.Remove(tmp)
	args := []string{"-O0"}
	for _, opt := range b.flags() {
		args = append(args, "-D"+opt)
	}
	src := filepath.Join(dir, "src")
	args = append(args, filepath.Join(src, "shell.c"), filepath.Join(src, "sqlite3.c"), "-o", tmp, "-lm", "-ldl", "-lpthread")
	fmt.Fprintf(progress, "building %s: %s %s\n", b.name, cc, strings.Join(args, " "))
	cmd := exec.CommandContext(ctx, cc, args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("building %s: %w\n%s", b.name, err, strings.TrimSpace(string(out)))
	}
	return os.Rename(tmp, dest)
}