snapshot of https://github.com/sqlc-dev/sqlc.git (main, 2026-09-16; blobs > 131072B omitted)

internal/goldeneye/sqlite/analyze.go

@@ -0,0 +1,269 @@
+package sqlite
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/analysis"
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend"
+)
+
+// The analyze cases are checked against the analysis shell, which is asked
+// three things about each query. What the library reports about the
+// prepared statement, through `.stats stmt`: each result column's name and,
+// for one read straight from a table, its declared type and which table
+// column it is, with the column's NOT NULL looked up in the catalog. What
+// the query returns, over the case's fixture and again over no rows, for
+// the columns the library has no answer for: an expression's type is the
+// storage class of its value, since SQLite types values rather than
+// expressions, and any column is nullable when a NULL comes back — an
+// aggregate over no rows, the far side of an outer join. And the bytecode
+// the statement compiles to, which is where the parameters are found,
+// since the library reports nothing about them but their number.
+
+// placeholder is one parameter of a query as sqlc numbers them: each ? in
+// turn, and each sqlc.arg name once, at its first appearance. SQLite
+// numbers ?NNN the same way, so the query is rewritten with those.
+type placeholder struct {
+	Number int
+	Name   string
+}
+
+func bind(sql string) (string, []placeholder) {
+	var phs []placeholder
+	numbers := map[string]int{}
+	out := endtoend.Rewrite(sql, func(name, _ string) string {
+		n, ok := numbers[name]
+		if name == "" || !ok {
+			n = len(phs) + 1
+			phs = append(phs, placeholder{Number: n, Name: name})
+			if name != "" {
+				numbers[name] = n
+			}
+		}
+		return fmt.Sprintf("?%d", n)
+	})
+	return out, phs
+}
+
+// Analyze runs a case's queries through the analysis shell and returns what
+// SQLite reports in the JSON shape sqlc analyze prints.
+func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) {
+	if err := checkOptions(ctx, dir, analysisShell); err != nil {
+		return nil, err
+	}
+	binary := analysisShell.binary(dir)
+	schema, err := os.ReadFile(c.Schema)
+	if err != nil {
+		return nil, err
+	}
+	var fixture []byte
+	if c.Fixture != "" {
+		if fixture, err = os.ReadFile(c.Fixture); err != nil {
+			return nil, err
+		}
+	}
+	queries, err := c.Queries()
+	if err != nil {
+		return nil, err
+	}
+	out := make([]analysis.Query, 0, len(queries))
+	for _, q := range queries {
+		aq, err := analyzeQuery(ctx, binary, string(schema), string(fixture), q)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", q.Name, err)
+		}
+		out = append(out, aq)
+	}
+	return analysis.Encode(out)
+}
+
+// Check compares what SQLite reports for a case with the output the case
+// committed, returning a diff when they differ.
+func Check(ctx context.Context, dir string, c endtoend.Case) (string, error) {
+	got, err := Analyze(ctx, dir, c)
+	if err != nil {
+		return "", err
+	}
+	return c.Compare(got)
+}
+
+func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoend.Query) (analysis.Query, error) {
+	sql, phs := bind(q.SQL)
+
+	// What the library says: the catalog, the bytecode, and the statement
+	// prepared, with nothing bound to its parameters.
+	s := newScript()
+	s.sql(schema)
+	s.sql(fixture)
+	s.add(".mode json")
+	s.add(".explain off")
+	for _, cq := range catalogQueries {
+		s.section(cq.section)
+		s.sql(cq.sql)
+	}
+	s.section("explain")
+	s.sql("EXPLAIN " + sql)
+	s.section("query")
+	s.add(".stats stmt")
+	line := s.sql(sql)
+	s.section("end")
+	out, err := run(ctx, binary, s)
+	if err != nil {
+		return analysis.Query{}, err
+	}
+	if errs := out.errorsBefore(line); len(errs) > 0 {
+		return analysis.Query{}, errors.New(strings.Join(errs, "\n"))
+	}
+	query := out.sections["query"]
+	if query == nil || !query.prepared {
+		msg := strings.TrimSpace(out.stderr)
+		if msg == "" {
+			msg = "the statement was not prepared"
+		}
+		return analysis.Query{}, errors.New(msg)
+	}
+	cat, err := readCatalog(out)
+	if err != nil {
+		return analysis.Query{}, err
+	}
+	var prog []instr
+	if explain := out.sections["explain"]; explain != nil && len(explain.blocks) > 0 {
+		if err := explain.decode(0, &prog); err != nil {
+			return analysis.Query{}, fmt.Errorf("reading the bytecode: %w", err)
+		}
+	}
+	var names []string
+	for _, m := range query.columns {
+		names = append(names, m.Name)
+	}
+	t := newTracer(cat, names)
+	t.run(prog)
+	params := make([]analysis.Column, len(phs))
+	for i, ph := range phs {
+		params[i] = t.param(ph.Number)
+	}
+
+	// What the statement returns: over the fixture, with each parameter
+	// bound to a value of the column it stands in for, so that the rows
+	// come through the WHERE clause, and over no rows.
+	s = newScript()
+	s.sql(schema)
+	s.sql(fixture)
+	s.add(".parameter init")
+	for i, ph := range phs {
+		if v := sample(params[i]); v != "" {
+			s.sql(fmt.Sprintf("REPLACE INTO temp.sqlite_parameters(key, value) VALUES ('?%d', %s)", ph.Number, v))
+		}
+	}
+	s.add(".mode quote")
+	s.section("query")
+	s.sql(sql)
+	s.section("end")
+	bound, err := run(ctx, binary, s)
+	if err != nil {
+		return analysis.Query{}, err
+	}
+	s = newScript()
+	s.sql(schema)
+	s.add(".mode quote")
+	s.section("query")
+	s.sql(sql)
+	s.section("end")
+	empty, err := run(ctx, binary, s)
+	if err != nil {
+		return analysis.Query{}, err
+	}
+	var rows []string
+	for _, o := range []*output{bound, empty} {
+		if q := o.sections["query"]; q != nil {
+			rows = append(rows, q.rows...)
+		}
+	}
+
+	aq := analysis.Query{
+		Name:    q.Name,
+		Cmd:     q.Cmd,
+		Columns: []analysis.Column{},
+		Params:  []analysis.Param{},
+	}
+	for i, m := range query.columns {
+		aq.Columns = append(aq.Columns, describeColumn(cat, m, classes(rows, i)))
+	}
+	for i, ph := range phs {
+		ac := params[i]
+		if ph.Name != "" {
+			ac.Name = ph.Name
+		}
+		aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac})
+	}
+	return aq, nil
+}
+
+// sample is an expression for a value to bind to a parameter: one of its
+// column's values in the fixture, or a value of its type when all that is
+// known is the type. Empty when nothing is known.
+func sample(ac analysis.Column) string {
+	if ac.Table != "" && ac.Name != "" {
+		col, tbl := quoteIdent(ac.Name), quoteIdent(ac.Table)
+		return fmt.Sprintf("(SELECT %s FROM %s WHERE %s IS NOT NULL LIMIT 1)", col, tbl, col)
+	}
+	if ac.Type == nil {
+		return ""
+	}
+	switch ac.Type.Name {
+	case "integer", "numeric":
+		return "1"
+	case "real":
+		return "1.0"
+	case "text":
+		return "'a'"
+	case "blob":
+		return "x'00'"
+	}
+	return ""
+}
+
+func quoteIdent(s string) string {
+	return `"` + strings.ReplaceAll(s, `"`, `""`) + `"`
+}
+
+// classes lists the storage classes of a column's values across rows.
+func classes(rows []string, i int) []string {
+	var out []string
+	for _, row := range rows {
+		if c := cells(row); i < len(c) {
+			out = append(out, storageClass(c[i]))
+		}
+	}
+	return out
+}
+
+// describeColumn describes a result column: as the table column it is read
+// from when it is one, otherwise by the storage class of its values, and
+// nullable when any of its values was NULL.
+func describeColumn(cat *catalog, m columnMeta, classes []string) analysis.Column {
+	ac := analysis.Column{Name: m.Name}
+	if col := cat.lookup(m.Table, m.Origin); col != nil {
+		d := col.describe()
+		ac.Type, ac.Table = d.Type, d.Table
+	} else {
+		for _, class := range classes {
+			if class != "null" {
+				ac.Type = &analysis.TypeExpr{Name: class}
+				break
+			}
+		}
+	}
+	if ac.Type != nil {
+		for _, class := range classes {
+			if class == "null" {
+				ac.Type.Nullable = true
+			}
+		}
+	}
+	return ac
+}

internal/goldeneye/sqlite/bytecode.go

@@ -0,0 +1,456 @@
+package sqlite
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/analysis"
+)
+
+// SQLite has nothing to say about a parameter's type: a bound value is
+// whatever it is, and the column a parameter stands in for appears nowhere
+// in what the library reports about a prepared statement. What it does
+// report is the program it compiled the statement into. EXPLAIN prints the
+// bytecode, in which a parameter is a Variable loading a register, and the
+// column it stands in for is the other operand of the comparison the
+// register reaches, the row the seek it drives lands on, or the position
+// in the record that Insert writes. So each parameter is followed from the
+// register it loads, through copies and through the expressions it is an
+// argument of, to the first opcode that uses it against something the
+// catalog can name, the way the ClickHouse check follows a placeholder
+// through the query tree to the column it is compared with.
+//
+// Registers hold what the last opcode to write them wrote, and remember
+// which parameters have been loaded into them, which a later constant does
+// not clear: coalesce(?, 'x') overwrites the parameter's register with the
+// fallback on the branch where the parameter is NULL, and the comparison
+// that follows is still the parameter's. A parameter with no such use is
+// described by what the program does to it before using it, when it does
+// anything: MustBeInt says it has to be an integer, as LIMIT's does, and
+// Affinity says which affinity it is coerced to.
+
+// instr is one line of EXPLAIN output.
+type instr struct {
+	Addr   int     `json:"addr"`
+	Opcode string  `json:"opcode"`
+	P1     int     `json:"p1"`
+	P2     int     `json:"p2"`
+	P3     int     `json:"p3"`
+	P4     *string `json:"p4"`
+	P5     int     `json:"p5"`
+}
+
+func (in instr) p4() string {
+	if in.P4 == nil {
+		return ""
+	}
+	return *in.P4
+}
+
+type valueKind int
+
+const (
+	vNone     valueKind = iota
+	vParam              // a Variable
+	vColumn             // Column: a stored column of a cursor
+	vRowid              // Rowid, IdxRowid: the rowid of a cursor
+	vConstant           // a literal, with its storage class
+	vFunction           // the result of a function call, with its name
+	vExpr               // the result of an operator
+	vRecord             // MakeRecord: the registers packed into a record
+)
+
+// value is what a register holds.
+type value struct {
+	kind   valueKind
+	cursor int
+	index  int
+	class  string // vConstant
+	fn     string // vFunction
+	regs   []int  // vRecord
+}
+
+type register struct {
+	desc   value
+	params []int // every parameter loaded into the register
+}
+
+// tracer follows the parameters of one statement through its bytecode.
+type tracer struct {
+	cat   *catalog
+	names []string // the statement's result columns
+	// cursors is what each open cursor is on; ephemeral holds, for the
+	// cursors on ephemeral tables and sorters, the parameters stored in
+	// each column, so that a parameter put into an IN list's table comes
+	// back out with the Column that reads it.
+	cursors   map[int]object
+	ephemeral map[int]map[int][]int
+	regs      map[int]*register
+	// found is the column each parameter was found to stand in for, and
+	// hint what the program said about a parameter's type.
+	found map[int]analysis.Column
+	hint  map[int]string
+}
+
+func newTracer(cat *catalog, names []string) *tracer {
+	return &tracer{
+		cat:       cat,
+		names:     names,
+		cursors:   map[int]object{},
+		ephemeral: map[int]map[int][]int{},
+		regs:      map[int]*register{},
+		found:     map[int]analysis.Column{},
+		hint:      map[int]string{},
+	}
+}
+
+// run walks the program in the order it runs: the prologue Init jumps to,
+// where the Variables of a query are loaded, and then the body.
+func (t *tracer) run(prog []instr) {
+	order := prog
+	if len(prog) > 0 && prog[0].Opcode == "Init" && prog[0].P2 > 0 && prog[0].P2 < len(prog) {
+		order = append(append([]instr{}, prog[prog[0].P2:]...), prog[1:prog[0].P2]...)
+	}
+	for _, in := range order {
+		t.step(in)
+	}
+}
+
+// param describes what the parameter was found to stand in for.
+func (t *tracer) param(number int) analysis.Column {
+	if ac, ok := t.found[number]; ok {
+		return ac
+	}
+	if typ := t.hint[number]; typ != "" {
+		return analysis.Column{Type: &analysis.TypeExpr{Name: typ}}
+	}
+	return analysis.Column{}
+}
+
+func (t *tracer) reg(n int) *register {
+	r := t.regs[n]
+	if r == nil {
+		r = &register{}
+		t.regs[n] = r
+	}
+	return r
+}
+
+func (t *tracer) params(n int) []int {
+	if r := t.regs[n]; r != nil {
+		return r.params
+	}
+	return nil
+}
+
+// set records a write to a register. A Variable is the parameter and
+// nothing else; a column read replaces whatever the register held, since
+// the compiler reuses registers for the columns it reads; a constant keeps
+// the parameters, for the coalesce case above; anything else carries the
+// parameters it was computed from.
+func (t *tracer) set(n int, v value, params []int) {
+	r := t.reg(n)
+	r.desc = v
+	switch v.kind {
+	case vConstant:
+	default:
+		r.params = params
+	}
+}
+
+func (t *tracer) copy(from, to int) {
+	src := t.reg(from)
+	dst := t.reg(to)
+	dst.desc = src.desc
+	dst.params = append([]int{}, src.params...)
+}
+
+// assoc records the column the parameters in a register stand in for,
+// keeping the first found for each.
+func (t *tracer) assoc(params []int, ac analysis.Column) {
+	for _, p := range params {
+		if _, ok := t.found[p]; !ok {
+			t.found[p] = ac
+		}
+	}
+}
+
+// typeHint records what the program requires of a register's parameters.
+func (t *tracer) typeHint(n int, typ string) {
+	if typ == "" {
+		return
+	}
+	for _, p := range t.params(n) {
+		if _, ok := t.hint[p]; !ok {
+			t.hint[p] = typ
+		}
+	}
+}
+
+// describe turns what a register holds into the column a parameter
+// compared with it stands in for, when it can be named: a stored column
+// or rowid, a constant's storage class, or a function's name.
+func (t *tracer) describe(v value) (analysis.Column, bool) {
+	switch v.kind {
+	case vColumn:
+		return t.stored(v.cursor, v.index)
+	case vRowid:
+		if owner := t.cursors[v.cursor].owner(); owner != nil {
+			return owner.rowid(), true
+		}
+	case vConstant:
+		if v.class != "null" && v.class != "" {
+			return analysis.Column{Type: &analysis.TypeExpr{Name: v.class}}, true
+		}
+	case vFunction:
+		return analysis.Column{Name: v.fn}, true
+	}
+	return analysis.Column{}, false
+}
+
+// stored describes the ith stored column of a cursor.
+func (t *tracer) stored(cursor, i int) (analysis.Column, bool) {
+	col, owner, ok := t.cursors[cursor].column(i)
+	switch {
+	case !ok:
+		return analysis.Column{}, false
+	case col != nil:
+		return col.describe(), true
+	default:
+		return owner.rowid(), true
+	}
+}
+
+// rank orders what a parameter's fellow operands may be described by: a
+// column first, then a function, then a constant.
+func rank(v value) int {
+	switch v.kind {
+	case vColumn, vRowid:
+		return 0
+	case vFunction:
+		return 1
+	case vConstant:
+		if v.class != "null" && v.class != "" {
+			return 2
+		}
+	}
+	return -1
+}
+
+// expression handles an opcode computing a result from operand registers:
+// each parameter among the operands is described by the best other
+// operand, and one with no other operand to be described by is carried
+// into the result, for whatever the result is then compared with.
+func (t *tracer) expression(args []int, dest int, result value) {
+	var carried []int
+	for _, a := range args {
+		params := t.params(a)
+		if len(params) == 0 {
+			continue
+		}
+		best := -1
+		for _, b := range args {
+			if b == a {
+				continue
+			}
+			if r := rank(t.reg(b).desc); r >= 0 && (best < 0 || r < rank(t.reg(best).desc)) {
+				best = b
+			}
+		}
+		if best >= 0 {
+			if ac, ok := t.describe(t.reg(best).desc); ok {
+				t.assoc(params, ac)
+				continue
+			}
+		}
+		carried = append(carried, params...)
+	}
+	t.set(dest, result, carried)
+}
+
+// compare handles a comparison of two registers: the parameters of each
+// stand in for what the other holds.
+func (t *tracer) compare(a, b int) {
+	if ac, ok := t.describe(t.reg(b).desc); ok {
+		t.assoc(t.params(a), ac)
+	}
+	if ac, ok := t.describe(t.reg(a).desc); ok {
+		t.assoc(t.params(b), ac)
+	}
+}
+
+// store handles a register written as the ith column of a cursor's row:
+// an ephemeral table remembers the parameters for the reads to come, and
+// a real one is the column the parameters stand in for.
+func (t *tracer) store(cursor, i, reg int) {
+	params := t.params(reg)
+	if len(params) == 0 {
+		return
+	}
+	if cols, ok := t.ephemeral[cursor]; ok {
+		cols[i] = append(cols[i], params...)
+		return
+	}
+	if ac, ok := t.stored(cursor, i); ok {
+		t.assoc(params, ac)
+	}
+}
+
+// key handles registers used as a key into a cursor: n registers from
+// start, or the record in start when n is zero.
+func (t *tracer) key(cursor, start, n int) {
+	regs := t.span(start, n)
+	if n == 0 {
+		if r := t.reg(start); r.desc.kind == vRecord {
+			regs = r.desc.regs
+		}
+	}
+	for i, reg := range regs {
+		t.store(cursor, i, reg)
+	}
+}
+
+func (t *tracer) span(start, n int) []int {
+	regs := make([]int, 0, max(n, 0))
+	for i := 0; i < n; i++ {
+		regs = append(regs, start+i)
+	}
+	return regs
+}
+
+// affinity names the type an affinity character coerces to.
+func affinity(c byte) string {
+	switch c {
+	case 'A':
+		return "blob"
+	case 'B':
+		return "text"
+	case 'C':
+		return "numeric"
+	case 'D':
+		return "integer"
+	case 'E':
+		return "real"
+	}
+	return ""
+}
+
+func (t *tracer) step(in instr) {
+	switch in.Opcode {
+	case "OpenRead", "OpenWrite", "ReopenIdx":
+		t.cursors[in.P1] = t.cat.root(in.P3, in.P2)
+	case "OpenDup":
+		t.cursors[in.P1] = t.cursors[in.P2]
+		if cols, ok := t.ephemeral[in.P2]; ok {
+			t.ephemeral[in.P1] = cols
+		}
+	case "OpenEphemeral", "OpenAutoindex", "SorterOpen", "OpenPseudo":
+		t.cursors[in.P1] = object{}
+		t.ephemeral[in.P1] = map[int][]int{}
+	case "Variable":
+		t.set(in.P2, value{kind: vParam}, []int{in.P1})
+	case "Column", "VColumn":
+		t.set(in.P3, value{kind: vColumn, cursor: in.P1, index: in.P2}, t.ephemeral[in.P1][in.P2])
+	case "Rowid", "IdxRowid":
+		t.set(in.P2, value{kind: vRowid, cursor: in.P1}, nil)
+	case "Copy":
+		for i := 0; i <= in.P3; i++ {
+			t.copy(in.P1+i, in.P2+i)
+		}
+	case "SCopy", "IntCopy":
+		t.copy(in.P1, in.P2)
+	case "Null", "SoftNull":
+		end := max(in.P2, in.P3)
+		if in.Opcode == "SoftNull" {
+			end = in.P1
+			in.P2 = in.P1
+		}
+		for r := in.P2; r <= end; r++ {
+			t.set(r, value{kind: vConstant, class: "null"}, nil)
+		}
+	case "Integer", "Int64":
+		t.set(in.P2, value{kind: vConstant, class: "integer"}, nil)
+	case "Real":
+		t.set(in.P2, value{kind: vConstant, class: "real"}, nil)
+	case "String8", "String":
+		t.set(in.P2, value{kind: vConstant, class: "text"}, nil)
+	case "Blob":
+		t.set(in.P2, value{kind: vConstant, class: "blob"}, nil)
+	case "Function", "PureFunc":
+		// P4 prints the function's arity, which for a variadic function
+		// is negative and says nothing about this call. Its arguments
+		// are the registers from P2 written so far, allocated together
+		// for the call.
+		name, n := parseCall(in.p4())
+		if n < 0 {
+			n = 0
+			for t.regs[in.P2+n] != nil {
+				n++
+			}
+		}
+		t.expression(t.span(in.P2, n), in.P3, value{kind: vFunction, fn: name})
+	case "Add", "Subtract", "Multiply", "Divide", "Remainder", "Concat", "BitAnd", "BitOr", "ShiftLeft", "ShiftRight":
+		t.expression([]int{in.P2, in.P1}, in.P3, value{kind: vExpr})
+	case "MakeRecord":
+		t.set(in.P3, value{kind: vRecord, regs: t.span(in.P1, in.P2)}, nil)
+	case "Insert":
+		if r := t.reg(in.P2); r.desc.kind == vRecord {
+			for i, reg := range r.desc.regs {
+				t.store(in.P1, i, reg)
+			}
+		}
+		if owner := t.cursors[in.P1].owner(); owner != nil {
+			t.assoc(t.params(in.P3), owner.rowid())
+		}
+	case "IdxInsert", "SorterInsert":
+		n, _ := strconv.Atoi(in.p4())
+		if n > 0 {
+			t.key(in.P1, in.P3, n)
+		} else {
+			t.key(in.P1, in.P2, 0)
+		}
+	case "Eq", "Ne", "Lt", "Le", "Gt", "Ge":
+		t.compare(in.P1, in.P3)
+	case "SeekRowid", "NotExists":
+		if owner := t.cursors[in.P1].owner(); owner != nil {
+			t.assoc(t.params(in.P3), owner.rowid())
+		}
+	case "SeekGE", "SeekGT", "SeekLE", "SeekLT":
+		// On a table with a rowid the seek is by the rowid in P3; on an
+		// index it is by the P4 registers from P3.
+		if o := t.cursors[in.P1]; o.table != nil && !o.table.withoutRowid {
+			t.assoc(t.params(in.P3), o.table.rowid())
+			break
+		}
+		n, _ := strconv.Atoi(in.p4())
+		t.key(in.P1, in.P3, n)
+	case "IdxGE", "IdxGT", "IdxLE", "IdxLT", "Found", "NotFound", "NoConflict":
+		n, _ := strconv.Atoi(in.p4())
+		t.key(in.P1, in.P3, n)
+	case "MustBeInt":
+		t.typeHint(in.P1, "integer")
+	case "Affinity":
+		for i, c := range []byte(in.p4()) {
+			if i < in.P2 {
+				t.typeHint(in.P1+i, affinity(c))
+			}
+		}
+	case "ResultRow":
+		for i := 0; i < in.P2; i++ {
+			if i < len(t.names) {
+				t.assoc(t.params(in.P1+i), analysis.Column{Name: t.names[i]})
+			}
+		}
+	}
+}
+
+// parseCall splits the `name(n)` a Function's P4 prints into the function's
+// name and the number of arguments it is called with.
+func parseCall(p4 string) (string, int) {
+	open := strings.LastIndexByte(p4, '(')
+	if open < 0 || !strings.HasSuffix(p4, ")") {
+		return p4, 0
+	}
+	n, _ := strconv.Atoi(p4[open+1 : len(p4)-1])
+	return p4[:open], n
+}

internal/goldeneye/sqlite/catalog.go

@@ -0,0 +1,294 @@
+package sqlite
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/analysis"
+)
+
+// catalog is what the database says about a schema: its tables and their
+// columns, its indexes, and the b-tree each is stored in, which is how the
+// bytecode names them.
+type catalog struct {
+	tables map[string]*table
+	roots  map[int]object // by root page
+}
+
+type table struct {
+	name         string
+	withoutRowid bool
+	cols         []*tableColumn // by cid
+	// stored lists the columns in the order a row's record holds them,
+	// which is cid order less the generated columns that are not stored.
+	stored []*tableColumn
+	// pk is the PRIMARY KEY index of a WITHOUT ROWID table, whose order is
+	// the order the table's own b-tree stores a row in.
+	pk *index
+}
+
+type tableColumn struct {
+	table    *table
+	cid      int
+	name     string
+	declType string
+	notNull  bool
+	pk       int // position in the primary key, 0 for none
+}
+
+type index struct {
+	name  string
+	table *table
+	cids  []int // by seqno: -1 for the rowid, -2 for an expression
+}
+
+// object is what a cursor is open on: a table, an index, or, with neither
+// set, something the catalog does not describe — an ephemeral table, a
+// sorter, a virtual table.
+type object struct {
+	table *table
+	index *index
+}
+
+// The catalog is read with these, each in a section of its own, in json
+// mode. A query that finds nothing prints nothing, so a section may be
+// empty.
+var catalogQueries = []struct{ section, sql string }{
+	{"tables", `SELECT name, type, wr FROM pragma_table_list WHERE schema = 'main' ORDER BY name`},
+	{"columns", `SELECT t.name AS tbl, c.cid, c.name, c.type, c."notnull" AS "notnull", c.pk, c.hidden
+	 FROM pragma_table_list t, pragma_table_xinfo(t.name) c
+	 WHERE t.schema = 'main' AND t.type IN ('table', 'virtual') ORDER BY t.name, c.cid`},
+	{"indexes", `SELECT t.name AS tbl, i.name AS idx, i.origin, x.seqno, x.cid
+	 FROM pragma_table_list t, pragma_index_list(t.name) i, pragma_index_xinfo(i.name) x
+	 WHERE t.schema = 'main' ORDER BY t.name, i.name, x.seqno`},
+	{"roots", `SELECT type, name, tbl_name, rootpage FROM sqlite_schema WHERE rootpage > 0`},
+}
+
+// readCatalog builds the catalog from the sections catalogQueries printed.
+func readCatalog(out *output) (*catalog, error) {
+	var tables []struct {
+		Name string `json:"name"`
+		Type string `json:"type"`
+		WR   int    `json:"wr"`
+	}
+	var columns []struct {
+		Table   string `json:"tbl"`
+		CID     int    `json:"cid"`
+		Name    string `json:"name"`
+		Type    string `json:"type"`
+		NotNull int    `json:"notnull"`
+		PK      int    `json:"pk"`
+		Hidden  int    `json:"hidden"`
+	}
+	var indexes []struct {
+		Table  string `json:"tbl"`
+		Index  string `json:"idx"`
+		Origin string `json:"origin"`
+		Seqno  int    `json:"seqno"`
+		CID    int    `json:"cid"`
+	}
+	var roots []struct {
+		Type     string `json:"type"`
+		Name     string `json:"name"`
+		Table    string `json:"tbl_name"`
+		Rootpage int    `json:"rootpage"`
+	}
+	for name, v := range map[string]any{"tables": &tables, "columns": &columns, "indexes": &indexes, "roots": &roots} {
+		s := out.sections[name]
+		if s == nil {
+			return nil, fmt.Errorf("reading the catalog: no %s section in the output", name)
+		}
+		if len(s.blocks) > 0 {
+			if err := s.decode(0, v); err != nil {
+				return nil, fmt.Errorf("reading the catalog's %s: %w", name, err)
+			}
+		}
+	}
+
+	c := &catalog{tables: map[string]*table{}, roots: map[int]object{}}
+	for _, t := range tables {
+		if t.Type == "table" || t.Type == "virtual" {
+			c.tables[t.Name] = &table{name: t.Name, withoutRowid: t.WR == 1}
+		}
+	}
+	for _, col := range columns {
+		t := c.tables[col.Table]
+		if t == nil {
+			continue
+		}
+		tc := &tableColumn{table: t, cid: col.CID, name: col.Name, declType: col.Type, notNull: col.NotNull == 1, pk: col.PK}
+		for len(t.cols) <= col.CID {
+			t.cols = append(t.cols, nil)
+		}
+		t.cols[col.CID] = tc
+		// A generated column that is not stored — hidden is 2 — has no
+		// place in the record.
+		if col.Hidden != 2 {
+			t.stored = append(t.stored, tc)
+		}
+	}
+	byName := map[string]*index{}
+	for _, ic := range indexes {
+		t := c.tables[ic.Table]
+		if t == nil {
+			continue
+		}
+		ix := byName[ic.Index]
+		if ix == nil {
+			ix = &index{name: ic.Index, table: t}
+			byName[ic.Index] = ix
+			if ic.Origin == "pk" && t.withoutRowid {
+				t.pk = ix
+			}
+		}
+		for len(ix.cids) <= ic.Seqno {
+			ix.cids = append(ix.cids, -2)
+		}
+		ix.cids[ic.Seqno] = ic.CID
+	}
+	for _, r := range roots {
+		switch r.Type {
+		case "table":
+			if t := c.tables[r.Name]; t != nil {
+				c.roots[r.Rootpage] = object{table: t}
+			}
+		case "index":
+			if ix := byName[r.Name]; ix != nil {
+				c.roots[r.Rootpage] = object{index: ix}
+			}
+		}
+	}
+	return c, nil
+}
+
+// root returns what is stored in a b-tree of the main database.
+func (c *catalog) root(db, page int) object {
+	if db != 0 {
+		return object{}
+	}
+	return c.roots[page]
+}
+
+// lookup finds a table column by the names `.stats stmt` prints.
+func (c *catalog) lookup(tbl, col string) *tableColumn {
+	t := c.tables[tbl]
+	if t == nil {
+		return nil
+	}
+	for _, tc := range t.cols {
+		if tc != nil && tc.name == col {
+			return tc
+		}
+	}
+	return nil
+}
+
+// rowidAlias returns the column that is another name for the rowid: the
+// lone INTEGER PRIMARY KEY of a table that has a rowid.
+func (t *table) rowidAlias() *tableColumn {
+	if t.withoutRowid {
+		return nil
+	}
+	var alias *tableColumn
+	for _, tc := range t.cols {
+		if tc == nil || tc.pk == 0 {
+			continue
+		}
+		if alias != nil || !strings.EqualFold(tc.declType, "INTEGER") {
+			return nil
+		}
+		alias = tc
+	}
+	return alias
+}
+
+// describe is the column as sqlc analyze describes one: its declared type,
+// nullable unless declared NOT NULL or the rowid, which is never NULL.
+func (tc *tableColumn) describe() analysis.Column {
+	typ := parseType(tc.declType)
+	typ.Nullable = !tc.notNull && tc.table.rowidAlias() != tc
+	return analysis.Column{Name: tc.name, Type: typ, Table: tc.table.name}
+}
+
+// rowid describes the table's rowid: the column that aliases it, or the
+// rowid itself.
+func (t *table) rowid() analysis.Column {
+	if alias := t.rowidAlias(); alias != nil {
+		return alias.describe()
+	}
+	return analysis.Column{Name: "rowid", Type: &analysis.TypeExpr{Name: "integer"}, Table: t.name}
+}
+
+// column returns what the object's ith stored column is: a table column,
+// or the rowid of a table. An index stores the columns it is on and then
+// the rowid; a WITHOUT ROWID table is stored in the order of its primary
+// key; any other table is stored in column order.
+func (o object) column(i int) (*tableColumn, *table, bool) {
+	ix := o.index
+	if o.table != nil && o.table.withoutRowid {
+		ix = o.table.pk
+	}
+	switch {
+	case ix != nil:
+		if i < 0 || i >= len(ix.cids) {
+			return nil, nil, false
+		}
+		cid := ix.cids[i]
+		switch {
+		case cid == -1:
+			return nil, ix.table, true
+		case cid >= 0 && cid < len(ix.table.cols) && ix.table.cols[cid] != nil:
+			return ix.table.cols[cid], nil, true
+		}
+	case o.table != nil:
+		if i >= 0 && i < len(o.table.stored) {
+			return o.table.stored[i], nil, true
+		}
+	}
+	return nil, nil, false
+}
+
+// owner is the table the object stores rows of.
+func (o object) owner() *table {
+	if o.index != nil {
+		return o.index.table
+	}
+	return o.table
+}
+
+// parseType reads a declared type the way sqlc's catalog does: the name
+// lowercased, with whatever is in parentheses after it as arguments. A
+// column declared with no type at all can hold anything.
+func parseType(decl string) *analysis.TypeExpr {
+	decl = strings.TrimSpace(decl)
+	if decl == "" {
+		return &analysis.TypeExpr{Name: "any"}
+	}
+	name, args := decl, ""
+	if open := strings.IndexByte(decl, '('); open >= 0 && strings.HasSuffix(decl, ")") {
+		name, args = decl[:open], decl[open+1:len(decl)-1]
+	}
+	t := &analysis.TypeExpr{Name: strings.ToLower(strings.TrimSpace(name))}
+	if strings.TrimSpace(args) == "" {
+		return t
+	}
+	for _, a := range strings.Split(args, ",") {
+		a = strings.TrimSpace(a)
+		switch {
+		case strings.HasPrefix(a, "'") && strings.HasSuffix(a, "'") && len(a) >= 2:
+			s := strings.ReplaceAll(a[1:len(a)-1], "''", "'")
+			t.Args = append(t.Args, analysis.TypeArg{String: &s})
+		case strings.EqualFold(a, "true") || strings.EqualFold(a, "false"):
+			b := strings.EqualFold(a, "true")
+			t.Args = append(t.Args, analysis.TypeArg{Bool: &b})
+		default:
+			if n, err := strconv.ParseInt(a, 10, 64); err == nil {
+				t.Args = append(t.Args, analysis.TypeArg{Int: &n})
+			} else {
+				t.Args = append(t.Args, analysis.TypeArg{Type: &analysis.TypeExpr{Name: strings.ToLower(a)}})
+			}
+		}
+	}
+	return t
+}

internal/goldeneye/sqlite/install.go

@@ -0,0 +1,347 @@
+package sqlite
+
+import (
+	"archive/zip"
+	"context"
+	"crypto/sha3"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"slices"
+	"strings"
+	"sync"
+)
+
+// DefaultVersion is the SQLite release the dialect is generated from. It is
+// the release the ncruces/go-sqlite3 driver in the main module embeds, so
+// the functions the dialect knows are the ones the tests run against.
+// Bumping it is a deliberate change: releases add functions and overloads,
+// so regenerate and review the dialect after changing it, and add the new
+// release's amalgamation to the table below.
+const DefaultVersion = "3.53.4"
+
+// defaultOptions are the compile options the base dialect is built with:
+// the ones sqlite.org's own configure turns on by default, and so the ones
+// a stock sqlite3 has. JSON is part of the library unless omitted, so only
+// the math functions need naming.
+var defaultOptions = []string{"SQLITE_ENABLE_MATH_FUNCTIONS"}
+
+// extensions are the compile options that each become a directory under
+// the dialect's extensions/, holding the functions a build with the option
+// adds over the default build — the way each PostgreSQL contrib extension
+// holds what CREATE EXTENSION adds. Each is named after its option as
+// pragma compile_options spells it, in lower case, and lists the options
+// its build needs: GEOPOLY lives inside the RTREE module, so enabling it
+// alone adds nothing. Options that add virtual tables but no functions,
+// such as SESSION and DBSTAT, are not listed, since the dialect has nothing
+// to say about them, and ENABLE_FTS4 is the same module as ENABLE_FTS3.
+var extensions = []build{
+	{"soundex", []string{"SQLITE_SOUNDEX"}},
+	{"enable_fts3", []string{"SQLITE_ENABLE_FTS3"}},
+	{"enable_fts5", []string{"SQLITE_ENABLE_FTS5"}},
+	{"enable_geopoly", []string{"SQLITE_ENABLE_RTREE", "SQLITE_ENABLE_GEOPOLY"}},
+	{"enable_offset_sql_func", []string{"SQLITE_ENABLE_OFFSET_SQL_FUNC"}},
+	{"enable_percentile", []string{"SQLITE_ENABLE_PERCENTILE"}},
+	{"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}},
+}
+
+// analysisShell is the shell the analyze cases run through, which is no
+// dialect build: nothing is generated from it. It is built with column
+// metadata, so that `.stats stmt` can say which table column each result
+// column of a statement comes from, and with every extension option at
+// once, so that whatever a case's schema asks for is there.
+var analysisShell = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)}
+
+// extensionOptions is every option an extension build turns on, once each.
+func extensionOptions() []string {
+	var opts []string
+	for _, b := range extensions {
+		for _, opt := range b.options {
+			if !slices.Contains(opts, opt) {
+				opts = append(opts, opt)
+			}
+		}
+	}
+	return opts
+}
+
+// asset is one downloadable amalgamation of SQLite: the zip published on
+// sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c.
+type asset struct {
+	Version string
+	// Path is the download's address under https://sqlite.org/, the
+	// release year included, as the download page's index lists it.
+	Path string
+	// SHA3 is the SHA3-256 of the zip, as the download page lists it.
+	SHA3 string
+}
+
+// assets lists every amalgamation Install knows how to fetch, with the
+// SHA3-256 of the download. A version that is not in this table cannot be
+// installed: verifying the download is the point of the table. The
+// checksums are the ones in the index at the foot of
+// https://sqlite.org/download.html.
+var assets = []asset{
+	{"3.53.4", "2026/sqlite-amalgamation-3530400.zip", "628a44cfe82c66aed1ccbbe85a562d2e33ebe64b3288981ed76285612227934e"},
+}
+
+// sources are the files taken out of the amalgamation.
+var sources = []string{"sqlite3.c", "sqlite3.h", "shell.c"}
+
+func releaseAsset(version string) (asset, error) {
+	for _, a := range assets {
+		if a.Version == version {
+			return a, nil
+		}
+	}
+	return asset{}, fmt.Errorf("SQLite %s is not in the asset table; add its amalgamation and checksum to install.go", version)
+}
+
+func (a asset) url() string {
+	return "https://sqlite.org/" + a.Path
+}
+
+// cacheDir is where Install puts a version: the sources under src/, and one
+// shell per build under default/, under each option's extension name and
+// under analysis/.
+func cacheDir(version string) (string, error) {
+	dir, err := os.UserCacheDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(dir, "sqlc-sqlite", version), nil
+}
+
+// build is one compiled shell: its name, which is the directory it is
+// under in the version's cache directory and, for an extension, under the
+// dialect's extensions/, and the options it is built with beyond the
+// default ones.
+type build struct {
+	name    string
+	options []string
+}
+
+// builds lists the dialect builds: the default build first, then one per
+// extension.
+func builds() []build {
+	return append([]build{{"default", nil}}, extensions...)
+}
+
+// shells lists every build Install makes: the dialect builds and the
+// analysis shell.
+func shells() []build {
+	return append(builds(), analysisShell)
+}
+
+// flags are every option a build is compiled with.
+func (b build) flags() []string {
+	return append(append([]string{}, defaultOptions...), b.options...)
+}
+
+func (b build) binary(dir string) string {
+	return filepath.Join(dir, b.name, "sqlite3")
+}
+
+// Locate finds the directory of shells Install made for DefaultVersion.
+func Locate() (string, error) {
+	dir, err := cacheDir(DefaultVersion)
+	if err != nil {
+		return "", err
+	}
+	for _, b := range shells() {
+		if _, err := os.Stat(b.binary(dir)); err != nil {
+			return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " "))
+		}
+	}
+	return dir, nil
+}
+
+// Install downloads the amalgamation for a version into the cache and
+// compiles the shell from it once per build, returning the directory the
+// shells are under. Builds already there are kept, so adding an option
+// compiles only its shell. The download is checked against the table's
+// SHA3-256 before it is unpacked. Compiling takes the compiler CC names, or
+// cc, and a few seconds per build without optimisation, which a shell that
+// only reads catalogs does not need.
+//
+// goos and goarch are what every installer is handed; the sources build the
+// same everywhere a C compiler is, but no default compiler or link line is
+// known for Windows.
+func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) {
+	if goos == "windows" {
+		return "", errors.New("building SQLite from source is not supported on Windows")
+	}
+	dir, err := cacheDir(version)
+	if err != nil {
+		return "", err
+	}
+	var missing []build
+	for _, b := range shells() {
+		if _, err := os.Stat(b.binary(dir)); err != nil {
+			missing = append(missing, b)
+		}
+	}
+	if len(missing) == 0 {
+		return dir, nil
+	}
+	if err := download(ctx, version, dir, progress); err != nil {
+		return "", err
+	}
+	if err := compile(ctx, dir, missing, progress); err != nil {
+		return "", err
+	}
+	return dir, nil
+}
+
+// download fetches the amalgamation and unpacks the sources into src/,
+// unless they are already there.
+func download(ctx context.Context, version, dir string, progress io.Writer) error {
+	src := filepath.Join(dir, "src")
+	have := true
+	for _, name := range sources {
+		if _, err := os.Stat(filepath.Join(src, name)); err != nil {
+			have = false
+		}
+	}
+	if have {
+		return nil
+	}
+	a, err := releaseAsset(version)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(src, 0o755); err != nil {
+		return err
+	}
+
+	url := a.url()
+	fmt.Fprintf(progress, "downloading %s\n", url)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("downloading %s: %s", url, resp.Status)
+	}
+
+	// The zip has to land on disk before it can be read — its directory is
+	// at the end — so download it whole and hash every byte on the way.
+	archive, err := os.CreateTemp(dir, "sqlite-amalgamation-*.partial")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(archive.Name())
+	sum := sha3.New256()
+	if _, err := io.Copy(io.MultiWriter(archive, sum), resp.Body); err != nil {
+		archive.Close()
+		return err
+	}
+	if err := archive.Close(); err != nil {
+		return err
+	}
+	if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA3 {
+		return fmt.Errorf("downloading %s: SHA3-256 mismatch: got %s, want %s", url, got, a.SHA3)
+	}
+
+	zr, err := zip.OpenReader(archive.Name())
+	if err != nil {
+		return err
+	}
+	defer zr.Close()
+	for _, name := range sources {
+		if err := extract(zr, name, filepath.Join(src, name)); err != nil {
+			return fmt.Errorf("downloading %s: %w", url, err)
+		}
+	}
+	return nil
+}
+
+// extract copies the named file out of the zip. The amalgamation's files
+// sit in one directory named after the release, so the name is matched on
+// its base.
+func extract(zr *zip.ReadCloser, name, dest string) error {
+	for _, f := range zr.File {
+		if filepath.Base(f.Name) != name || f.FileInfo().IsDir() {
+			continue
+		}
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer rc.Close()
+		tmp, err := os.CreateTemp(filepath.Dir(dest), name+"-*.partial")
+		if err != nil {
+			return err
+		}
+		defer os.Remove(tmp.Name())
+		if _, err := io.Copy(tmp, rc); err != nil {
+			tmp.Close()
+			return err
+		}
+		if err := tmp.Close(); err != nil {
+			return err
+		}
+		return os.Rename(tmp.Name(), dest)
+	}
+	return fmt.Errorf("zip does not contain %s", name)
+}
+
+// compile builds the shells, as many at a time as there are CPUs.
+func compile(ctx context.Context, dir string, todo []build, progress io.Writer) error {
+	cc := os.Getenv("CC")
+	if cc == "" {
+		cc = "cc"
+	}
+	if _, err := exec.LookPath(cc); err != nil {
+		return fmt.Errorf("no C compiler found: put cc on PATH or set CC to one")
+	}
+	var wg sync.WaitGroup
+	slots := make(chan struct{}, max(1, runtime.NumCPU()))
+	errs := make([]error, len(todo))
+	for i, b := range todo {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			slots <- struct{}{}
+			defer func() { <-slots }()
+			errs[i] = b.compile(ctx, cc, dir, progress)
+		}()
+	}
+	wg.Wait()
+	return errors.Join(errs...)
+}
+
+// compile builds one shell from the sources in src/, writing it beside its
+// destination and renaming so a failed build never masquerades as one.
+func (b build) compile(ctx context.Context, cc, dir string, progress io.Writer) error {
+	dest := b.binary(dir)
+	if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
+		return err
+	}
+	tmp := dest + ".partial"
+	defer os.Remove(tmp)
+	args := []string{"-O0"}
+	for _, opt := range b.flags() {
+		args = append(args, "-D"+opt)
+	}
+	src := filepath.Join(dir, "src")
+	args = append(args, filepath.Join(src, "shell.c"), filepath.Join(src, "sqlite3.c"), "-o", tmp, "-lm", "-ldl", "-lpthread")
+	fmt.Fprintf(progress, "building %s: %s %s\n", b.name, cc, strings.Join(args, " "))
+	cmd := exec.CommandContext(ctx, cc, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("building %s: %w\n%s", b.name, err, strings.TrimSpace(string(out)))
+	}
+	return os.Rename(tmp, dest)
+}

internal/goldeneye/sqlite/shell.go

@@ -0,0 +1,254 @@
+package sqlite
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// The analysis shell is driven with a script on its standard input, made
+// of SQL and of the shell's own dot-commands: `.print` labels the sections
+// of the output so that each statement's results can be found again,
+// `.mode` chooses how rows print — json for the catalog and the bytecode,
+// quote for a query's own rows, since quote mode is the one that tells a
+// blob from a string and a real from an integer — and `.stats stmt` makes
+// the shell print what it knows about each result column of the statement
+// it has just run, which with column metadata compiled in is the column's
+// name, declared type, database, table and origin column.
+
+// script is a shell script under construction, counting its lines so that
+// an error the shell reports "near line N" can be placed.
+type script struct {
+	buf  strings.Builder
+	line int // the number of the next line written
+}
+
+func newScript() *script {
+	return &script{line: 1}
+}
+
+// add writes text on lines of its own, and returns the line it starts on.
+func (s *script) add(text string) int {
+	start := s.line
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return start
+	}
+	s.buf.WriteString(text)
+	s.buf.WriteByte('\n')
+	s.line += strings.Count(text, "\n") + 1
+	return start
+}
+
+// sql writes statements, ending them with a semicolon when they lack one.
+func (s *script) sql(text string) int {
+	text = strings.TrimRight(strings.TrimSpace(text), "; \t\r\n")
+	if text == "" {
+		return s.line
+	}
+	return s.add(text + ";")
+}
+
+// section labels what follows, up to the next label.
+func (s *script) section(name string) {
+	s.add(".print @@" + name)
+}
+
+// columnMeta is what `.stats stmt` prints about one result column. Table
+// and Origin name the table column a result column is read from, and are
+// empty for an expression.
+type columnMeta struct {
+	Name     string
+	DeclType string
+	Database string
+	Table    string
+	Origin   string
+}
+
+// section is the output between two labels.
+type section struct {
+	// blocks holds every JSON result set printed in the section.
+	blocks []json.RawMessage
+	// rows holds every other line printed before the statement's
+	// statistics: in quote mode, one row each.
+	rows []string
+	// prepared says a statement's statistics were printed, which the shell
+	// does only for a statement it could prepare.
+	prepared bool
+	columns  []columnMeta
+}
+
+// decode reads the section's nth JSON result set.
+func (s *section) decode(n int, v any) error {
+	if n >= len(s.blocks) {
+		return fmt.Errorf("expected at least %d result set(s), got %d", n+1, len(s.blocks))
+	}
+	return json.Unmarshal(s.blocks[n], v)
+}
+
+// output is what one run of the shell printed, by section.
+type output struct {
+	sections map[string]*section
+	stderr   string
+}
+
+var errLineRe = regexp.MustCompile(`near line (\d+):`)
+
+// errorsBefore returns the errors the shell reported for statements before
+// the given line: the ones that are not the statement under analysis
+// failing to run, which it may, with nothing bound to its parameters.
+func (o *output) errorsBefore(line int) []string {
+	var errs []string
+	for _, l := range strings.Split(o.stderr, "\n") {
+		if m := errLineRe.FindStringSubmatch(l); m != nil {
+			if n, _ := strconv.Atoi(m[1]); n < line {
+				errs = append(errs, strings.TrimSpace(l))
+			}
+		}
+	}
+	return errs
+}
+
+// run feeds a script to the shell over a fresh in-memory database and
+// parses what it printed. A statement that fails does not stop the shell,
+// so a failure is read from the output: a section whose statement has no
+// statistics was not prepared, and stderr says why.
+func run(ctx context.Context, binary string, s *script) (*output, error) {
+	cmd := exec.CommandContext(ctx, binary, ":memory:")
+	cmd.Stdin = strings.NewReader(s.buf.String())
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		var exit *exec.ExitError
+		if !errors.As(err, &exit) {
+			return nil, fmt.Errorf("sqlite3: %w", err)
+		}
+	}
+	out := &output{sections: map[string]*section{}, stderr: stderr.String()}
+	var (
+		cur   *section
+		block []string // lines of the JSON result set being read
+		row   []string // lines of the quote-mode row being read
+	)
+	for _, line := range strings.Split(stdout.String(), "\n") {
+		switch {
+		case block != nil:
+			block = append(block, line)
+			if strings.HasSuffix(line, "]") {
+				cur.blocks = append(cur.blocks, json.RawMessage(strings.Join(block, "\n")))
+				block = nil
+			}
+		case row != nil:
+			row = append(row, line)
+			if joined := strings.Join(row, "\n"); strings.Count(joined, "'")%2 == 0 {
+				cur.rows = append(cur.rows, joined)
+				row = nil
+			}
+		case strings.HasPrefix(line, "@@"):
+			cur = &section{}
+			out.sections[line[2:]] = cur
+		case cur == nil || line == "":
+		case strings.HasPrefix(line, "["):
+			if strings.HasSuffix(line, "]") {
+				cur.blocks = append(cur.blocks, json.RawMessage(line))
+			} else {
+				block = []string{line}
+			}
+		case strings.HasPrefix(line, "Number of output columns:"):
+			cur.prepared = true
+		case cur.prepared:
+			if i, field, value, ok := statLine(line); ok {
+				for len(cur.columns) <= i {
+					cur.columns = append(cur.columns, columnMeta{})
+				}
+				c := &cur.columns[i]
+				switch field {
+				case "name":
+					c.Name = value
+				case "declared type":
+					c.DeclType = value
+				case "database name":
+					c.Database = value
+				case "table name":
+					c.Table = value
+				case "origin name":
+					c.Origin = value
+				}
+			}
+		default:
+			// A quote-mode row, complete unless a string in it spans lines.
+			if strings.Count(line, "'")%2 == 0 {
+				cur.rows = append(cur.rows, line)
+			} else {
+				row = []string{line}
+			}
+		}
+	}
+	return out, nil
+}
+
+var statLineRe = regexp.MustCompile(`^Column (\d+) ([a-z ]+):`)
+
+// statLine reads one line of `.stats stmt` column output, which the shell
+// prints as the label padded to 36 columns, a space and the value; a value
+// the library has no answer for prints as the C library prints a null
+// string.
+func statLine(line string) (int, string, string, bool) {
+	m := statLineRe.FindStringSubmatch(line)
+	if m == nil {
+		return 0, "", "", false
+	}
+	i, _ := strconv.Atoi(m[1])
+	label := m[0]
+	value := ""
+	if start := max(len(label), 36) + 1; start < len(line) {
+		value = line[start:]
+	}
+	if value == "(null)" {
+		value = ""
+	}
+	return i, m[2], value, true
+}
+
+// cells splits a quote-mode row into its values as printed.
+func cells(row string) []string {
+	var out []string
+	start := 0
+	quoted := false
+	for i := 0; i < len(row); i++ {
+		switch {
+		case row[i] == '\'':
+			quoted = !quoted
+		case row[i] == ',' && !quoted:
+			out = append(out, row[start:i])
+			start = i + 1
+		}
+	}
+	return append(out, row[start:])
+}
+
+// storageClass reads the storage class of a value as quote mode prints
+// it: a string is quoted, a blob is quoted with an X, NULL is spelled
+// out, and a real has a point, an exponent or is infinite or not a number
+// where an integer is digits.
+func storageClass(cell string) string {
+	switch {
+	case cell == "NULL":
+		return "null"
+	case strings.HasPrefix(cell, "'"):
+		return "text"
+	case strings.HasPrefix(cell, "x'") || strings.HasPrefix(cell, "X'"):
+		return "blob"
+	case strings.ContainsAny(cell, ".eEIN"):
+		return "real"
+	default:
+		return "integer"
+	}
+}

internal/goldeneye/sqlite/signatures.go

@@ -0,0 +1,136 @@
+package sqlite
+
+import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
+
+// signature is what a function returns and what its arguments hold, read
+// from the amalgamation by source.signature. Args types the leading
+// arguments in order; a position it does not cover is "any", which the
+// analyzer resolves to the argument's own type. Variadic types the
+// arguments an overload of variable arity repeats, "any" when empty. How
+// many arguments an overload takes, and how many of them it requires,
+// comes from the shell. Nullable is decided afterwards: for an aggregate by
+// running it over no rows, for a scalar by the nullable list below.
+// Numeric says the function returns an integer for an integer argument
+// and a real for a real one, and Returns what it returns for anything
+// else.
+type signature struct {
+	Args     []string
+	Variadic string
+	Returns  string
+	Numeric  bool
+	Nullable bool
+}
+
+// args builds an overload's parameters from the shell's argument count. A
+// fixed count is that many parameters; a negative count is the required
+// leading parameters, any further ones the signature types marked as
+// having a default, and a variadic tail.
+func (s signature) args(narg int) []dialect.Arg {
+	if narg >= 0 {
+		args := make([]dialect.Arg, narg)
+		for i := range args {
+			args[i] = dialect.Arg{Type: s.argType(i)}
+		}
+		return args
+	}
+	required := minArgs(narg)
+	n := max(required, len(s.Args))
+	args := make([]dialect.Arg, 0, n+1)
+	for i := 0; i < n; i++ {
+		args = append(args, dialect.Arg{Type: s.argType(i), HasDefault: i >= required})
+	}
+	tail := s.Variadic
+	if tail == "" {
+		tail = "any"
+	}
+	return append(args, dialect.Arg{Type: tail, Mode: "v"})
+}
+
+func (s signature) argType(i int) string {
+	if i < len(s.Args) {
+		return s.Args[i]
+	}
+	return "any"
+}
+
+// inlineReturns is what the functions the VDBE implements in bytecode
+// return, by the INLINEFUNC_* constant their registration carries. All but
+// one hand back one of their arguments, which is what the default of "any"
+// says; sqlite_offset is a byte offset.
+var inlineReturns = map[string]string{
+	"INLINEFUNC_sqlite_offset": "integer",
+}
+
+// realResults are the functions that return an integer or a real on
+// something other than which their argument is, and so are typed real
+// alone: unixepoch takes a time string and returns a real only with the
+// 'subsec' modifier.
+var realResults = map[string]bool{
+	"unixepoch": true,
+}
+
+// omitted are functions the dialect leaves out: ones that exist for their
+// side effect and return nothing a query can use, and ones an extension
+// uses to pass pointers to itself.
+var omitted = map[string]bool{
+	// Loads a shared library and returns NULL.
+	"load_extension": true,
+	// Writes to the error log and returns NULL.
+	"sqlite_log": true,
+	// FTS3's and FTS5's ways of passing pointers to their virtual tables,
+	// not functions a query calls.
+	"fts3_tokenizer": true,
+	"fts5":           true,
+	// A debugging aid of GEOPOLY's.
+	"geopoly_debug": true,
+}
+
+// nullable are the scalar functions that return NULL for arguments that
+// are not: a lookup that finds nothing, an input that does not parse, a
+// value with no sign. The source cannot say this — a SQLite function
+// returns NULL as often by setting no result as by calling
+// sqlite3_result_null — so it is listed by the documentation of each.
+// Aggregates are not listed: whether one returns NULL over no rows is
+// found by running it over none.
+var nullable = map[string]bool{
+	// Core functions.
+	"nullif":                   true,
+	"sign":                     true,
+	"sqlite_compileoption_get": true,
+	"sqlite_offset":            true,
+	"unhex":                    true,
+	// Date and time functions, for a time value they cannot parse.
+	"date":      true,
+	"datetime":  true,
+	"julianday": true,
+	"strftime":  true,
+	"time":      true,
+	"timediff":  true,
+	"unixepoch": true,
+	// JSON functions, for a path that leads nowhere.
+	"->":                true,
+	"->>":               true,
+	"json_array_length": true,
+	"json_extract":      true,
+	"json_type":         true,
+	"jsonb_extract":     true,
+	// Window functions, for a row outside the frame.
+	"first_value": true,
+	"lag":         true,
+	"last_value":  true,
+	"lead":        true,
+	"nth_value":   true,
+	// FTS5, for a table with no locale.
+	"fts5_get_locale": true,
+	// GEOPOLY functions, for an argument that is not a polygon.
+	"geopoly_area":           true,
+	"geopoly_bbox":           true,
+	"geopoly_blob":           true,
+	"geopoly_ccw":            true,
+	"geopoly_contains_point": true,
+	"geopoly_json":           true,
+	"geopoly_overlap":        true,
+	"geopoly_svg":            true,
+	"geopoly_within":         true,
+	"geopoly_xform":          true,
+}

internal/goldeneye/sqlite/source.go

@@ -0,0 +1,521 @@
+package sqlite
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// source is what the amalgamation says about how SQLite's functions answer.
+// A function is registered in one of a few shapes — the FuncDef macros of
+// the built-in tables, a sqlite3_create_function call, or a struct table an
+// extension walks — each naming the C functions that implement it. Those
+// implementations set their result through sqlite3_result_* and read their
+// arguments through sqlite3_value_*, which is as close as SQLite comes to
+// declaring a signature.
+type source struct {
+	funcs   map[string]cfunc
+	aliases map[string]string
+	regs    map[string]*registration
+}
+
+// cfunc is one C function definition: its parameter list and its body.
+type cfunc struct {
+	params string
+	body   string
+}
+
+// registration is the union of everything the source registers under one
+// SQL function name.
+type registration struct {
+	// scalar implements a scalar overload: its results and its arguments.
+	scalar []string
+	// step and final implement an aggregate or window overload: step reads
+	// the arguments, final and value set the result.
+	step  []string
+	final []string
+	// inline names the INLINEFUNC_* constant of a function the VDBE
+	// implements in bytecode, which has no C body to read.
+	inline string
+	// json records what a JSON function's registration says it returns,
+	// "text" or "blob", since the json_ and jsonb_ forms share an
+	// implementation; jsonAlways says the registration promises JSON text
+	// whatever the implementation might otherwise return, as -> does.
+	json       string
+	jsonAlways bool
+	// table marks a registration found only in a struct table, which is
+	// consulted when nothing more direct registered the name.
+	table bool
+}
+
+// The FuncDef macros, with the positions of the C functions in each.
+var macroEntry = regexp.MustCompile(`\b(FUNCTION2|FUNCTION|VFUNCTION|SFUNCTION|MFUNCTION|JFUNCTION|INLINE_FUNC|DFUNCTION|PURE_DATE|STR_FUNCTION|LIKEFUNC|WAGGREGATE|WINDOWFUNCX|WINDOWFUNCALL|WINDOWFUNCNOOP)\(`)
+
+var (
+	createFunction = regexp.MustCompile(`\bsqlite3_create_(window_)?function\(`)
+	defineAlias    = regexp.MustCompile(`(?m)^#define\s+(\w+)\s+(\w+)\s*$`)
+	definition     = regexp.MustCompile(`(?m)^(?:static\s+|SQLITE_PRIVATE\s+)?(?:const\s+)?(?:unsigned\s+)?[A-Za-z_]\w*(?:\s*\*+\s*|\s+)([A-Za-z_]\w*)\(`)
+	initializer    = regexp.MustCompile(`\{[^{}]*"([a-z_0-9>-]+)"[^{}]*\}`)
+	identifier     = regexp.MustCompile(`\b([A-Za-z_]\w*)\b`)
+	resultCall     = regexp.MustCompile(`\bsqlite3_result_(\w+)\(`)
+	valueCall      = regexp.MustCompile(`\bsqlite3_value_(\w+)\(\s*(?:argv|apVal|apArg)\[(\w+)\]`)
+	callee         = regexp.MustCompile(`\b([A-Za-z_]\w*)\s*\(`)
+)
+
+// readSource reads the amalgamation.
+func readSource(path string) (*source, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	text := string(data)
+	s := &source{
+		funcs:   map[string]cfunc{},
+		aliases: map[string]string{},
+		regs:    map[string]*registration{},
+	}
+	for _, m := range defineAlias.FindAllStringSubmatch(text, -1) {
+		s.aliases[m[1]] = m[2]
+	}
+	s.readDefinitions(text)
+	s.readMacros(text)
+	s.readCreateFunctions(text)
+	s.readTables(text)
+	return s, nil
+}
+
+// readDefinitions indexes every function defined at the left margin.
+func (s *source) readDefinitions(text string) {
+	for _, m := range definition.FindAllStringSubmatchIndex(text, -1) {
+		name := text[m[2]:m[3]]
+		params, end, ok := balanced(text, m[1]-1, '(', ')')
+		if !ok {
+			continue
+		}
+		i := end
+		for i < len(text) && (text[i] == ' ' || text[i] == '\n' || text[i] == '\r' || text[i] == '\t') {
+			i++
+		}
+		if i >= len(text) || text[i] != '{' {
+			continue
+		}
+		body, _, ok := balanced(text, i, '{', '}')
+		if !ok {
+			continue
+		}
+		if _, dup := s.funcs[name]; !dup {
+			s.funcs[name] = cfunc{params: params, body: body}
+		}
+	}
+}
+
+// balanced returns the text between the bracket at open and its match,
+// skipping string and character literals and comments, and the index after
+// the closing bracket.
+func balanced(text string, open int, lb, rb byte) (string, int, bool) {
+	depth := 0
+	for i := open; i < len(text); i++ {
+		switch c := text[i]; {
+		case c == '"' || c == '\'':
+			for i++; i < len(text) && text[i] != c; i++ {
+				if text[i] == '\\' {
+					i++
+				}
+			}
+		case c == '/' && i+1 < len(text) && text[i+1] == '*':
+			end := strings.Index(text[i+2:], "*/")
+			if end < 0 {
+				return "", 0, false
+			}
+			i += end + 3
+		case c == '/' && i+1 < len(text) && text[i+1] == '/':
+			end := strings.IndexByte(text[i:], '\n')
+			if end < 0 {
+				return "", 0, false
+			}
+			i += end
+		case c == lb:
+			depth++
+		case c == rb:
+			depth--
+			if depth == 0 {
+				return text[open+1 : i], i + 1, true
+			}
+		}
+	}
+	return "", 0, false
+}
+
+// arguments splits a bracketed argument list at the commas of its own
+// level, and reports the index after the closing bracket.
+func arguments(text string, open int) ([]string, int, bool) {
+	inner, end, ok := balanced(text, open, '(', ')')
+	if !ok {
+		return nil, 0, false
+	}
+	var args []string
+	depth, start := 0, 0
+	for i := 0; i < len(inner); i++ {
+		switch inner[i] {
+		case '(', '[', '{':
+			depth++
+		case ')', ']', '}':
+			depth--
+		case '"':
+			for i++; i < len(inner) && inner[i] != '"'; i++ {
+				if inner[i] == '\\' {
+					i++
+				}
+			}
+		case ',':
+			if depth == 0 {
+				args = append(args, strings.TrimSpace(inner[start:i]))
+				start = i + 1
+			}
+		}
+	}
+	args = append(args, strings.TrimSpace(inner[start:]))
+	return args, end, true
+}
+
+func (s *source) reg(name string) *registration {
+	name = strings.ToLower(name)
+	r, ok := s.regs[name]
+	if !ok {
+		r = &registration{}
+		s.regs[name] = r
+	}
+	return r
+}
+
+// readMacros reads the FuncDef tables. The macro definitions themselves
+// match too, and are told apart by their formal parameter names.
+func (s *source) readMacros(text string) {
+	for _, m := range macroEntry.FindAllStringSubmatchIndex(text, -1) {
+		macro := text[m[2]:m[3]]
+		args, _, ok := arguments(text, m[1]-1)
+		if !ok || len(args) < 2 || args[0] == "zName" || args[0] == "name" {
+			continue
+		}
+		name := args[0]
+		switch macro {
+		case "FUNCTION", "FUNCTION2", "VFUNCTION", "SFUNCTION", "DFUNCTION", "PURE_DATE", "STR_FUNCTION":
+			if len(args) > 4 {
+				r := s.reg(name)
+				r.scalar = append(r.scalar, args[4])
+			}
+		case "MFUNCTION":
+			if len(args) > 3 {
+				r := s.reg(name)
+				r.scalar = append(r.scalar, args[3])
+			}
+		case "JFUNCTION":
+			// JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc)
+			if len(args) > 7 {
+				r := s.reg(name)
+				r.scalar = append(r.scalar, args[7])
+				r.json = "text"
+				if args[5] == "1" {
+					r.json = "blob"
+				}
+				r.jsonAlways = strings.Contains(args[6], "JSON_JSON")
+			}
+		case "INLINE_FUNC":
+			if len(args) > 2 {
+				s.reg(name).inline = args[2]
+			}
+		case "LIKEFUNC":
+			r := s.reg(name)
+			r.scalar = append(r.scalar, "likeFunc")
+		case "WAGGREGATE":
+			// WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f)
+			// The JSON aggregates use it too, told by their implementation,
+			// with JSON_BLOB as the user data of the jsonb_ forms.
+			if len(args) > 6 {
+				r := s.reg(name)
+				r.step = append(r.step, args[4])
+				r.final = append(r.final, args[5], args[6])
+				if strings.HasPrefix(args[4], "json") {
+					r.json = "text"
+					if strings.Contains(args[2], "JSON_BLOB") {
+						r.json = "blob"
+					}
+				}
+			}
+		case "WINDOWFUNCX", "WINDOWFUNCALL":
+			r := s.reg(name)
+			r.step = append(r.step, name+"StepFunc")
+			r.final = append(r.final, name+"ValueFunc")
+		case "WINDOWFUNCNOOP":
+			s.reg(name).inline = "bytecode"
+		}
+	}
+}
+
+// readCreateFunctions reads the functions extensions register by calling
+// sqlite3_create_function or sqlite3_create_window_function with a literal
+// name.
+func (s *source) readCreateFunctions(text string) {
+	for _, m := range createFunction.FindAllStringSubmatchIndex(text, -1) {
+		window := m[2] != -1
+		args, _, ok := arguments(text, m[1]-1)
+		if !ok || len(args) < 8 || !strings.HasPrefix(args[1], `"`) {
+			continue
+		}
+		r := s.reg(strings.Trim(args[1], `"`))
+		if window {
+			// (db, zName, nArg, eTextRep, pApp, xStep, xFinal, xValue, xInverse, xDestroy)
+			r.step = given(r.step, args[5])
+			r.final = given(r.final, args[6], args[7])
+		} else {
+			// (db, zName, nArg, eTextRep, pApp, xFunc, xStep, xFinal)
+			r.scalar = given(r.scalar, args[5])
+			r.step = given(r.step, args[6])
+			r.final = given(r.final, args[7])
+		}
+	}
+}
+
+// given appends the implementations a registration names, leaving out the
+// methods it passes as 0 or NULL.
+func given(impls []string, names ...string) []string {
+	for _, name := range names {
+		if name != "0" && name != "NULL" {
+			impls = append(impls, name)
+		}
+	}
+	return impls
+}
+
+// readTables reads the struct tables extensions walk to register their
+// functions — geopoly's aFunc, FTS5's aBuiltin, FTS3's aOverload — each row
+// an initializer holding the function's name and the C functions that
+// implement it. Any C function in the row counts as an implementation;
+// which reads arguments and which sets the result comes out in the
+// reading. A row is consulted only for a name nothing else registered.
+func (s *source) readTables(text string) {
+	for _, m := range initializer.FindAllStringSubmatch(text, -1) {
+		var impls []string
+		for _, id := range identifier.FindAllStringSubmatch(m[0], -1) {
+			if _, ok := s.funcs[s.resolve(id[1])]; ok {
+				impls = append(impls, id[1])
+			}
+		}
+		if len(impls) == 0 {
+			continue
+		}
+		name := strings.ToLower(m[1])
+		r, ok := s.regs[name]
+		if ok && !r.table {
+			continue
+		}
+		r = s.reg(name)
+		r.table = true
+		r.scalar = append(r.scalar, impls...)
+	}
+}
+
+// resolve follows #define aliases from a C function name to the one that
+// is defined.
+func (s *source) resolve(name string) string {
+	for i := 0; i < 8; i++ {
+		alias, ok := s.aliases[name]
+		if !ok {
+			return name
+		}
+		name = alias
+	}
+	return name
+}
+
+// noop reports a C function that does nothing: the stand-in for a function
+// the VDBE implements in bytecode, whose result is one of its arguments.
+func noop(name string) bool {
+	return strings.HasPrefix(name, "noop")
+}
+
+// The kinds a result or argument call names, by the sqlite3_result_* or
+// sqlite3_value_* suffix with any 64 dropped. Suffixes not listed — error,
+// subtype, type, bytes, dup — say nothing about a type.
+var (
+	resultKinds = map[string]string{
+		"int": "integer", "double": "real",
+		"text": "text", "text16": "text", "text16le": "text", "text16be": "text",
+		"blob": "blob", "zeroblob": "blob",
+		"value": "any", "pointer": "any",
+	}
+	valueKinds = map[string]string{
+		"int": "integer", "double": "real",
+		"text": "text", "text16": "text", "text16le": "text", "text16be": "text",
+		"blob": "blob",
+	}
+)
+
+func kindOf(kinds map[string]string, suffix string) (string, bool) {
+	k, ok := kinds[strings.TrimSuffix(suffix, "64")]
+	return k, ok
+}
+
+// results collects the kinds a C function sets its result to, following
+// the helpers it calls a few levels down, since many functions hand their
+// result to one.
+func (s *source) results(name string, depth int, seen map[string]bool) map[string]bool {
+	kinds := map[string]bool{}
+	name = s.resolve(name)
+	if noop(name) {
+		kinds["any"] = true
+		return kinds
+	}
+	fn, ok := s.funcs[name]
+	if !ok || seen[name] || depth > 3 {
+		return kinds
+	}
+	seen[name] = true
+	for _, m := range resultCall.FindAllStringSubmatch(fn.body, -1) {
+		if k, ok := kindOf(resultKinds, m[1]); ok {
+			kinds[k] = true
+		}
+	}
+	for _, m := range callee.FindAllStringSubmatch(fn.body, -1) {
+		c := m[1]
+		if c == name || strings.HasPrefix(c, "sqlite3_") || strings.HasPrefix(c, "sqlite3Vdbe") {
+			continue
+		}
+		if _, ok := s.funcs[c]; ok {
+			for k := range s.results(c, depth+1, seen) {
+				kinds[k] = true
+			}
+		}
+	}
+	return kinds
+}
+
+// args collects the kinds a C function reads each of its arguments as, by
+// position, and the kind it reads a run of arguments as under a loop
+// index. An implementation called through the FTS5 extension API is handed
+// the table as an implicit first argument, so its positions shift by one.
+func (s *source) args(name string, positions map[int]map[string]bool, variadic map[string]bool) {
+	fn, ok := s.funcs[s.resolve(name)]
+	if !ok {
+		return
+	}
+	shift := 0
+	if strings.Contains(fn.params, "Fts5ExtensionApi") {
+		shift = 1
+		if positions[0] == nil {
+			positions[0] = map[string]bool{}
+		}
+		positions[0]["any"] = true
+	}
+	for _, m := range valueCall.FindAllStringSubmatch(fn.body, -1) {
+		k, ok := kindOf(valueKinds, m[1])
+		if !ok {
+			continue
+		}
+		var pos int
+		if _, err := fmt.Sscanf(m[2], "%d", &pos); err != nil {
+			variadic[k] = true
+			continue
+		}
+		pos += shift
+		if positions[pos] == nil {
+			positions[pos] = map[string]bool{}
+		}
+		positions[pos][k] = true
+	}
+}
+
+// single reduces the kinds seen at one position to a type: the one kind
+// seen, or "any" for a mixture or nothing.
+func single(kinds map[string]bool) string {
+	if len(kinds) == 1 {
+		for k := range kinds {
+			return k
+		}
+	}
+	return "any"
+}
+
+// signature derives what the source says a SQL function returns and takes.
+// A result of one kind is that type. A function that returns one of its
+// arguments, or a mixture of kinds, takes the type of its first argument,
+// which the seed spells "any" — except that integer and real together are
+// real, and Numeric, since abs, ceil and sum hand back an integer for an
+// integer and a real for a real, and a real for the text or blob they
+// convert, unless the function is one of the few whose choice turns on
+// something else; and text and blob together are text, since a function
+// that returns either is handing back the bytes it was given, and the
+// legacy compiler cannot follow "any" to an argument.
+func (s *source) signature(name string) (signature, error) {
+	r, ok := s.regs[strings.ToLower(name)]
+	if !ok {
+		return signature{}, fmt.Errorf("the amalgamation registers no function named %s", name)
+	}
+	var sig signature
+	switch {
+	case r.inline != "":
+		sig.Returns = inlineReturns[r.inline]
+		if sig.Returns == "" {
+			sig.Returns = "any"
+		}
+	default:
+		kinds := map[string]bool{}
+		for _, fn := range append(append([]string{}, r.scalar...), r.final...) {
+			for k := range s.results(fn, 0, map[string]bool{}) {
+				kinds[k] = true
+			}
+		}
+		jsonOnly := r.json != "" && len(kinds) <= 2 && !kinds["integer"] && !kinds["real"] && !kinds["any"]
+		switch {
+		case len(kinds) == 0:
+			return signature{}, fmt.Errorf("cannot tell what %s returns: no sqlite3_result call in %s", name, strings.Join(append(append([]string{}, r.scalar...), r.final...), ", "))
+		case r.jsonAlways:
+			sig.Returns = "text"
+		case kinds["any"]:
+			sig.Returns = "any"
+		case len(kinds) == 1:
+			for k := range kinds {
+				sig.Returns = k
+			}
+		case jsonOnly:
+			// The shared implementation writes JSON text or JSONB; the
+			// registration says which this form gets.
+			sig.Returns = r.json
+		case len(kinds) == 2 && kinds["integer"] && kinds["real"]:
+			sig.Returns = "real"
+			sig.Numeric = !realResults[strings.ToLower(name)]
+		case len(kinds) == 2 && kinds["text"] && kinds["blob"]:
+			sig.Returns = "text"
+		default:
+			sig.Returns = "any"
+		}
+	}
+
+	positions := map[int]map[string]bool{}
+	variadic := map[string]bool{}
+	for _, fn := range append(append([]string{}, r.scalar...), r.step...) {
+		s.args(fn, positions, variadic)
+	}
+	if len(positions) > 0 {
+		var order []int
+		for pos := range positions {
+			order = append(order, pos)
+		}
+		sort.Ints(order)
+		sig.Args = make([]string, order[len(order)-1]+1)
+		for i := range sig.Args {
+			sig.Args[i] = single(positions[i])
+		}
+		// Trailing positions nothing typed say nothing, unless a variadic
+		// tail follows them, when they keep it from starting early.
+		for len(variadic) == 0 && len(sig.Args) > 0 && sig.Args[len(sig.Args)-1] == "any" {
+			sig.Args = sig.Args[:len(sig.Args)-1]
+		}
+	}
+	if len(variadic) == 1 {
+		sig.Variadic = single(variadic)
+	}
+	return sig, nil
+}

internal/goldeneye/sqlite/sqlite.go

@@ -0,0 +1,444 @@
+// Package sqlite generates the SQLite dialect seed under
+// internal/engine/sqlite/dialect from sqlite3 shells built from the
+// amalgamation sqlite.org publishes, run against in-memory databases that
+// need no server, and from the amalgamation itself.
+//
+// SQLite describes its functions as far as their names, their kinds and the
+// number of arguments each takes — pragma_function_list — and no further:
+// it types values rather than columns or functions, so nothing in the
+// database says what a function returns or what it expects. The source
+// does, in its way. Every function is registered with the C functions that
+// implement it, and those set their result through sqlite3_result_* and
+// read their arguments through sqlite3_value_*, which is as close as SQLite
+// comes to declaring a signature. So functions.jsonl is built from both:
+// the shell says which functions exist, how many arguments each overload
+// takes and whether it aggregates, and the amalgamation says what each
+// returns and what its arguments hold. A function the shell reports that
+// the source does not register, or whose implementation sets no result,
+// fails generation rather than being guessed at. What neither can say —
+// which scalar functions return NULL for arguments that are not — is a
+// short list in signatures.go; for aggregates it is found by running each
+// over no rows.
+//
+// Which functions a SQLite has is decided when it is compiled, so the
+// dialect treats compile options the way the PostgreSQL dialect treats
+// contrib extensions. functions.jsonl is what a build with the default
+// options has, and each further option gets a directory under extensions/
+// holding the functions a build with that option adds — found the way
+// CREATE EXTENSION's additions are, by comparing the catalog with and
+// without. SQLite has no catalog of types or operators, so types.jsonl and
+// operators.jsonl are hand-written.
+//
+// The shells are built once per pinned version by Install.
+package sqlite
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"slices"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
+)
+
+// Engine is the name of the engine directory the dialect lives under.
+const Engine = "sqlite"
+
+// functionList is every function the connection knows, one row per
+// overload: name, whether the library builds it in or the shell or an
+// extension registered it, 's', 'a' or 'w' for how it is called, and how
+// many arguments it takes. Fixed arities come before the variable one so
+// that an exact match is found first by anything reading the file in
+// order.
+const functionList = `
+SELECT name, builtin, type, narg
+FROM pragma_function_list
+ORDER BY name, narg < 0, narg`
+
+type functionRow struct {
+	Name    string `json:"name"`
+	Builtin int    `json:"builtin"`
+	Type    string `json:"type"`
+	NArg    int    `json:"narg"`
+}
+
+// key tells one overload from another.
+func (r functionRow) key() string {
+	return r.Name + "/" + strconv.Itoa(r.NArg)
+}
+
+// Version reports the release the default shell is.
+func Version(ctx context.Context, dir string) (string, error) {
+	out, err := exec.CommandContext(ctx, builds()[0].binary(dir), "--version").Output()
+	if err != nil {
+		return "", fmt.Errorf("sqlite3 --version: %w", err)
+	}
+	return "SQLite " + strings.TrimSpace(string(out)), nil
+}
+
+// query runs a SQL statement against an in-memory database and decodes the
+// shell's JSON output into rows.
+func query(ctx context.Context, binary, sql string, rows any) error {
+	cmd := exec.CommandContext(ctx, binary, "-json", "-bail", ":memory:", sql)
+	out, err := cmd.Output()
+	if err != nil {
+		var exit *exec.ExitError
+		if errors.As(err, &exit) {
+			return fmt.Errorf("sqlite3: %s: %s", err, bytes.TrimSpace(exit.Stderr))
+		}
+		return fmt.Errorf("sqlite3: %w", err)
+	}
+	// An empty result set prints nothing rather than [].
+	if len(bytes.TrimSpace(out)) == 0 {
+		return nil
+	}
+	return json.Unmarshal(out, rows)
+}
+
+// minArgs decodes a negative argument count from pragma_function_list,
+// which prints the count SQLite keeps for the function as it is: -1 for any
+// number of arguments, and the two values reserved for built-ins, -3 for
+// one or more and -4 for two or more — see matchQuality in the SQLite
+// source. No function is counted as -2.
+func minArgs(narg int) int {
+	if narg < -2 {
+		return -2 - narg
+	}
+	return 0
+}
+
+// functionKind maps the type the shell reports onto the seed's letters.
+// The shell says 's' for a scalar function, 'a' for an aggregate and 'w'
+// for one with a window implementation — but every one of SQLite's
+// aggregates has one, so 'w' covers avg and count as well as row_number.
+// The two are told apart by calling the function without an OVER clause,
+// which only a window function refuses.
+func functionKind(ctx context.Context, binary string, row functionRow) (string, error) {
+	switch row.Type {
+	case "s":
+		return "", nil
+	case "a":
+		return "a", nil
+	case "w":
+		window, err := isWindowFunction(ctx, binary, row)
+		if err != nil {
+			return "", err
+		}
+		if window {
+			return "w", nil
+		}
+		return "a", nil
+	}
+	return "", fmt.Errorf("sqlite: function %s has unknown type %q", row.Name, row.Type)
+}
+
+// isWindowFunction reports whether the shell refuses to call a function
+// without an OVER clause. Any other complaint — an aggregate objecting to
+// the NULLs it is handed — means the call was accepted as an aggregate.
+func isWindowFunction(ctx context.Context, binary string, row functionRow) (bool, error) {
+	n := row.NArg
+	if n < 0 {
+		n = minArgs(n)
+	}
+	args := strings.TrimSuffix(strings.Repeat("NULL, ", n), ", ")
+	cmd := exec.CommandContext(ctx, binary, ":memory:", fmt.Sprintf("SELECT %s(%s)", row.Name, args))
+	var stderr bytes.Buffer
+	cmd.Stdout = io.Discard
+	cmd.Stderr = &stderr
+	err := cmd.Run()
+	if err == nil {
+		return false, nil
+	}
+	if strings.Contains(stderr.String(), "misuse of window function") {
+		return true, nil
+	}
+	var exit *exec.ExitError
+	if errors.As(err, &exit) {
+		return false, nil
+	}
+	return false, fmt.Errorf("sqlite3: %w", err)
+}
+
+// shell is one built sqlite3 and the functions it reports.
+type shell struct {
+	build  build
+	binary string
+	rows   []functionRow
+}
+
+// readShell lists a build's functions, after checking that the shell was
+// built with the options the build says — a cached shell from before the
+// option lists changed would otherwise describe the wrong dialect.
+func readShell(ctx context.Context, dir string, b build) (*shell, error) {
+	s := &shell{build: b, binary: b.binary(dir)}
+	if err := checkOptions(ctx, dir, b); err != nil {
+		return nil, err
+	}
+	if err := query(ctx, s.binary, functionList, &s.rows); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+// checkOptions asks a build's shell whether it was compiled with every
+// option any build turns on, and fails when the answers do not match the
+// build's own list.
+func checkOptions(ctx context.Context, dir string, b build) error {
+	var used []struct {
+		Option string `json:"option"`
+		Used   int    `json:"used"`
+	}
+	known := map[string]bool{}
+	for _, b := range shells() {
+		for _, opt := range b.flags() {
+			known[opt] = true
+		}
+	}
+	var clauses []string
+	for _, opt := range slices.Sorted(maps.Keys(known)) {
+		clauses = append(clauses, fmt.Sprintf("SELECT '%s' AS option, sqlite_compileoption_used('%s') AS used", opt, opt))
+	}
+	if err := query(ctx, b.binary(dir), strings.Join(clauses, " UNION ALL "), &used); err != nil {
+		return err
+	}
+	for _, u := range used {
+		want := 0
+		if slices.Contains(b.flags(), u.Option) {
+			want = 1
+		}
+		if u.Used != want {
+			return fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, filepath.Join(dir, b.name))
+		}
+	}
+	return nil
+}
+
+// generator accumulates the functions of every build, reading their
+// signatures from the amalgamation, and remembers which names were
+// reported so that the lists in signatures.go can be checked against the
+// shells at the end.
+type generator struct {
+	ctx      context.Context
+	src      *source
+	reported map[string]bool
+	// numeric holds every spelling types.jsonl gives integer and real, in
+	// its order, for the functions written once per spelling.
+	numeric map[string][]string
+}
+
+// overload is one row of a shell's list with what was found out about it.
+type overload struct {
+	row  functionRow
+	kind string
+	sig  signature
+}
+
+// functions turns rows into records, one per overload. Every row has to be
+// a function the amalgamation registers unless omitted. In the default
+// build only what the library builds in counts: the rest is what the shell
+// registers on top — edit, sha3, the extensions it bundles — which is not
+// the dialect's business, and which a comparison with the default build has
+// already removed from an option's rows.
+func (g *generator) functions(s *shell, base bool) ([]dialect.Function, error) {
+	var overloads []overload
+	seen := map[string]bool{}
+	var missing []string
+	for _, row := range s.rows {
+		g.reported[row.Name] = true
+		if omitted[row.Name] || seen[row.key()] || base && row.Builtin == 0 {
+			continue
+		}
+		seen[row.key()] = true
+		sig, err := g.src.signature(row.Name)
+		if err != nil {
+			if !seen[row.Name] {
+				seen[row.Name] = true
+				missing = append(missing, err.Error())
+			}
+			continue
+		}
+		kind, err := functionKind(g.ctx, s.binary, row)
+		if err != nil {
+			return nil, err
+		}
+		overloads = append(overloads, overload{row: row, kind: kind, sig: sig})
+	}
+	if len(missing) > 0 {
+		return nil, fmt.Errorf("sqlite: %s build: %s", s.build.name, strings.Join(missing, "; "))
+	}
+	empty, err := overNoRows(g.ctx, s.binary, overloads)
+	if err != nil {
+		return nil, err
+	}
+	funcs := make([]dialect.Function, 0, len(overloads))
+	for _, o := range overloads {
+		isNullable := nullable[o.row.Name]
+		if o.kind == "a" {
+			isNullable = empty[o.row.key()] == "null"
+		}
+		fn := dialect.Function{
+			Name:     o.row.Name,
+			Kind:     o.kind,
+			Args:     o.sig.args(o.row.NArg),
+			Returns:  o.sig.Returns,
+			Nullable: isNullable,
+		}
+		funcs = append(funcs, fn)
+		// A function that returns an integer for an integer and a real for
+		// a real is written once more per spelling of each, the way
+		// PostgreSQL's catalog has a sum per numeric type. The overload over
+		// any comes first: the analysis core picks the overload whose
+		// parameter is the argument's type, but the legacy compiler takes
+		// the first of the right arity, and keeps what it had.
+		if o.sig.Numeric && len(fn.Args) > 0 {
+			for _, typ := range []string{"integer", "real"} {
+				for _, spelling := range g.numeric[typ] {
+					typed := fn
+					typed.Args = append([]dialect.Arg{{Type: spelling}}, fn.Args[1:]...)
+					typed.Returns = typ
+					funcs = append(funcs, typed)
+				}
+			}
+		}
+	}
+	return funcs, nil
+}
+
+// overNoRows runs every aggregate over no rows and reports the type of
+// what each returns, "null" for the ones — avg, max, group_concat — that
+// return NULL when there is nothing to aggregate, and not count or total.
+// The probes go through one shell process, each statement labelled with
+// its overload so that the answers can be told apart.
+func overNoRows(ctx context.Context, binary string, overloads []overload) (map[string]string, error) {
+	var script strings.Builder
+	for _, o := range overloads {
+		if o.kind != "a" {
+			continue
+		}
+		n := o.row.NArg
+		if n < 0 {
+			n = minArgs(n)
+		}
+		args := strings.TrimSuffix(strings.Repeat("x, ", n), ", ")
+		fmt.Fprintf(&script, "SELECT '%s', typeof(%s(%s)) FROM (SELECT NULL AS x) WHERE 0;\n", o.row.key(), o.row.Name, args)
+	}
+	results := map[string]string{}
+	if script.Len() == 0 {
+		return results, nil
+	}
+	cmd := exec.CommandContext(ctx, binary, "-list", ":memory:", script.String())
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		var exit *exec.ExitError
+		if !errors.As(err, &exit) {
+			return nil, fmt.Errorf("sqlite3: %w", err)
+		}
+	}
+	for _, line := range strings.Split(strings.TrimSpace(stdout.String()), "\n") {
+		key, typ, ok := strings.Cut(line, "|")
+		if ok {
+			results[key] = typ
+		}
+	}
+	for _, o := range overloads {
+		if o.kind == "a" {
+			if _, ok := results[o.row.key()]; !ok {
+				return nil, fmt.Errorf("sqlite: cannot run %s over no rows: %s", o.row.Name, strings.TrimSpace(stderr.String()))
+			}
+		}
+	}
+	return results, nil
+}
+
+// added returns the rows of an option's shell that the default shell does
+// not have: what the option adds.
+func added(opt, base *shell) *shell {
+	have := map[string]bool{}
+	for _, row := range base.rows {
+		have[row.key()] = true
+	}
+	diff := &shell{build: opt.build, binary: opt.binary}
+	for _, row := range opt.rows {
+		if !have[row.key()] {
+			diff.rows = append(diff.rows, row)
+		}
+	}
+	return diff
+}
+
+// Generate reads the dialect from the shells under dir and the
+// amalgamation they were built from.
+func Generate(ctx context.Context, dir string) (dialect.Files, error) {
+	src, err := readSource(filepath.Join(dir, "src", "sqlite3.c"))
+	if err != nil {
+		return nil, err
+	}
+	g := &generator{ctx: ctx, src: src, reported: map[string]bool{}, numeric: map[string][]string{}}
+	dialectDir, err := dialect.Dir(Engine)
+	if err != nil {
+		return nil, err
+	}
+	types, err := dialect.ReadTypes(dialectDir)
+	if err != nil {
+		return nil, err
+	}
+	for _, t := range types {
+		if t.Name == "integer" || t.Name == "real" {
+			g.numeric[t.Name] = append([]string{t.Name}, t.Aliases...)
+		}
+	}
+	all := builds()
+	base, err := readShell(ctx, dir, all[0])
+	if err != nil {
+		return nil, err
+	}
+	funcs, err := g.functions(base, true)
+	if err != nil {
+		return nil, err
+	}
+	files := dialect.Files{}
+	if files[dialect.FunctionsFile], err = dialect.JSONL(funcs); err != nil {
+		return nil, err
+	}
+	for _, b := range all[1:] {
+		s, err := readShell(ctx, dir, b)
+		if err != nil {
+			return nil, err
+		}
+		funcs, err := g.functions(added(s, base), false)
+		if err != nil {
+			return nil, err
+		}
+		if len(funcs) == 0 {
+			return nil, fmt.Errorf("sqlite: %s adds no functions over the default build; drop it from the extensions in install.go", strings.Join(b.options, " "))
+		}
+		if files[path.Join(dialect.ExtensionsDir, b.name, dialect.FunctionsFile)], err = dialect.JSONL(funcs); err != nil {
+			return nil, err
+		}
+	}
+	var stale []string
+	for _, list := range []map[string]bool{omitted, nullable, realResults} {
+		for name := range list {
+			if !g.reported[name] {
+				stale = append(stale, name)
+			}
+		}
+	}
+	if len(stale) > 0 {
+		sort.Strings(stale)
+		return nil, fmt.Errorf("sqlite: signatures.go lists function(s) no shell reports: %s", strings.Join(stale, ", "))
+	}
+	return files, nil
+}

internal/goldeneye/sqlite/sqlite_test.go

@@ -0,0 +1,66 @@
+package sqlite
+
+import (
+	"context"
+	"testing"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend"
+)
+
+// TestDialect verifies the committed SQLite dialect against what the pinned
+// sqlite3 shell reports. It skips unless the shell is installed.
+func TestDialect(t *testing.T) {
+	binary, err := Locate()
+	if err != nil {
+		t.Skip(err)
+	}
+	ctx := context.Background()
+	version, err := Version(ctx, binary)
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, err := Generate(ctx, binary)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir, err := dialect.Dir(Engine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	report, err := dialect.Check(dir, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report != "" {
+		t.Errorf("%s does not match what %s reports:\n%s", dir, version, report)
+	}
+}
+
+// TestAnalyzeCases verifies every SQLite analyze case under
+// internal/endtoend/testdata against what SQLite reports. It skips unless
+// the shells are installed.
+func TestAnalyzeCases(t *testing.T) {
+	dir, err := Locate()
+	if err != nil {
+		t.Skip(err)
+	}
+	cases, err := endtoend.Cases(Engine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(cases) == 0 {
+		t.Fatal("no sqlite analyze cases found")
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			diff, err := Check(context.Background(), dir, c)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff != "" {
+				t.Errorf("%s does not match what SQLite reports (-committed +sqlite):\n%s", c.Output, diff)
+			}
+		})
+	}
+}