snapshot of https://github.com/sqlc-dev/sqlc.git (main, 2026-09-16; blobs > 131072B omitted)
Some checks failed
kotlin / test (push) Has been skipped
python / test (push) Has been skipped
typescript / test (push) Has been skipped
go / build darwin/amd64 (push) Has been cancelled
go / build linux/amd64 (push) Has been cancelled
go / build windows/amd64 (push) Has been cancelled
go / build darwin/arm64 (push) Has been cancelled
go / build linux/arm64 (push) Has been cancelled
go / build windows/arm64 (push) Has been cancelled
go / test (push) Has been cancelled
go / vuln_check (push) Has been cancelled
Some checks failed
kotlin / test (push) Has been skipped
python / test (push) Has been skipped
typescript / test (push) Has been skipped
go / build darwin/amd64 (push) Has been cancelled
go / build linux/amd64 (push) Has been cancelled
go / build windows/amd64 (push) Has been cancelled
go / build darwin/arm64 (push) Has been cancelled
go / build linux/arm64 (push) Has been cancelled
go / build windows/arm64 (push) Has been cancelled
go / test (push) Has been cancelled
go / vuln_check (push) Has been cancelled
This commit is contained in:
846
internal/goldeneye/duckdb/analyze.go
Normal file
846
internal/goldeneye/duckdb/analyze.go
Normal file
@@ -0,0 +1,846 @@
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/analysis"
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend"
|
||||
)
|
||||
|
||||
// The analyze cases are checked against the DuckDB CLI, which runs each
|
||||
// case's schema and fixture into an in-memory database of their own, one
|
||||
// process per question, and is asked four things about each query. What
|
||||
// its parameters are: the query is prepared and explained, as JSON, with
|
||||
// a string sentinel bound to each parameter, and the unoptimized logical
|
||||
// plan the CLI prints first shows each as CAST('goldeneye_k' AS T), T
|
||||
// being the type the binder gave the parameter, or bare when that type
|
||||
// is VARCHAR. What its result columns are:
|
||||
// DESCRIBE, with each parameter replaced by a NULL of that type, names and
|
||||
// types them. Which column each is read from and which column a parameter
|
||||
// stands in for: DuckDB prints a plan with every column by its bare name,
|
||||
// so these are read from the query text, the select list's items and the
|
||||
// operand beside each parameter resolved against the FROM clause and the
|
||||
// catalog, from which a column read from a table takes its declared type
|
||||
// and nullability. And whether an expression can be NULL, which DuckDB
|
||||
// does not track: the query is run, with each parameter bound to a value
|
||||
// of its type, over the fixture and over no rows, and a column is nullable
|
||||
// when either run returns a NULL for it.
|
||||
|
||||
// placeholder is one parameter of a query as sqlc numbers them: a $n by
|
||||
// its number, and each $name, sqlc.arg name or ? in turn at its first
|
||||
// appearance, taking the lowest number no $n took.
|
||||
type placeholder struct {
|
||||
Number int
|
||||
Name string
|
||||
}
|
||||
|
||||
var (
|
||||
numberedRe = regexp.MustCompile(`^\$([0-9]+)`)
|
||||
namedRe = regexp.MustCompile(`^\$([A-Za-z_][A-Za-z0-9_]*)`)
|
||||
sqlcArgRe = regexp.MustCompile(`^sqlc\.(n?arg|slice)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`)
|
||||
)
|
||||
|
||||
// bind rewrites the query so that every parameter is a $k numbered as
|
||||
// sqlc numbers it, and lists the parameters in that order.
|
||||
func bind(query string) (string, []placeholder) {
|
||||
type occurrence struct {
|
||||
start, end int
|
||||
number int // for $n
|
||||
name string // for $name or sqlc.arg, "" for ?
|
||||
}
|
||||
var occ []occurrence
|
||||
i := 0
|
||||
for i < len(query) {
|
||||
c := query[i]
|
||||
switch {
|
||||
case c == '\'' || c == '"':
|
||||
i = quotedEnd(query, i)
|
||||
case strings.HasPrefix(query[i:], "--"):
|
||||
end := strings.IndexByte(query[i:], '\n')
|
||||
if end < 0 {
|
||||
i = len(query)
|
||||
} else {
|
||||
i += end
|
||||
}
|
||||
case strings.HasPrefix(query[i:], "/*"):
|
||||
end := strings.Index(query[i:], "*/")
|
||||
if end < 0 {
|
||||
i = len(query)
|
||||
} else {
|
||||
i += end + 2
|
||||
}
|
||||
case c == '?':
|
||||
occ = append(occ, occurrence{start: i, end: i + 1})
|
||||
i++
|
||||
case c == '$' && numberedRe.MatchString(query[i:]):
|
||||
m := numberedRe.FindStringSubmatch(query[i:])
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
occ = append(occ, occurrence{start: i, end: i + len(m[0]), number: n})
|
||||
i += len(m[0])
|
||||
case c == '$' && namedRe.MatchString(query[i:]):
|
||||
m := namedRe.FindStringSubmatch(query[i:])
|
||||
occ = append(occ, occurrence{start: i, end: i + len(m[0]), name: m[1]})
|
||||
i += len(m[0])
|
||||
case c == 's' && sqlcArgRe.MatchString(query[i:]):
|
||||
m := sqlcArgRe.FindStringSubmatch(query[i:])
|
||||
occ = append(occ, occurrence{start: i, end: i + len(m[0]), name: m[2]})
|
||||
i += len(m[0])
|
||||
default:
|
||||
i++
|
||||
}
|
||||
}
|
||||
taken := map[int]bool{}
|
||||
for _, o := range occ {
|
||||
if o.number > 0 {
|
||||
taken[o.number] = true
|
||||
}
|
||||
}
|
||||
byName := map[string]int{}
|
||||
next := 1
|
||||
assign := func(o occurrence) int {
|
||||
if o.number > 0 {
|
||||
return o.number
|
||||
}
|
||||
if n, ok := byName[o.name]; ok && o.name != "" {
|
||||
return n
|
||||
}
|
||||
for taken[next] {
|
||||
next++
|
||||
}
|
||||
n := next
|
||||
taken[n] = true
|
||||
if o.name != "" {
|
||||
byName[o.name] = n
|
||||
}
|
||||
return n
|
||||
}
|
||||
var phs []placeholder
|
||||
numbered := map[int]bool{}
|
||||
var out strings.Builder
|
||||
last := 0
|
||||
for _, o := range occ {
|
||||
n := assign(o)
|
||||
if !numbered[n] {
|
||||
numbered[n] = true
|
||||
phs = append(phs, placeholder{Number: n, Name: o.name})
|
||||
}
|
||||
out.WriteString(query[last:o.start])
|
||||
out.WriteString("$" + strconv.Itoa(n))
|
||||
last = o.end
|
||||
}
|
||||
out.WriteString(query[last:])
|
||||
for i := 1; i < len(phs); i++ {
|
||||
for j := i; j > 0 && phs[j-1].Number > phs[j].Number; j-- {
|
||||
phs[j-1], phs[j] = phs[j], phs[j-1]
|
||||
}
|
||||
}
|
||||
return out.String(), phs
|
||||
}
|
||||
|
||||
// nullMarker is what a NULL is printed as when the CLI prints CSV, unlike
|
||||
// any value a case holds.
|
||||
const nullMarker = "<goldeneye:null>"
|
||||
|
||||
// analyzer holds the CLI a case runs through, the case's schema and
|
||||
// fixture, and the catalog of the schema.
|
||||
type analyzer struct {
|
||||
binary string
|
||||
schema string // the schema's statements, each ending in ;
|
||||
fixture string // the fixture's, or ""
|
||||
tables map[string][]column
|
||||
enums map[string][]string // the labels of each enum type the schema created
|
||||
// canonical names the type the dialect reports a spelling as, for
|
||||
// each alias types.jsonl lists: DuckDB spells a JSON column json,
|
||||
// which its own catalog lists as a spelling of varchar.
|
||||
canonical map[string]string
|
||||
}
|
||||
|
||||
// readAliases reads the aliases the generated types.jsonl gives each
|
||||
// type, keyed by alias.
|
||||
func readAliases() (map[string]string, error) {
|
||||
dir, err := dialect.Dir(Engine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
types, err := dialect.ReadTypes(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
canonical := map[string]string{}
|
||||
for _, t := range types {
|
||||
for _, alias := range t.Aliases {
|
||||
canonical[strings.ToLower(alias)] = strings.ToLower(t.Name)
|
||||
}
|
||||
}
|
||||
return canonical, nil
|
||||
}
|
||||
|
||||
// run executes a script in a fresh in-memory database loaded with the
|
||||
// schema and, unless told otherwise, the fixture, and returns what the
|
||||
// CLI printed in the mode asked for: "json" prints one JSON array per
|
||||
// statement that returns rows, "csv" prints rows with a header and NULL
|
||||
// as nullMarker, and "" prints the CLI's own text, which is how EXPLAIN
|
||||
// draws a plan. A statement that fails ends the script with DuckDB's own
|
||||
// error message.
|
||||
func (a *analyzer) run(ctx context.Context, script, mode string, withFixture bool) (string, error) {
|
||||
args := []string{"-bail"}
|
||||
switch mode {
|
||||
case "json":
|
||||
args = append(args, "-json")
|
||||
case "csv":
|
||||
args = append(args, "-csv", "-header", "-nullvalue", nullMarker)
|
||||
}
|
||||
args = append(args, ":memory:")
|
||||
cmd := exec.CommandContext(ctx, a.binary, args...)
|
||||
prelude := a.schema
|
||||
if withFixture {
|
||||
prelude += a.fixture
|
||||
}
|
||||
cmd.Stdin = strings.NewReader(prelude + script)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
msg := strings.TrimSpace(stderr.String())
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
return "", errors.New(msg)
|
||||
}
|
||||
return stdout.String(), nil
|
||||
}
|
||||
|
||||
// query runs one statement and decodes the rows it printed.
|
||||
func (a *analyzer) query(ctx context.Context, statement string) ([]map[string]json.RawMessage, error) {
|
||||
out, err := a.run(ctx, statement+";\n", "json", true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var rows []map[string]json.RawMessage
|
||||
dec := json.NewDecoder(strings.NewReader(out))
|
||||
for {
|
||||
var rs []map[string]json.RawMessage
|
||||
err := dec.Decode(&rs)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding duckdb output: %w", err)
|
||||
}
|
||||
rows = append(rows, rs...)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func str(v json.RawMessage) string {
|
||||
var s string
|
||||
json.Unmarshal(v, &s)
|
||||
return s
|
||||
}
|
||||
|
||||
// column is what the catalog says about a column of a table.
|
||||
type column struct {
|
||||
name string
|
||||
spelling string // the type as DuckDB spells it
|
||||
typ *analysis.TypeExpr
|
||||
nullable bool
|
||||
}
|
||||
|
||||
// statements joins a script's statements, each ending in a semicolon.
|
||||
func statements(src string) string {
|
||||
s := strings.TrimRight(strings.TrimSpace(src), ";")
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return s + ";\n"
|
||||
}
|
||||
|
||||
// Analyze runs a case's queries through the CLI and returns what DuckDB
|
||||
// reports in the JSON shape sqlc analyze prints.
|
||||
func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error) {
|
||||
schema, err := os.ReadFile(c.Schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var fixture []byte
|
||||
if c.Fixture != "" {
|
||||
if fixture, err = os.ReadFile(c.Fixture); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
queries, err := c.Queries()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a := &analyzer{binary: binary, schema: statements(string(schema)), fixture: statements(string(fixture))}
|
||||
if a.canonical, err = readAliases(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := a.readCatalog(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]analysis.Query, 0, len(queries))
|
||||
for _, q := range queries {
|
||||
aq, err := a.analyzeQuery(ctx, q)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", q.Name, err)
|
||||
}
|
||||
out = append(out, aq)
|
||||
}
|
||||
return analysis.Encode(out)
|
||||
}
|
||||
|
||||
// Check compares what DuckDB reports for a case with the output the case
|
||||
// committed, returning a diff when they differ.
|
||||
func Check(ctx context.Context, binary string, c endtoend.Case) (string, error) {
|
||||
got, err := Analyze(ctx, binary, c)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return c.Compare(got)
|
||||
}
|
||||
|
||||
// readCatalog reads the tables and enum types the schema created.
|
||||
func (a *analyzer) readCatalog(ctx context.Context) error {
|
||||
rows, err := a.query(ctx, `SELECT type_name, labels FROM duckdb_types() WHERE database_name = 'memory' AND NOT internal AND logical_type = 'ENUM' ORDER BY type_name`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.enums = map[string][]string{}
|
||||
for _, row := range rows {
|
||||
var labels []string
|
||||
json.Unmarshal(row["labels"], &labels)
|
||||
a.enums[strings.ToLower(str(row["type_name"]))] = labels
|
||||
}
|
||||
rows, err = a.query(ctx, `SELECT table_name, column_name, data_type, is_nullable FROM duckdb_columns() WHERE database_name = 'memory' ORDER BY table_oid, column_index`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.tables = map[string][]column{}
|
||||
for _, row := range rows {
|
||||
table := strings.ToLower(str(row["table_name"]))
|
||||
var nullable bool
|
||||
json.Unmarshal(row["is_nullable"], &nullable)
|
||||
spelling := str(row["data_type"])
|
||||
a.tables[table] = append(a.tables[table], column{
|
||||
name: str(row["column_name"]),
|
||||
spelling: spelling,
|
||||
typ: a.parseType(spelling),
|
||||
nullable: nullable,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookup finds a column of a table, which DuckDB matches in any case.
|
||||
func (a *analyzer) lookup(table, name string) (column, bool) {
|
||||
for _, col := range a.tables[strings.ToLower(table)] {
|
||||
if strings.EqualFold(col.name, name) {
|
||||
return col, true
|
||||
}
|
||||
}
|
||||
return column{}, false
|
||||
}
|
||||
|
||||
// resolve finds the table a reference names in a scope: the table its
|
||||
// qualifier aliases, or the one table of the scope that has the column.
|
||||
func (a *analyzer) resolve(sc scope, r ref) (string, column, bool) {
|
||||
tables := sc.tables
|
||||
if sc.target.name != "" {
|
||||
tables = append([]tableRef{sc.target}, tables...)
|
||||
}
|
||||
if r.qualifier != "" {
|
||||
if sc.ctes[r.qualifier] {
|
||||
return "", column{}, false
|
||||
}
|
||||
for _, t := range tables {
|
||||
if t.alias == r.qualifier || (t.alias == "" && t.name == r.qualifier) {
|
||||
col, ok := a.lookup(t.name, r.column)
|
||||
return t.name, col, ok
|
||||
}
|
||||
}
|
||||
return "", column{}, false
|
||||
}
|
||||
var found string
|
||||
var fc column
|
||||
for _, t := range tables {
|
||||
if col, ok := a.lookup(t.name, r.column); ok {
|
||||
if found != "" && found != t.name {
|
||||
return "", column{}, false
|
||||
}
|
||||
found, fc = t.name, col
|
||||
}
|
||||
}
|
||||
return found, fc, found != ""
|
||||
}
|
||||
|
||||
// describe reads what the catalog says about a column read from a table.
|
||||
func describe(table string, col column) analysis.Column {
|
||||
return analysis.Column{Name: col.name, Type: withNullable(col.typ, col.nullable), Table: table}
|
||||
}
|
||||
|
||||
func withNullable(t *analysis.TypeExpr, nullable bool) *analysis.TypeExpr {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
out := *t
|
||||
out.Nullable = nullable
|
||||
return &out
|
||||
}
|
||||
|
||||
// binding is what is known about one parameter: the type the binder gave
|
||||
// it, spelled as DuckDB spells it, and the column it stands in for.
|
||||
type binding struct {
|
||||
spelling string
|
||||
typ *analysis.TypeExpr
|
||||
column *analysis.Column
|
||||
}
|
||||
|
||||
var (
|
||||
sentinelCastRe = regexp.MustCompile(`CAST\('goldeneye_([0-9]+)' AS `)
|
||||
sentinelRe = regexp.MustCompile(`'goldeneye_([0-9]+)'`)
|
||||
sentinelErrRe = regexp.MustCompile(`goldeneye_([0-9]+)`)
|
||||
)
|
||||
|
||||
// explain prepares the query, explains it with a sentinel bound to each
|
||||
// parameter and returns the type the binder gave each, keyed by number.
|
||||
// A parameter whose sentinel the binder converts on the spot, as an
|
||||
// INSERT's VALUES are, is bound to NULL instead and reported by nothing:
|
||||
// the error names the sentinel for most types, and for the rest the
|
||||
// parameters not yet bound to NULL are tried in turn.
|
||||
func (a *analyzer) explain(ctx context.Context, sql string, phs []placeholder) (map[int]string, error) {
|
||||
null := map[int]bool{}
|
||||
for attempt := 0; attempt <= len(phs); attempt++ {
|
||||
args := make([]string, len(phs))
|
||||
for i, ph := range phs {
|
||||
if null[ph.Number] {
|
||||
args[i] = "NULL"
|
||||
} else {
|
||||
args[i] = fmt.Sprintf("'goldeneye_%d'", ph.Number)
|
||||
}
|
||||
}
|
||||
script := "PREPARE goldeneye AS " + sql + ";\nSET explain_output = 'all';\nEXPLAIN (FORMAT json) EXECUTE goldeneye(" + strings.Join(args, ", ") + ");\n"
|
||||
out, err := a.run(ctx, script, "", true)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "Conversion Error") && !strings.Contains(err.Error(), "can't be cast") {
|
||||
return nil, err
|
||||
}
|
||||
retry := false
|
||||
if m := sentinelErrRe.FindStringSubmatch(err.Error()); m != nil {
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
if !null[n] {
|
||||
null[n] = true
|
||||
retry = true
|
||||
}
|
||||
}
|
||||
if !retry {
|
||||
for _, ph := range phs {
|
||||
if !null[ph.Number] {
|
||||
null[ph.Number] = true
|
||||
retry = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !retry {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
return sentinelTypes(out)
|
||||
}
|
||||
return nil, errors.New("could not bind the parameters")
|
||||
}
|
||||
|
||||
// planNode is one operator of a plan the CLI prints as JSON. Its
|
||||
// extra_info holds the operator's expressions, each a string or a list
|
||||
// of strings.
|
||||
type planNode struct {
|
||||
Name string `json:"name"`
|
||||
Children []planNode `json:"children"`
|
||||
ExtraInfo map[string]json.RawMessage `json:"extra_info"`
|
||||
}
|
||||
|
||||
// sentinelTypes reads the type each sentinel is cast to out of the plans
|
||||
// the CLI printed, one JSON array per plan: the unoptimized logical plan
|
||||
// comes first, and the first cast of a sentinel wins. A sentinel the
|
||||
// plan prints bare is a VARCHAR, which the binder needs no cast for.
|
||||
func sentinelTypes(out string) (map[int]string, error) {
|
||||
var texts []string
|
||||
var walk func(n planNode)
|
||||
walk = func(n planNode) {
|
||||
for _, raw := range n.ExtraInfo {
|
||||
var one string
|
||||
if json.Unmarshal(raw, &one) == nil {
|
||||
texts = append(texts, one)
|
||||
continue
|
||||
}
|
||||
var many []string
|
||||
if json.Unmarshal(raw, &many) == nil {
|
||||
texts = append(texts, many...)
|
||||
}
|
||||
}
|
||||
for _, c := range n.Children {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
dec := json.NewDecoder(strings.NewReader(out))
|
||||
for {
|
||||
var plan []planNode
|
||||
err := dec.Decode(&plan)
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decoding the plan: %w", err)
|
||||
}
|
||||
for _, n := range plan {
|
||||
walk(n)
|
||||
}
|
||||
}
|
||||
types := map[int]string{}
|
||||
for _, text := range texts {
|
||||
for _, m := range sentinelCastRe.FindAllStringSubmatchIndex(text, -1) {
|
||||
n, _ := strconv.Atoi(text[m[2]:m[3]])
|
||||
if _, ok := types[n]; ok {
|
||||
continue
|
||||
}
|
||||
// The type runs to the parenthesis closing the CAST.
|
||||
depth := 1
|
||||
i := m[1]
|
||||
for ; i < len(text); i++ {
|
||||
if text[i] == '(' {
|
||||
depth++
|
||||
} else if text[i] == ')' {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
types[n] = strings.Join(strings.Fields(text[m[1]:i]), " ")
|
||||
}
|
||||
}
|
||||
for _, text := range texts {
|
||||
for _, m := range sentinelRe.FindAllStringSubmatchIndex(text, -1) {
|
||||
n, _ := strconv.Atoi(text[m[2]:m[3]])
|
||||
if _, ok := types[n]; !ok && !strings.HasSuffix(text[:m[0]], "CAST(") {
|
||||
types[n] = "VARCHAR"
|
||||
}
|
||||
}
|
||||
}
|
||||
return types, nil
|
||||
}
|
||||
|
||||
// analyzeQuery describes one query.
|
||||
func (a *analyzer) analyzeQuery(ctx context.Context, q endtoend.Query) (analysis.Query, error) {
|
||||
sql, phs := bind(q.SQL)
|
||||
aq := analysis.Query{
|
||||
Name: q.Name,
|
||||
Cmd: q.Cmd,
|
||||
Columns: []analysis.Column{},
|
||||
Params: []analysis.Param{},
|
||||
}
|
||||
t := text(tokenize(sql))
|
||||
sc := t.readScope()
|
||||
|
||||
// Parameters: a partner column first, then the cast the query wraps
|
||||
// the parameter in, then the type the binder gave it.
|
||||
bindings := map[int]*binding{}
|
||||
var casts []string
|
||||
var castNumbers []int
|
||||
for _, ph := range phs {
|
||||
k := strconv.Itoa(ph.Number)
|
||||
b := &binding{}
|
||||
bindings[ph.Number] = b
|
||||
if t.inSubquery(k) {
|
||||
// A parameter inside a subquery is compared with a column of
|
||||
// the subquery's own tables, which the statement's scope does
|
||||
// not name.
|
||||
if typ, ok := t.castOf(k); ok {
|
||||
casts = append(casts, typ)
|
||||
castNumbers = append(castNumbers, ph.Number)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sc.kind == "insert" {
|
||||
if pos, ok := t.valuesPosition(k); ok {
|
||||
cols := t.insertColumns()
|
||||
if cols == nil {
|
||||
for _, col := range a.tables[sc.target.name] {
|
||||
cols = append(cols, col.name)
|
||||
}
|
||||
}
|
||||
if pos < len(cols) {
|
||||
if col, ok := a.lookup(sc.target.name, cols[pos]); ok {
|
||||
c := describe(sc.target.name, col)
|
||||
b.column, b.spelling, b.typ = &c, col.spelling, col.typ
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if r, ok := t.partner(k); ok {
|
||||
if table, col, ok := a.resolve(sc, r); ok {
|
||||
c := describe(table, col)
|
||||
b.column, b.spelling, b.typ = &c, col.spelling, col.typ
|
||||
continue
|
||||
}
|
||||
}
|
||||
if typ, ok := t.castOf(k); ok {
|
||||
casts = append(casts, typ)
|
||||
castNumbers = append(castNumbers, ph.Number)
|
||||
}
|
||||
}
|
||||
if len(casts) > 0 {
|
||||
// DuckDB spells the cast's type: VARCHAR(5) is VARCHAR, mood is
|
||||
// ENUM('sad', 'ok').
|
||||
var items []string
|
||||
for i, typ := range casts {
|
||||
items = append(items, fmt.Sprintf("CAST(NULL AS %s) AS p%d", typ, i))
|
||||
}
|
||||
rows, err := a.query(ctx, "DESCRIBE SELECT "+strings.Join(items, ", "))
|
||||
if err != nil {
|
||||
return analysis.Query{}, err
|
||||
}
|
||||
for i, row := range rows {
|
||||
if i < len(castNumbers) {
|
||||
b := bindings[castNumbers[i]]
|
||||
b.spelling = str(row["column_type"])
|
||||
b.typ = a.parseType(b.spelling)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(phs) > 0 {
|
||||
types, err := a.explain(ctx, sql, phs)
|
||||
if err != nil {
|
||||
return analysis.Query{}, err
|
||||
}
|
||||
for n, spelling := range types {
|
||||
if b := bindings[n]; b != nil && b.typ == nil {
|
||||
b.spelling = spelling
|
||||
b.typ = a.parseType(spelling)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Result columns.
|
||||
var columns []analysis.Column
|
||||
switch sc.kind {
|
||||
case "select":
|
||||
rows, err := a.query(ctx, "DESCRIBE "+a.substitute(sql, phs, bindings, false))
|
||||
if err != nil {
|
||||
return analysis.Query{}, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
columns = append(columns, analysis.Column{Name: str(row["column_name"]), Type: a.parseType(str(row["column_type"]))})
|
||||
}
|
||||
a.attribute(sc, t.selectItems(), columns)
|
||||
default:
|
||||
// A RETURNING column is the target table's column it names;
|
||||
// DuckDB describes no DML, so an expression is typed by nothing.
|
||||
for _, it := range t.returningItems() {
|
||||
switch {
|
||||
case it.star:
|
||||
for _, col := range a.tables[sc.target.name] {
|
||||
columns = append(columns, describe(sc.target.name, col))
|
||||
}
|
||||
case it.ref != nil:
|
||||
if table, col, ok := a.resolve(sc, *it.ref); ok {
|
||||
c := describe(table, col)
|
||||
c.Name = it.name
|
||||
columns = append(columns, c)
|
||||
continue
|
||||
}
|
||||
columns = append(columns, analysis.Column{Name: it.name})
|
||||
default:
|
||||
columns = append(columns, analysis.Column{Name: it.name})
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(columns) > 0 {
|
||||
nullable := a.observe(ctx, sql, phs, bindings, len(columns))
|
||||
for i := range columns {
|
||||
if columns[i].Table == "" && columns[i].Type != nil && nullable[i] {
|
||||
columns[i].Type = withNullable(columns[i].Type, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
aq.Columns = append(aq.Columns, columns...)
|
||||
|
||||
for _, ph := range phs {
|
||||
b := bindings[ph.Number]
|
||||
ac := analysis.Column{Type: b.typ}
|
||||
if b.column != nil {
|
||||
ac = *b.column
|
||||
}
|
||||
aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac})
|
||||
}
|
||||
return aq, nil
|
||||
}
|
||||
|
||||
// attribute says which table each result column is read from, by
|
||||
// matching the select list's items, a star expanded to its table's
|
||||
// columns, against the columns DuckDB described. A list that expands to
|
||||
// another number of columns than DuckDB describes attributes nothing.
|
||||
func (a *analyzer) attribute(sc scope, items []item, columns []analysis.Column) {
|
||||
var origins []*analysis.Column
|
||||
for _, it := range items {
|
||||
switch {
|
||||
case it.star && it.ref == nil:
|
||||
for _, t := range sc.tables {
|
||||
for _, col := range a.tables[t.name] {
|
||||
c := describe(t.name, col)
|
||||
origins = append(origins, &c)
|
||||
}
|
||||
}
|
||||
case it.star:
|
||||
table, ok := a.aliased(sc, it.ref.qualifier)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for _, col := range a.tables[table] {
|
||||
c := describe(table, col)
|
||||
origins = append(origins, &c)
|
||||
}
|
||||
case it.ref != nil:
|
||||
if table, col, ok := a.resolve(sc, *it.ref); ok {
|
||||
c := describe(table, col)
|
||||
origins = append(origins, &c)
|
||||
} else {
|
||||
origins = append(origins, nil)
|
||||
}
|
||||
default:
|
||||
origins = append(origins, nil)
|
||||
}
|
||||
}
|
||||
if len(origins) != len(columns) {
|
||||
return
|
||||
}
|
||||
for i, o := range origins {
|
||||
if o != nil {
|
||||
columns[i].Type = o.Type
|
||||
columns[i].Table = o.Table
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// aliased finds the table a qualifier names in a scope.
|
||||
func (a *analyzer) aliased(sc scope, qualifier string) (string, bool) {
|
||||
for _, t := range sc.tables {
|
||||
if t.alias == qualifier || (t.alias == "" && t.name == qualifier) {
|
||||
if _, ok := a.tables[t.name]; ok {
|
||||
return t.name, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// substitute replaces each parameter with a NULL of its type, or with a
|
||||
// value of its type when values is set, for DuckDB to describe or run the
|
||||
// query. A parameter of unknown type becomes a bare NULL. Strings,
|
||||
// quoted identifiers and comments are left alone.
|
||||
func (a *analyzer) substitute(sql string, phs []placeholder, bindings map[int]*binding, values bool) string {
|
||||
repl := map[int]string{}
|
||||
for _, ph := range phs {
|
||||
r := "NULL"
|
||||
if b := bindings[ph.Number]; b != nil && b.spelling != "" {
|
||||
r = "CAST(NULL AS " + b.spelling + ")"
|
||||
if values {
|
||||
// The value is chosen by the type as spelled, since an
|
||||
// alias such as JSON takes values its canonical type
|
||||
// does not.
|
||||
if v, ok := a.zero(a.parseTypeWith(b.spelling, false)); ok {
|
||||
r = "CAST('" + strings.ReplaceAll(v, "'", "''") + "' AS " + b.spelling + ")"
|
||||
}
|
||||
}
|
||||
}
|
||||
repl[ph.Number] = r
|
||||
}
|
||||
var out strings.Builder
|
||||
i := 0
|
||||
for i < len(sql) {
|
||||
c := sql[i]
|
||||
switch {
|
||||
case c == '\'' || c == '"':
|
||||
end := quotedEnd(sql, i)
|
||||
out.WriteString(sql[i:end])
|
||||
i = end
|
||||
case strings.HasPrefix(sql[i:], "--"):
|
||||
end := strings.IndexByte(sql[i:], '\n')
|
||||
if end < 0 {
|
||||
end = len(sql)
|
||||
} else {
|
||||
end += i
|
||||
}
|
||||
out.WriteString(sql[i:end])
|
||||
i = end
|
||||
case strings.HasPrefix(sql[i:], "/*"):
|
||||
end := strings.Index(sql[i:], "*/")
|
||||
if end < 0 {
|
||||
end = len(sql)
|
||||
} else {
|
||||
end += i + 2
|
||||
}
|
||||
out.WriteString(sql[i:end])
|
||||
i = end
|
||||
case c == '$' && numberedRe.MatchString(sql[i:]):
|
||||
m := numberedRe.FindStringSubmatch(sql[i:])
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
if r, ok := repl[n]; ok {
|
||||
out.WriteString(r)
|
||||
} else {
|
||||
out.WriteString(m[0])
|
||||
}
|
||||
i += len(m[0])
|
||||
default:
|
||||
out.WriteByte(c)
|
||||
i++
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// observe runs the query with a value bound to each parameter, over the
|
||||
// fixture and over no rows, and reports which of the first n result
|
||||
// columns came back NULL in either run. A run the CLI rejects observes
|
||||
// nothing.
|
||||
func (a *analyzer) observe(ctx context.Context, sql string, phs []placeholder, bindings map[int]*binding, n int) []bool {
|
||||
nullable := make([]bool, n)
|
||||
statement := a.substitute(sql, phs, bindings, true) + ";\n"
|
||||
runs := []bool{true}
|
||||
if a.fixture != "" {
|
||||
runs = append(runs, false)
|
||||
}
|
||||
for _, withFixture := range runs {
|
||||
out, err := a.run(ctx, statement, "csv", withFixture)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
records, err := csv.NewReader(strings.NewReader(out)).ReadAll()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, rec := range records[min(1, len(records)):] {
|
||||
for j, v := range rec {
|
||||
if j < n && v == nullMarker {
|
||||
nullable[j] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullable
|
||||
}
|
||||
351
internal/goldeneye/duckdb/duckdb.go
Normal file
351
internal/goldeneye/duckdb/duckdb.go
Normal file
@@ -0,0 +1,351 @@
|
||||
// Package duckdb generates the DuckDB dialect seed under
|
||||
// internal/engine/duckdb/dialect — types.jsonl, functions.jsonl and
|
||||
// operators.jsonl — from a live DuckDB CLI, the same way the postgresql
|
||||
// package generates PostgreSQL's from a live server, and verifies the
|
||||
// DuckDB analyze cases under internal/endtoend/testdata against the same
|
||||
// CLI.
|
||||
//
|
||||
// The CLI is a DuckDB 2.0 build, the release darkwing is pinned against,
|
||||
// which has no release to download yet: Install fetches the current build
|
||||
// of DuckDB's v2.0 preview channel into the user cache directory. The CLI
|
||||
// is located through the DUCKDB environment variable, then the cached
|
||||
// build, then "duckdb" on PATH.
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
|
||||
)
|
||||
|
||||
// Engine is the name of the engine directory the dialect lives under.
|
||||
const Engine = "duckdb"
|
||||
|
||||
// Locate finds the DuckDB CLI: the DUCKDB environment variable wins, then
|
||||
// the cached build of DefaultVersion, then "duckdb" on PATH.
|
||||
func Locate() (string, error) {
|
||||
if path := os.Getenv("DUCKDB"); path != "" {
|
||||
return path, nil
|
||||
}
|
||||
if path, err := cachedBinary(DefaultVersion); err == nil {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
path, err := exec.LookPath("duckdb")
|
||||
if err != nil {
|
||||
return "", errors.New("no duckdb CLI found: run `go run ./cmd/goldeneye install duckdb` in internal/goldeneye, set DUCKDB to a DuckDB 2.0 binary, or put duckdb on PATH")
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Version reports the release a CLI is.
|
||||
func Version(ctx context.Context, binary string) (string, error) {
|
||||
out, err := exec.CommandContext(ctx, binary, "--version").Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("duckdb --version: %w", err)
|
||||
}
|
||||
return "DuckDB " + strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// query runs a SQL statement against an in-memory database and decodes the
|
||||
// CLI's JSON output into rows.
|
||||
func query(ctx context.Context, binary, sql string, rows any) error {
|
||||
cmd := exec.CommandContext(ctx, binary, "-json", ":memory:", "-c", sql)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
if exit, ok := err.(*exec.ExitError); ok {
|
||||
return fmt.Errorf("duckdb: %s: %s", err, exit.Stderr)
|
||||
}
|
||||
return fmt.Errorf("duckdb: %w", err)
|
||||
}
|
||||
// An empty result set prints nothing rather than [].
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(out, rows)
|
||||
}
|
||||
|
||||
type typeRow struct {
|
||||
TypeName string `json:"type_name"`
|
||||
LogicalType string `json:"logical_type"`
|
||||
Category *string `json:"type_category"`
|
||||
}
|
||||
|
||||
type functionRow struct {
|
||||
Name string `json:"function_name"`
|
||||
FunctionType string `json:"function_type"`
|
||||
ParameterTypes []string `json:"parameter_types"`
|
||||
Varargs *string `json:"varargs"`
|
||||
ReturnType *string `json:"return_type"`
|
||||
}
|
||||
|
||||
// metaTypes are type ids that never describe a column's value: sentinels and
|
||||
// binder-internal types the dump lists alongside the real ones.
|
||||
var metaTypes = map[string]bool{
|
||||
"null": true,
|
||||
"unknown": true,
|
||||
"invalid": true,
|
||||
"any": true,
|
||||
"type": true,
|
||||
"lambda": true,
|
||||
"table": true,
|
||||
"pointer": true,
|
||||
}
|
||||
|
||||
// categoryLetter maps duckdb_types() categories onto the PostgreSQL category
|
||||
// letters the seed package uses.
|
||||
func categoryLetter(category *string) string {
|
||||
if category == nil {
|
||||
return "U"
|
||||
}
|
||||
switch *category {
|
||||
case "NUMERIC":
|
||||
return "N"
|
||||
case "STRING":
|
||||
return "S"
|
||||
case "DATETIME":
|
||||
return "D"
|
||||
case "BOOLEAN":
|
||||
return "B"
|
||||
case "COMPOSITE":
|
||||
return "C"
|
||||
default:
|
||||
return "U"
|
||||
}
|
||||
}
|
||||
|
||||
func readTypes(ctx context.Context, binary string) ([]dialect.Type, error) {
|
||||
var rows []typeRow
|
||||
err := query(ctx, binary, `
|
||||
SELECT type_name, logical_type, type_category
|
||||
FROM duckdb_types()
|
||||
WHERE database_name = 'system'
|
||||
ORDER BY type_name`, &rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Group the dump's one-row-per-spelling by logical type: the spelling
|
||||
// matching the logical type id is the canonical name, the rest are
|
||||
// aliases. A spelling is listed once per schema it is visible in, so
|
||||
// an alias is kept once.
|
||||
grouped := map[string]*dialect.Type{}
|
||||
var order []string
|
||||
seen := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
logical := strings.ToLower(row.LogicalType)
|
||||
if metaTypes[logical] {
|
||||
continue
|
||||
}
|
||||
t, ok := grouped[logical]
|
||||
if !ok {
|
||||
t = &dialect.Type{Name: logical, Category: categoryLetter(row.Category)}
|
||||
grouped[logical] = t
|
||||
order = append(order, logical)
|
||||
}
|
||||
if t.Category == "U" {
|
||||
t.Category = categoryLetter(row.Category)
|
||||
}
|
||||
if name := strings.ToLower(row.TypeName); name != logical && !seen[logical+"\x00"+name] {
|
||||
seen[logical+"\x00"+name] = true
|
||||
t.Aliases = append(t.Aliases, name)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(order)
|
||||
types := make([]dialect.Type, 0, len(order)+1)
|
||||
// "any" stands in for the generic parameters of DuckDB's polymorphic
|
||||
// functions (ANY, T, K, V); the analyzer resolves a call returning it
|
||||
// to the type of the call's first argument.
|
||||
types = append(types, dialect.Type{Name: "any", Category: "U"})
|
||||
for _, name := range order {
|
||||
types = append(types, *grouped[name])
|
||||
}
|
||||
return types, nil
|
||||
}
|
||||
|
||||
// typeNames is the set of names the seed declares, for filtering out function
|
||||
// overloads over generic or binder-internal types.
|
||||
func typeNames(types []dialect.Type) map[string]bool {
|
||||
names := map[string]bool{}
|
||||
for _, t := range types {
|
||||
names[t.Name] = true
|
||||
for _, alias := range t.Aliases {
|
||||
names[alias] = true
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// seedTypeName maps a duckdb_functions() type spelling to a seeded type name:
|
||||
// lowercased, with modifiers dropped (DECIMAL(18,3) is a decimal), array
|
||||
// markers kept (fixed sizes and generic bounds included: DOUBLE[3] and
|
||||
// ANY[] are arrays), and the generic parameters of polymorphic functions
|
||||
// mapped to "any". It reports false for the internal types a seed cannot
|
||||
// describe.
|
||||
func seedTypeName(name string, known map[string]bool) (string, bool) {
|
||||
name = strings.ToLower(strings.TrimSpace(name))
|
||||
if close := strings.LastIndexByte(name, ']'); close == len(name)-1 {
|
||||
if open := strings.LastIndexByte(name, '['); open != -1 {
|
||||
elementName, ok := seedTypeName(name[:open], known)
|
||||
return elementName + "[]", ok
|
||||
}
|
||||
}
|
||||
if open := strings.IndexByte(name, '('); open != -1 {
|
||||
name = name[:open]
|
||||
}
|
||||
switch name {
|
||||
case "any", "t", "k", "v":
|
||||
return "any", true
|
||||
}
|
||||
if !known[name] {
|
||||
return "", false
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
// isOperatorName reports a function named by symbols rather than an
|
||||
// identifier — the spelling of a binary or prefix operator.
|
||||
func isOperatorName(name string) bool {
|
||||
return strings.IndexFunc(name, func(r rune) bool {
|
||||
return (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '$'
|
||||
}) != -1
|
||||
}
|
||||
|
||||
func functionKind(functionType string) string {
|
||||
switch functionType {
|
||||
case "aggregate":
|
||||
return "a"
|
||||
case "window":
|
||||
return "w"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func readFunctions(ctx context.Context, binary string, known map[string]bool) ([]dialect.Function, []dialect.Operator, error) {
|
||||
var rows []functionRow
|
||||
err := query(ctx, binary, `
|
||||
SELECT DISTINCT function_name, function_type, parameter_types, varargs, return_type
|
||||
FROM duckdb_functions()
|
||||
WHERE database_name = 'system'
|
||||
AND schema_name = 'main'
|
||||
AND function_type IN ('scalar', 'aggregate', 'window', 'macro')
|
||||
ORDER BY function_name, parameter_types::VARCHAR, return_type`, &rows)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var funcs []dialect.Function
|
||||
var operators []dialect.Operator
|
||||
seenFunc := map[string]bool{}
|
||||
seenOp := map[string]bool{}
|
||||
for _, row := range rows {
|
||||
if row.ReturnType == nil {
|
||||
continue
|
||||
}
|
||||
returns, ok := seedTypeName(*row.ReturnType, known)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
args := make([]string, 0, len(row.ParameterTypes))
|
||||
resolved := true
|
||||
for _, param := range row.ParameterTypes {
|
||||
arg, ok := seedTypeName(param, known)
|
||||
if !ok {
|
||||
resolved = false
|
||||
break
|
||||
}
|
||||
args = append(args, arg)
|
||||
}
|
||||
if !resolved {
|
||||
continue
|
||||
}
|
||||
|
||||
if isOperatorName(row.Name) {
|
||||
// Operator-named functions become operator entries; sqlc's
|
||||
// engine reports them as operator applications. Only the
|
||||
// binary form over concrete scalar types is representable —
|
||||
// the seed's operator table, unlike its function table, does
|
||||
// not register generic or array types on demand.
|
||||
if row.FunctionType != "scalar" || len(args) != 2 || row.Varargs != nil {
|
||||
continue
|
||||
}
|
||||
if concrete := !strings.Contains(returns, "any") && !strings.Contains(returns, "[]") &&
|
||||
!strings.Contains(args[0], "any") && !strings.Contains(args[0], "[]") &&
|
||||
!strings.Contains(args[1], "any") && !strings.Contains(args[1], "[]"); !concrete {
|
||||
continue
|
||||
}
|
||||
key := row.Name + "\x00" + strings.Join(args, "\x00")
|
||||
if seenOp[key] {
|
||||
continue
|
||||
}
|
||||
seenOp[key] = true
|
||||
operators = append(operators, dialect.Operator{
|
||||
Name: row.Name,
|
||||
Left: args[0],
|
||||
Right: args[1],
|
||||
Result: returns,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
fn := dialect.Function{
|
||||
Name: row.Name,
|
||||
Kind: functionKind(row.FunctionType),
|
||||
Returns: returns,
|
||||
// An aggregate over no rows returns NULL — except count,
|
||||
// which returns 0.
|
||||
Nullable: row.FunctionType == "aggregate" && !strings.HasPrefix(row.Name, "count"),
|
||||
}
|
||||
for _, arg := range args {
|
||||
fn.Args = append(fn.Args, dialect.Arg{Type: arg})
|
||||
}
|
||||
if row.Varargs != nil {
|
||||
vararg, ok := seedTypeName(*row.Varargs, known)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fn.Args = append(fn.Args, dialect.Arg{Type: vararg, Mode: "v"})
|
||||
}
|
||||
|
||||
key := fn.Name + "\x00" + fn.Kind + "\x00" + strings.Join(args, "\x00")
|
||||
if seenFunc[key] {
|
||||
continue
|
||||
}
|
||||
seenFunc[key] = true
|
||||
funcs = append(funcs, fn)
|
||||
}
|
||||
return funcs, operators, nil
|
||||
}
|
||||
|
||||
// Generate reads the dialect from the CLI.
|
||||
func Generate(ctx context.Context, binary string) (dialect.Files, error) {
|
||||
types, err := readTypes(ctx, binary)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
funcs, operators, err := readFunctions(ctx, binary, typeNames(types))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
files := dialect.Files{}
|
||||
if files[dialect.TypesFile], err = dialect.JSONL(types); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if files[dialect.FunctionsFile], err = dialect.JSONL(funcs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if files[dialect.OperatorsFile], err = dialect.JSONL(operators); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
66
internal/goldeneye/duckdb/duckdb_test.go
Normal file
66
internal/goldeneye/duckdb/duckdb_test.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend"
|
||||
)
|
||||
|
||||
// TestDialect verifies the committed DuckDB dialect against what the DuckDB
|
||||
// CLI reports. It skips unless a CLI is found.
|
||||
func TestDialect(t *testing.T) {
|
||||
binary, err := Locate()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
version, err := Version(ctx, binary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, err := Generate(ctx, binary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, err := dialect.Dir(Engine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, err := dialect.Check(dir, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report != "" {
|
||||
t.Errorf("%s does not match what %s reports:\n%s", dir, version, report)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyzeCases verifies every DuckDB analyze case under
|
||||
// internal/endtoend/testdata against what the CLI reports. It skips
|
||||
// unless a CLI is found.
|
||||
func TestAnalyzeCases(t *testing.T) {
|
||||
binary, err := Locate()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
cases, err := endtoend.Cases(Engine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cases) == 0 {
|
||||
t.Fatal("no duckdb analyze cases found")
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.Name, func(t *testing.T) {
|
||||
diff, err := Check(context.Background(), binary, c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if diff != "" {
|
||||
t.Errorf("%s does not match what DuckDB reports (-committed +duckdb):\n%s", c.Output, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
136
internal/goldeneye/duckdb/install.go
Normal file
136
internal/goldeneye/duckdb/install.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultVersion is the DuckDB preview channel the CLI is downloaded from:
|
||||
// the v2.0 development builds, which DuckDB publishes as a rolling tarball
|
||||
// per platform under https://artifacts.duckdb.org/<channel>/ until 2.0 is
|
||||
// released. A channel has no per-build download and no checksum to pin,
|
||||
// so Install fetches whatever build the channel holds and the version the
|
||||
// CLI reports is what a run logs; GeneratedFrom records the build the
|
||||
// committed dialect came from, and a check against a later build reports
|
||||
// what the later build added. Once 2.0 is released, pin the release and
|
||||
// its checksums here the way the clickhouse package does.
|
||||
const DefaultVersion = "v2.0-cyanoptera"
|
||||
|
||||
// GeneratedFrom is the build the committed dialect was generated from, as
|
||||
// `duckdb --version` reports it. Update it when regenerating.
|
||||
const GeneratedFrom = "v2.0.0-alpha41396 (Cyanoptera) d41e527b18"
|
||||
|
||||
// channelURL is the download address of a channel's CLI tarball for a
|
||||
// platform. DuckDB publishes one macOS build for both architectures.
|
||||
func channelURL(channel, goos, goarch string) (string, error) {
|
||||
var name string
|
||||
switch {
|
||||
case goos == "linux" && (goarch == "amd64" || goarch == "arm64"):
|
||||
name = "duckdb-cli-linux-" + goarch + ".tar.gz"
|
||||
case goos == "darwin":
|
||||
name = "duckdb-cli-osx-universal.tar.gz"
|
||||
default:
|
||||
return "", fmt.Errorf("no DuckDB %s build is published for %s/%s", channel, goos, goarch)
|
||||
}
|
||||
return "https://artifacts.duckdb.org/" + channel + "/" + name, nil
|
||||
}
|
||||
|
||||
// cachedBinary is where Install puts the binary for a channel.
|
||||
func cachedBinary(channel string) (string, error) {
|
||||
dir, err := os.UserCacheDir()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(dir, "sqlc-duckdb", channel, "duckdb"), nil
|
||||
}
|
||||
|
||||
// Install downloads the channel's current CLI into the cache and returns
|
||||
// its path. It is a no-op when the channel is already cached: remove the
|
||||
// cached directory to fetch the channel's newer build.
|
||||
func Install(ctx context.Context, channel, goos, goarch string, progress io.Writer) (string, error) {
|
||||
dest, err := cachedBinary(channel)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(dest); err == nil {
|
||||
return dest, nil
|
||||
}
|
||||
url, err := channelURL(channel, goos, goarch)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
fmt.Fprintf(progress, "downloading %s\n", url)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("downloading %s: %s", url, resp.Status)
|
||||
}
|
||||
|
||||
// Write next to the destination and rename so a partial download never
|
||||
// masquerades as an installed binary.
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dest), "duckdb-*.partial")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
src, err := binaryInTarball(resp.Body)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
return "", fmt.Errorf("downloading %s: %w", url, err)
|
||||
}
|
||||
if _, err := io.Copy(tmp, src); err != nil {
|
||||
tmp.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Chmod(tmp.Name(), 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), dest); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dest, nil
|
||||
}
|
||||
|
||||
// binaryInTarball positions a reader at the duckdb binary inside a CLI
|
||||
// tarball, which holds it at the top level.
|
||||
func binaryInTarball(r io.Reader) (io.Reader, error) {
|
||||
gz, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return nil, errors.New("tarball does not contain duckdb")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if hdr.Typeflag == tar.TypeReg && strings.TrimPrefix(hdr.Name, "./") == "duckdb" {
|
||||
return tr, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
657
internal/goldeneye/duckdb/text.go
Normal file
657
internal/goldeneye/duckdb/text.go
Normal file
@@ -0,0 +1,657 @@
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DuckDB prints a plan with every column by its bare name and every
|
||||
// aliased expression by its alias, and describes a query no further than
|
||||
// its result columns' names and types, so which table a result column is
|
||||
// read from and which column a parameter is compared with are read from
|
||||
// the query text: the select list's items, the FROM clause's tables and
|
||||
// aliases, and the operand beside each parameter. The text is tokenized
|
||||
// and read at the top level of the statement, outside the parentheses a
|
||||
// subquery or a CTE body sits in.
|
||||
|
||||
// token is one lexical element of a query.
|
||||
type token struct {
|
||||
kind byte // 'w' word, 'q' quoted identifier, 's' string, 'n' number, 'p' parameter, 'o' operator or punctuation
|
||||
text string
|
||||
}
|
||||
|
||||
var operators = []string{"::", "<>", "!=", "<=", ">=", "!~~", "~~", "->>", "->", "||", "=", "<", ">", "(", ")", ",", ".", "*", "+", "-", "/", "%", "[", "]", ";", "^", "&", "|", "~", "!", ":"}
|
||||
|
||||
// tokenize splits a query into tokens, dropping comments. A parameter is
|
||||
// $ followed by digits, which is how bind spells every parameter.
|
||||
func tokenize(src string) []token {
|
||||
var out []token
|
||||
i := 0
|
||||
for i < len(src) {
|
||||
c := src[i]
|
||||
switch {
|
||||
case c == ' ' || c == '\t' || c == '\n' || c == '\r':
|
||||
i++
|
||||
case strings.HasPrefix(src[i:], "--"):
|
||||
end := strings.IndexByte(src[i:], '\n')
|
||||
if end < 0 {
|
||||
i = len(src)
|
||||
} else {
|
||||
i += end
|
||||
}
|
||||
case strings.HasPrefix(src[i:], "/*"):
|
||||
end := strings.Index(src[i:], "*/")
|
||||
if end < 0 {
|
||||
i = len(src)
|
||||
} else {
|
||||
i += end + 2
|
||||
}
|
||||
case c == '\'':
|
||||
end := quotedEnd(src, i)
|
||||
out = append(out, token{'s', src[i:end]})
|
||||
i = end
|
||||
case c == '"':
|
||||
end := quotedEnd(src, i)
|
||||
out = append(out, token{'q', strings.ReplaceAll(src[i+1:end-1], `""`, `"`)})
|
||||
i = end
|
||||
case c == '$' && i+1 < len(src) && isDigit(src[i+1]):
|
||||
end := i + 1
|
||||
for end < len(src) && isDigit(src[end]) {
|
||||
end++
|
||||
}
|
||||
out = append(out, token{'p', src[i+1 : end]})
|
||||
i = end
|
||||
case isDigit(c) || (c == '.' && i+1 < len(src) && isDigit(src[i+1])):
|
||||
end := i
|
||||
for end < len(src) && (isDigit(src[end]) || src[end] == '.' || src[end] == 'e' || src[end] == 'E' || src[end] == '_') {
|
||||
end++
|
||||
}
|
||||
out = append(out, token{'n', src[i:end]})
|
||||
i = end
|
||||
case isWordStart(c):
|
||||
end := i
|
||||
for end < len(src) && isWordByte(src[end]) {
|
||||
end++
|
||||
}
|
||||
out = append(out, token{'w', src[i:end]})
|
||||
i = end
|
||||
default:
|
||||
matched := false
|
||||
for _, op := range operators {
|
||||
if strings.HasPrefix(src[i:], op) {
|
||||
out = append(out, token{'o', op})
|
||||
i += len(op)
|
||||
matched = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
out = append(out, token{'o', string(c)})
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// quotedEnd returns the index just past the quoted token starting at i,
|
||||
// whose delimiter is escaped by doubling it.
|
||||
func quotedEnd(s string, i int) int {
|
||||
q := s[i]
|
||||
j := i + 1
|
||||
for j < len(s) {
|
||||
switch {
|
||||
case s[j] == q && j+1 < len(s) && s[j+1] == q:
|
||||
j += 2
|
||||
case s[j] == q:
|
||||
return j + 1
|
||||
default:
|
||||
j++
|
||||
}
|
||||
}
|
||||
return len(s)
|
||||
}
|
||||
|
||||
func isDigit(c byte) bool { return c >= '0' && c <= '9' }
|
||||
func isWordStart(c byte) bool {
|
||||
return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= 0x80
|
||||
}
|
||||
func isWordByte(c byte) bool { return isWordStart(c) || isDigit(c) }
|
||||
|
||||
// text is a tokenized query.
|
||||
type text []token
|
||||
|
||||
func (t text) at(i int) token {
|
||||
if i < 0 || i >= len(t) {
|
||||
return token{}
|
||||
}
|
||||
return t[i]
|
||||
}
|
||||
|
||||
// isWord reports whether token i is the keyword, in any case.
|
||||
func (t text) isWord(i int, word string) bool {
|
||||
tok := t.at(i)
|
||||
return tok.kind == 'w' && strings.EqualFold(tok.text, word)
|
||||
}
|
||||
|
||||
// isOp reports whether token i is the operator.
|
||||
func (t text) isOp(i int, op string) bool {
|
||||
tok := t.at(i)
|
||||
return tok.kind == 'o' && tok.text == op
|
||||
}
|
||||
|
||||
// isName reports whether token i can name something: a word or a quoted
|
||||
// identifier.
|
||||
func (t text) isName(i int) bool {
|
||||
k := t.at(i).kind
|
||||
return k == 'w' || k == 'q'
|
||||
}
|
||||
|
||||
// group returns the index of the parenthesis closing the one at i.
|
||||
func (t text) group(i int) int {
|
||||
depth := 0
|
||||
for j := i; j < len(t); j++ {
|
||||
switch {
|
||||
case t.isOp(j, "("):
|
||||
depth++
|
||||
case t.isOp(j, ")"):
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return j
|
||||
}
|
||||
}
|
||||
}
|
||||
return len(t) - 1
|
||||
}
|
||||
|
||||
// ref is a column reference: a qualifier, empty for none, and a column,
|
||||
// or "*" for every column of the qualifier.
|
||||
type ref struct {
|
||||
qualifier, column string
|
||||
}
|
||||
|
||||
// readRef reads a dotted name starting at i, returning the parts and the
|
||||
// index after them.
|
||||
func (t text) readRef(i int) ([]string, int) {
|
||||
var parts []string
|
||||
for t.isName(i) || t.isOp(i, "*") {
|
||||
parts = append(parts, t.at(i).text)
|
||||
if !t.isOp(i+1, ".") {
|
||||
return parts, i + 1
|
||||
}
|
||||
i += 2
|
||||
}
|
||||
return parts, i
|
||||
}
|
||||
|
||||
// keywords that end a list of tables or select items at the top level.
|
||||
var clauseKeywords = map[string]bool{
|
||||
"where": true, "group": true, "order": true, "limit": true, "offset": true, "having": true,
|
||||
"qualify": true, "window": true, "returning": true, "set": true, "values": true, "select": true,
|
||||
"union": true, "except": true, "intersect": true, "on": true, "from": true, "join": true,
|
||||
"left": true, "right": true, "full": true, "inner": true, "cross": true, "outer": true,
|
||||
"natural": true, "asof": true, "semi": true, "anti": true, "positional": true, "lateral": true,
|
||||
"using": true, "with": true, "into": true, "default": true, "fetch": true, "for": true,
|
||||
"tablesample": true, "sample": true, "by": true, "repeatable": true,
|
||||
}
|
||||
|
||||
// typeWords are the words a multi-word type name continues with:
|
||||
// TIMESTAMP WITH TIME ZONE, DOUBLE PRECISION, CHARACTER VARYING.
|
||||
var typeWords = map[string]bool{"with": true, "without": true, "time": true, "zone": true, "precision": true, "varying": true}
|
||||
|
||||
// tableRef is one table of a statement's scope, by name and alias.
|
||||
type tableRef struct {
|
||||
name, alias string
|
||||
}
|
||||
|
||||
// scope is what the statement reads and writes: the tables of its FROM,
|
||||
// JOIN and USING clauses and the table a DML statement targets, with the
|
||||
// names of its CTEs, which are not tables.
|
||||
type scope struct {
|
||||
kind string // select, insert, update, delete
|
||||
target tableRef
|
||||
tables []tableRef
|
||||
ctes map[string]bool
|
||||
}
|
||||
|
||||
// readScope reads a statement's scope from its top-level tokens.
|
||||
func (t text) readScope() scope {
|
||||
sc := scope{ctes: map[string]bool{}}
|
||||
i := 0
|
||||
// WITH name [(cols)] AS (...), ...
|
||||
if t.isWord(0, "with") {
|
||||
i = 1
|
||||
if t.isWord(i, "recursive") {
|
||||
i++
|
||||
}
|
||||
for t.isName(i) {
|
||||
sc.ctes[strings.ToLower(t.at(i).text)] = true
|
||||
i++
|
||||
if t.isOp(i, "(") {
|
||||
i = t.group(i) + 1
|
||||
}
|
||||
if t.isWord(i, "as") {
|
||||
i++
|
||||
}
|
||||
if t.isWord(i, "not") || t.isWord(i, "materialized") {
|
||||
for !t.isOp(i, "(") && i < len(t) {
|
||||
i++
|
||||
}
|
||||
}
|
||||
if t.isOp(i, "(") {
|
||||
i = t.group(i) + 1
|
||||
}
|
||||
if t.isOp(i, ",") {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
sc.kind = "select"
|
||||
switch {
|
||||
case t.isWord(i, "insert"):
|
||||
sc.kind = "insert"
|
||||
for i < len(t) && !t.isWord(i, "into") {
|
||||
i++
|
||||
}
|
||||
sc.target, i = t.readTable(i + 1)
|
||||
case t.isWord(i, "update"):
|
||||
sc.kind = "update"
|
||||
sc.target, i = t.readTable(i + 1)
|
||||
case t.isWord(i, "delete"):
|
||||
sc.kind = "delete"
|
||||
if t.isWord(i+1, "from") {
|
||||
i++
|
||||
}
|
||||
sc.target, i = t.readTable(i + 1)
|
||||
}
|
||||
for ; i < len(t); i++ {
|
||||
switch {
|
||||
case t.isOp(i, "("):
|
||||
i = t.group(i)
|
||||
case t.isWord(i, "from") || t.isWord(i, "join"):
|
||||
i = t.readTables(i+1, &sc.tables) - 1
|
||||
case t.isWord(i, "using") && !t.isOp(i+1, "("):
|
||||
i = t.readTables(i+1, &sc.tables) - 1
|
||||
}
|
||||
}
|
||||
// A CTE is not a table, whatever table shares its name.
|
||||
tables := sc.tables[:0]
|
||||
for _, ref := range sc.tables {
|
||||
if !sc.ctes[ref.name] {
|
||||
tables = append(tables, ref)
|
||||
}
|
||||
}
|
||||
sc.tables = tables
|
||||
return sc
|
||||
}
|
||||
|
||||
// readTable reads a table name with an optional alias.
|
||||
func (t text) readTable(i int) (tableRef, int) {
|
||||
parts, j := t.readRef(i)
|
||||
if len(parts) == 0 {
|
||||
return tableRef{}, i
|
||||
}
|
||||
ref := tableRef{name: strings.ToLower(parts[len(parts)-1])}
|
||||
if t.isWord(j, "as") {
|
||||
j++
|
||||
}
|
||||
if t.isName(j) && !clauseKeywords[strings.ToLower(t.at(j).text)] {
|
||||
ref.alias = strings.ToLower(t.at(j).text)
|
||||
j++
|
||||
}
|
||||
return ref, j
|
||||
}
|
||||
|
||||
// readTables reads a comma-separated list of table references, skipping
|
||||
// subqueries and table functions, and returns the index after it.
|
||||
func (t text) readTables(i int, into *[]tableRef) int {
|
||||
for i < len(t) {
|
||||
if t.isOp(i, "(") {
|
||||
// A derived table or a table function's arguments.
|
||||
i = t.group(i) + 1
|
||||
if t.isWord(i, "as") {
|
||||
i++
|
||||
}
|
||||
if t.isName(i) && !clauseKeywords[strings.ToLower(t.at(i).text)] {
|
||||
i++
|
||||
}
|
||||
} else if t.isName(i) && !clauseKeywords[strings.ToLower(t.at(i).text)] {
|
||||
ref, j := t.readTable(i)
|
||||
if t.isOp(j, "(") {
|
||||
// A table function: its result is not a table.
|
||||
j = t.group(j) + 1
|
||||
} else {
|
||||
*into = append(*into, ref)
|
||||
}
|
||||
i = j
|
||||
} else {
|
||||
return i
|
||||
}
|
||||
if t.isOp(i, ",") {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
return i
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
// item is one entry of a select list or a RETURNING list.
|
||||
type item struct {
|
||||
star bool // every column of qualifier, or of every table
|
||||
ref *ref // a column reference, when the item is one
|
||||
name string // the alias, or the column name of a reference
|
||||
}
|
||||
|
||||
// items splits a select or RETURNING list into its items, given the index
|
||||
// of the first token after the keyword.
|
||||
func (t text) items(start int) []item {
|
||||
var out []item
|
||||
i := start
|
||||
for i < len(t) {
|
||||
end := i
|
||||
for end < len(t) && !t.isOp(end, ",") && !(t.at(end).kind == 'w' && clauseKeywords[strings.ToLower(t.at(end).text)] && !t.isWord(end, "on")) && !t.isOp(end, ";") {
|
||||
if t.isOp(end, "(") {
|
||||
end = t.group(end)
|
||||
}
|
||||
end++
|
||||
}
|
||||
if end > i {
|
||||
out = append(out, t.item(i, end))
|
||||
}
|
||||
if !t.isOp(end, ",") {
|
||||
break
|
||||
}
|
||||
i = end + 1
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// item classifies the tokens of one select item.
|
||||
func (t text) item(start, end int) item {
|
||||
parts, j := t.readRef(start)
|
||||
if len(parts) > 0 && parts[len(parts)-1] == "*" {
|
||||
it := item{star: true}
|
||||
if len(parts) > 1 {
|
||||
it.ref = &ref{qualifier: strings.ToLower(parts[len(parts)-2]), column: "*"}
|
||||
}
|
||||
return it
|
||||
}
|
||||
var it item
|
||||
if len(parts) > 0 && j == end {
|
||||
it.ref = &ref{column: parts[len(parts)-1]}
|
||||
if len(parts) > 1 {
|
||||
it.ref.qualifier = strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
it.name = parts[len(parts)-1]
|
||||
return it
|
||||
}
|
||||
if len(parts) > 0 && j == end-1 && t.isName(end-1) {
|
||||
// A reference followed by a bare alias.
|
||||
it.ref = &ref{column: parts[len(parts)-1]}
|
||||
if len(parts) > 1 {
|
||||
it.ref.qualifier = strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
it.name = t.at(end - 1).text
|
||||
return it
|
||||
}
|
||||
if len(parts) > 0 && j == end-2 && t.isWord(end-2, "as") && t.isName(end-1) {
|
||||
it.ref = &ref{column: parts[len(parts)-1]}
|
||||
if len(parts) > 1 {
|
||||
it.ref.qualifier = strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
it.name = t.at(end - 1).text
|
||||
return it
|
||||
}
|
||||
if t.isWord(end-2, "as") && t.isName(end-1) {
|
||||
it.name = t.at(end - 1).text
|
||||
}
|
||||
return it
|
||||
}
|
||||
|
||||
// selectItems finds the statement's select list: the items after its
|
||||
// top-level SELECT, or every column when a FROM-first query has none.
|
||||
func (t text) selectItems() []item {
|
||||
for i := 0; i < len(t); i++ {
|
||||
switch {
|
||||
case t.isOp(i, "("):
|
||||
i = t.group(i)
|
||||
case t.isWord(i, "select"):
|
||||
j := i + 1
|
||||
if t.isWord(j, "distinct") || t.isWord(j, "all") {
|
||||
j++
|
||||
if t.isWord(j, "on") && t.isOp(j+1, "(") {
|
||||
j = t.group(j+1) + 1
|
||||
}
|
||||
}
|
||||
return t.items(j)
|
||||
}
|
||||
}
|
||||
if t.isWord(0, "from") {
|
||||
return []item{{star: true}}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// returningItems finds the items of a top-level RETURNING clause.
|
||||
func (t text) returningItems() []item {
|
||||
for i := 0; i < len(t); i++ {
|
||||
switch {
|
||||
case t.isOp(i, "("):
|
||||
i = t.group(i)
|
||||
case t.isWord(i, "returning"):
|
||||
return t.items(i + 1)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// inSubquery reports whether parameter k sits inside a subquery: a
|
||||
// parenthesis that a SELECT, WITH, FROM or VALUES opens.
|
||||
func (t text) inSubquery(k string) bool {
|
||||
i := t.find(k)
|
||||
depth := 0
|
||||
for j := i - 1; j >= 0; j-- {
|
||||
switch {
|
||||
case t.isOp(j, ")"):
|
||||
depth++
|
||||
case t.isOp(j, "("):
|
||||
if depth > 0 {
|
||||
depth--
|
||||
continue
|
||||
}
|
||||
if t.isWord(j+1, "select") || t.isWord(j+1, "with") || t.isWord(j+1, "from") || t.isWord(j+1, "values") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// find returns the index of parameter k.
|
||||
func (t text) find(k string) int {
|
||||
for i, tok := range t {
|
||||
if tok.kind == 'p' && tok.text == k {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
var comparisons = map[string]bool{"=": true, "<>": true, "!=": true, "<": true, ">": true, "<=": true, ">=": true, "~~": true, "!~~": true}
|
||||
|
||||
// partner finds the column a parameter is compared with or assigned to:
|
||||
// the column reference on the other side of the comparison, LIKE or IN
|
||||
// it is an operand of, or the column a SET assigns it to.
|
||||
func (t text) partner(k string) (ref, bool) {
|
||||
i := t.find(k)
|
||||
if i < 0 {
|
||||
return ref{}, false
|
||||
}
|
||||
// col op $k, col [NOT] LIKE $k, col IN ($k, ...)
|
||||
j := i - 1
|
||||
if t.isOp(j, "(") || t.isOp(j, ",") {
|
||||
for j >= 0 && (t.isOp(j, ",") || t.at(j).kind == 'p' || t.at(j).kind == 's' || t.at(j).kind == 'n') {
|
||||
j--
|
||||
}
|
||||
if t.isOp(j, "(") {
|
||||
j--
|
||||
}
|
||||
}
|
||||
if t.at(j).kind == 'o' && comparisons[t.at(j).text] || t.isWord(j, "like") || t.isWord(j, "ilike") || t.isWord(j, "glob") || t.isWord(j, "in") {
|
||||
if t.isWord(j-1, "not") {
|
||||
j--
|
||||
}
|
||||
if r, ok := t.refEnding(j - 1); ok {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
// $k op col
|
||||
j = i + 1
|
||||
if t.at(j).kind == 'o' && comparisons[t.at(j).text] {
|
||||
if r, ok := t.refStarting(j + 1); ok {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return ref{}, false
|
||||
}
|
||||
|
||||
// refEnding reads the column reference whose last token is at i.
|
||||
func (t text) refEnding(i int) (ref, bool) {
|
||||
if !t.isName(i) {
|
||||
return ref{}, false
|
||||
}
|
||||
r := ref{column: t.at(i).text}
|
||||
if t.isOp(i-1, ".") && t.isName(i-2) {
|
||||
r.qualifier = strings.ToLower(t.at(i - 2).text)
|
||||
}
|
||||
return r, true
|
||||
}
|
||||
|
||||
// refStarting reads the column reference starting at i, which must not be
|
||||
// a function call.
|
||||
func (t text) refStarting(i int) (ref, bool) {
|
||||
parts, j := t.readRef(i)
|
||||
if len(parts) == 0 || t.isOp(j, "(") {
|
||||
return ref{}, false
|
||||
}
|
||||
r := ref{column: parts[len(parts)-1]}
|
||||
if len(parts) > 1 {
|
||||
r.qualifier = strings.ToLower(parts[len(parts)-2])
|
||||
}
|
||||
return r, true
|
||||
}
|
||||
|
||||
// castOf returns the type a parameter is cast to, as $k::T or CAST($k AS
|
||||
// T), spelled as the query spells it.
|
||||
func (t text) castOf(k string) (string, bool) {
|
||||
i := t.find(k)
|
||||
if i < 0 {
|
||||
return "", false
|
||||
}
|
||||
if t.isOp(i+1, "::") {
|
||||
return t.typeText(i + 2), true
|
||||
}
|
||||
if t.isWord(i-2, "cast") && t.isOp(i-1, "(") && t.isWord(i+1, "as") {
|
||||
return t.typeText(i + 2), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// typeText reads a type starting at i: a dotted name, optional arguments
|
||||
// in parentheses and any number of [] or [N] suffixes.
|
||||
func (t text) typeText(i int) string {
|
||||
start := i
|
||||
_, i = t.readRef(i)
|
||||
for t.at(i).kind == 'w' && typeWords[strings.ToLower(t.at(i).text)] {
|
||||
// Multi-word names: TIMESTAMP WITH TIME ZONE.
|
||||
i++
|
||||
}
|
||||
if t.isOp(i, "(") {
|
||||
i = t.group(i) + 1
|
||||
}
|
||||
for t.isOp(i, "[") {
|
||||
for i < len(t) && !t.isOp(i, "]") {
|
||||
i++
|
||||
}
|
||||
i++
|
||||
}
|
||||
var b strings.Builder
|
||||
for j := start; j < i && j < len(t); j++ {
|
||||
tok := t.at(j)
|
||||
if j > start && (tok.kind == 'w' || tok.kind == 'q' || tok.kind == 'n' || tok.kind == 's') && (t.at(j-1).kind == 'w' || t.at(j-1).kind == 'q' || t.isOp(j-1, ",")) {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
b.WriteString(tok.text)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// valuesPosition reports, for a parameter inside an INSERT's VALUES, the
|
||||
// index of the column it is a value for, counting positions within its
|
||||
// row.
|
||||
func (t text) valuesPosition(k string) (int, bool) {
|
||||
i := t.find(k)
|
||||
if i < 0 {
|
||||
return 0, false
|
||||
}
|
||||
for j := 0; j < len(t); j++ {
|
||||
if t.isOp(j, "(") {
|
||||
j = t.group(j)
|
||||
continue
|
||||
}
|
||||
if !t.isWord(j, "values") {
|
||||
continue
|
||||
}
|
||||
// Rows follow, each a parenthesised list.
|
||||
for r := j + 1; t.isOp(r, "("); r = t.group(r) + 2 {
|
||||
end := t.group(r)
|
||||
if i < r || i > end {
|
||||
if !t.isOp(end+1, ",") {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
pos, depth := 0, 0
|
||||
for q := r + 1; q < i; q++ {
|
||||
switch {
|
||||
case t.isOp(q, "("):
|
||||
depth++
|
||||
case t.isOp(q, ")"):
|
||||
depth--
|
||||
case t.isOp(q, ",") && depth == 0:
|
||||
pos++
|
||||
}
|
||||
}
|
||||
return pos, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// insertColumns lists the columns an INSERT names, or nil for all.
|
||||
func (t text) insertColumns() []string {
|
||||
for i := 0; i < len(t); i++ {
|
||||
if t.isWord(i, "into") {
|
||||
_, j := t.readRef(i + 1)
|
||||
if t.isWord(j, "as") {
|
||||
j += 2
|
||||
} else if t.isName(j) && !clauseKeywords[strings.ToLower(t.at(j).text)] {
|
||||
j++
|
||||
}
|
||||
if !t.isOp(j, "(") {
|
||||
return nil
|
||||
}
|
||||
var cols []string
|
||||
for q := j + 1; q < t.group(j); q++ {
|
||||
if t.isName(q) {
|
||||
cols = append(cols, t.at(q).text)
|
||||
}
|
||||
}
|
||||
return cols
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
220
internal/goldeneye/duckdb/types.go
Normal file
220
internal/goldeneye/duckdb/types.go
Normal file
@@ -0,0 +1,220 @@
|
||||
package duckdb
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/sqlc-dev/sqlc/internal/goldeneye/analysis"
|
||||
)
|
||||
|
||||
// parseType reads a type the way DuckDB spells one — DECIMAL(10,2),
|
||||
// INTEGER[], INTEGER[3], STRUCT(a INTEGER, b VARCHAR), MAP(VARCHAR,
|
||||
// INTEGER), UNION(num INTEGER, str VARCHAR), ENUM('a', 'b'), TIMESTAMP
|
||||
// WITH TIME ZONE — into an expression in lower case, named the way the
|
||||
// dialect names it when the spelling is one of the aliases types.jsonl
|
||||
// lists. DuckDB spells an enum column by its labels whether the schema
|
||||
// named the type or not, so labels that are those of an enum the schema
|
||||
// created name that type.
|
||||
func (a *analyzer) parseType(s string) *analysis.TypeExpr {
|
||||
return a.parseTypeWith(s, true)
|
||||
}
|
||||
|
||||
// parseTypeWith reads a type, naming an alias as the dialect does when
|
||||
// canonical is set and as DuckDB spells it otherwise.
|
||||
func (a *analyzer) parseTypeWith(s string, canonical bool) *analysis.TypeExpr {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasSuffix(s, "]") {
|
||||
open := strings.LastIndexByte(s, '[')
|
||||
if open > 0 {
|
||||
t := &analysis.TypeExpr{Name: "array", Args: []analysis.TypeArg{{Type: a.parseTypeWith(s[:open], canonical)}}}
|
||||
if n, err := strconv.ParseInt(strings.TrimSpace(s[open+1:len(s)-1]), 10, 64); err == nil {
|
||||
t.Args = append(t.Args, analysis.TypeArg{Int: &n})
|
||||
}
|
||||
return t
|
||||
}
|
||||
}
|
||||
name, args := s, ""
|
||||
if open := strings.IndexByte(s, '('); open >= 0 && strings.HasSuffix(s, ")") {
|
||||
name, args = strings.TrimSpace(s[:open]), s[open+1:len(s)-1]
|
||||
}
|
||||
name = strings.ToLower(name)
|
||||
if c, ok := a.canonical[name]; ok && canonical {
|
||||
name = c
|
||||
}
|
||||
t := &analysis.TypeExpr{Name: name}
|
||||
if args == "" {
|
||||
return t
|
||||
}
|
||||
switch name {
|
||||
case "struct", "union":
|
||||
for _, f := range splitTop(args, ',') {
|
||||
f = strings.TrimSpace(f)
|
||||
label, typ := f, ""
|
||||
if strings.HasPrefix(f, `"`) {
|
||||
end := quotedEnd(f, 0)
|
||||
label, typ = strings.ReplaceAll(f[1:end-1], `""`, `"`), f[end:]
|
||||
} else if i := strings.IndexByte(f, ' '); i > 0 {
|
||||
label, typ = f[:i], f[i+1:]
|
||||
}
|
||||
t.Args = append(t.Args, analysis.TypeArg{Label: label, Type: a.parseTypeWith(typ, canonical)})
|
||||
}
|
||||
case "enum":
|
||||
var labels []string
|
||||
for _, l := range splitTop(args, ',') {
|
||||
l = strings.TrimSpace(l)
|
||||
if strings.HasPrefix(l, "'") && strings.HasSuffix(l, "'") && len(l) >= 2 {
|
||||
l = strings.ReplaceAll(l[1:len(l)-1], "''", "'")
|
||||
}
|
||||
labels = append(labels, l)
|
||||
}
|
||||
if named, ok := a.enumNamed(labels); ok {
|
||||
return &analysis.TypeExpr{Name: named}
|
||||
}
|
||||
for _, l := range labels {
|
||||
l := l
|
||||
t.Args = append(t.Args, analysis.TypeArg{String: &l})
|
||||
}
|
||||
default:
|
||||
for _, arg := range splitTop(args, ',') {
|
||||
arg = strings.TrimSpace(arg)
|
||||
if n, err := strconv.ParseInt(arg, 10, 64); err == nil {
|
||||
t.Args = append(t.Args, analysis.TypeArg{Int: &n})
|
||||
} else {
|
||||
t.Args = append(t.Args, analysis.TypeArg{Type: a.parseTypeWith(arg, canonical)})
|
||||
}
|
||||
}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// enumNamed finds the enum type the schema created with these labels,
|
||||
// the first by name when two share them.
|
||||
func (a *analyzer) enumNamed(labels []string) (string, bool) {
|
||||
names := make([]string, 0, len(a.enums))
|
||||
for name := range a.enums {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
have := a.enums[name]
|
||||
if len(have) != len(labels) {
|
||||
continue
|
||||
}
|
||||
same := true
|
||||
for i := range have {
|
||||
if have[i] != labels[i] {
|
||||
same = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if same {
|
||||
return name, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// splitTop splits on a separator outside parentheses, brackets and
|
||||
// quotes.
|
||||
func splitTop(s string, sep byte) []string {
|
||||
var out []string
|
||||
depth, start := 0, 0
|
||||
var quote byte
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
switch {
|
||||
case quote != 0:
|
||||
if c == quote {
|
||||
quote = 0
|
||||
}
|
||||
case c == '\'' || c == '"':
|
||||
quote = c
|
||||
case c == '(' || c == '[':
|
||||
depth++
|
||||
case c == ')' || c == ']':
|
||||
depth--
|
||||
case c == sep && depth == 0:
|
||||
out = append(out, s[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
return append(out, s[start:])
|
||||
}
|
||||
|
||||
// zero is a value of a type, spelled as a string DuckDB casts to the type:
|
||||
// what a parameter is bound to when the query is run to see which of its
|
||||
// columns can be NULL. A type no value is known for reports false, and
|
||||
// the parameter is bound to NULL.
|
||||
func (a *analyzer) zero(t *analysis.TypeExpr) (string, bool) {
|
||||
if t == nil {
|
||||
return "", false
|
||||
}
|
||||
switch t.Name {
|
||||
case "tinyint", "smallint", "integer", "bigint", "hugeint", "utinyint", "usmallint", "uinteger", "ubigint", "uhugeint",
|
||||
"decimal", "float", "double", "bignum", "varint":
|
||||
return "1", true
|
||||
case "varchar", "text", "char", "bpchar":
|
||||
return "x", true
|
||||
case "json":
|
||||
return "{}", true
|
||||
case "blob":
|
||||
return "", true
|
||||
case "bit":
|
||||
return "0", true
|
||||
case "boolean":
|
||||
return "true", true
|
||||
case "uuid":
|
||||
return "00000000-0000-0000-0000-000000000000", true
|
||||
case "date":
|
||||
return "2000-01-01", true
|
||||
case "time", "time with time zone":
|
||||
return "00:00:00", true
|
||||
case "timestamp", "timestamp with time zone", "timestamp_s", "timestamp_ms", "timestamp_ns", "datetime":
|
||||
return "2000-01-01 00:00:00", true
|
||||
case "interval":
|
||||
return "1 day", true
|
||||
case "map":
|
||||
return "{}", true
|
||||
case "array":
|
||||
if len(t.Args) == 0 || t.Args[0].Type == nil {
|
||||
return "", false
|
||||
}
|
||||
elem, ok := a.zero(t.Args[0].Type)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
n := 1
|
||||
if len(t.Args) > 1 && t.Args[1].Int != nil {
|
||||
n = int(*t.Args[1].Int)
|
||||
}
|
||||
elems := make([]string, n)
|
||||
for i := range elems {
|
||||
elems[i] = elem
|
||||
}
|
||||
return "[" + strings.Join(elems, ", ") + "]", true
|
||||
case "struct":
|
||||
var fields []string
|
||||
for _, f := range t.Args {
|
||||
v, ok := a.zero(f.Type)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
fields = append(fields, f.Label+": "+v)
|
||||
}
|
||||
return "{" + strings.Join(fields, ", ") + "}", true
|
||||
case "union":
|
||||
if len(t.Args) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return a.zero(t.Args[0].Type)
|
||||
case "enum":
|
||||
if len(t.Args) > 0 && t.Args[0].String != nil {
|
||||
return *t.Args[0].String, true
|
||||
}
|
||||
}
|
||||
if labels, ok := a.enums[t.Name]; ok && len(labels) > 0 {
|
||||
return labels[0], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
Reference in New Issue
Block a user