snapshot of https://github.com/sqlc-dev/sqlc.git (main, 2026-09-16; blobs > 131072B omitted)
Some checks failed
kotlin / test (push) Has been skipped
python / test (push) Has been skipped
typescript / test (push) Has been skipped
go / build darwin/amd64 (push) Has been cancelled
go / build linux/amd64 (push) Has been cancelled
go / build windows/amd64 (push) Has been cancelled
go / build darwin/arm64 (push) Has been cancelled
go / build linux/arm64 (push) Has been cancelled
go / build windows/arm64 (push) Has been cancelled
go / test (push) Has been cancelled
go / vuln_check (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 02:17:34 +05:00
commit 0f4df23290
7340 changed files with 238743 additions and 0 deletions

275
internal/cmd/analyze.go Normal file
View File

@@ -0,0 +1,275 @@
package cmd
import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"github.com/spf13/cobra"
"github.com/sqlc-dev/sqlc/internal/compiler"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/core"
"github.com/sqlc-dev/sqlc/internal/multierr"
"github.com/sqlc-dev/sqlc/internal/opts"
"github.com/sqlc-dev/sqlc/internal/sql/ast"
)
func newAnalyzeCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "analyze [query-file]",
Short: "Analyze a query against a schema and output the result columns and parameters",
Long: `Analyze a query file against a schema file and output the inferred result
columns and parameters as JSON.
Unlike "sqlc generate", this command does not require a configuration file and
does not connect to a database. It uses sqlc's native static analysis to infer
types from the provided schema.
Examples:
# Analyze a PostgreSQL query
sqlc analyze --dialect postgresql --schema schema.sql query.sql
# Analyze a MySQL query
sqlc analyze --dialect mysql --schema schema.sql query.sql
# Analyze a SQLite query
sqlc analyze --dialect sqlite --schema schema.sql query.sql
# Analyze a ClickHouse query
sqlc analyze --dialect clickhouse --schema schema.sql query.sql
# Analyze a GoogleSQL (BigQuery, Spanner) query
sqlc analyze --dialect googlesql --schema schema.sql query.sql
# Analyze a SQL Server (T-SQL) query
sqlc analyze --dialect mssql --schema schema.sql query.sql
# Analyze a DuckDB query
sqlc analyze --dialect duckdb --schema schema.sql query.sql
# Analyze a query piped via stdin
echo "-- name: GetAuthor :one
SELECT * FROM authors WHERE id = $1;" | sqlc analyze --dialect postgresql --schema schema.sql
# Include the statement AST in the output
sqlc analyze --dialect postgresql --schema schema.sql --ast query.sql`,
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
dialect, err := cmd.Flags().GetString("dialect")
if err != nil {
return err
}
if dialect == "" {
return fmt.Errorf("--dialect flag is required (postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)")
}
schemaPath, err := cmd.Flags().GetString("schema")
if err != nil {
return err
}
if schemaPath == "" {
return fmt.Errorf("--schema flag is required")
}
includeAST, err := cmd.Flags().GetBool("ast")
if err != nil {
return err
}
// The query comes from a file argument or, when none is given, from
// stdin. The compiler reads queries from files, so stdin is written to
// a temporary file.
var queryPath string
if len(args) == 1 {
queryPath = args[0]
} else {
stat, err := os.Stdin.Stat()
if err != nil {
return fmt.Errorf("failed to stat stdin: %w", err)
}
if (stat.Mode() & os.ModeCharDevice) != 0 {
return fmt.Errorf("no query provided. Specify a query file or pipe SQL via stdin")
}
data, err := io.ReadAll(cmd.InOrStdin())
if err != nil {
return fmt.Errorf("failed to read stdin: %w", err)
}
tmp, err := os.CreateTemp("", "sqlc-analyze-*.sql")
if err != nil {
return fmt.Errorf("failed to create temp file: %w", err)
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(data); err != nil {
tmp.Close()
return fmt.Errorf("failed to write temp file: %w", err)
}
if err := tmp.Close(); err != nil {
return fmt.Errorf("failed to close temp file: %w", err)
}
queryPath = tmp.Name()
}
var engine config.Engine
switch dialect {
case "postgresql", "postgres", "pg":
engine = config.EnginePostgreSQL
case "mysql":
engine = config.EngineMySQL
case "sqlite":
engine = config.EngineSQLite
case "clickhouse":
engine = config.EngineClickHouse
case "googlesql":
engine = config.EngineGoogleSQL
case "mssql", "sqlserver":
engine = config.EngineMSSQL
case "duckdb":
engine = config.EngineDuckDB
default:
return fmt.Errorf("unsupported dialect: %s (use postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)", dialect)
}
sql := config.SQL{
Engine: engine,
Schema: config.Paths{schemaPath},
Queries: config.Paths{queryPath},
}
combo := config.Combine(config.Config{}, sql)
parserOpts := opts.Parser{}
ctx := cmd.Context()
c, err := compiler.NewCompiler(sql, combo, parserOpts, compiler.WithCoreAnalysis())
if err != nil {
return fmt.Errorf("error creating compiler: %w", err)
}
defer c.Close(ctx)
if err := c.ParseCatalog(sql.Schema); err != nil {
return fmt.Errorf("error parsing schema: %w", formatParseError(err))
}
if err := c.ParseQueries(sql.Queries, parserOpts); err != nil {
return fmt.Errorf("error parsing queries: %w", formatParseError(err))
}
result := c.Result()
out := make([]analyzedQuery, 0, len(result.Queries))
for _, q := range result.Queries {
out = append(out, newAnalyzedQuery(q, includeAST))
}
stdout := cmd.OutOrStdout()
encoder := json.NewEncoder(stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(out); err != nil {
return fmt.Errorf("failed to encode analysis: %w", err)
}
return nil
},
}
cmd.Flags().StringP("dialect", "d", "", "SQL dialect to use (postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)")
cmd.Flags().StringP("schema", "s", "", "path to the schema file")
cmd.Flags().BoolP("ast", "", false, "include the statement AST in the output")
return cmd
}
// formatParseError unwraps a multierr.Error into a single error containing all
// of the underlying file errors, so the analyze command can report each one with
// its file location.
func formatParseError(err error) error {
parserErr, ok := err.(*multierr.Error)
if !ok {
return err
}
var msgs []string
for _, fileErr := range parserErr.Errs() {
msgs = append(msgs, fmt.Sprintf("%s:%d:%d: %s",
fileErr.Filename, fileErr.Line, fileErr.Column, fileErr.Err))
}
if len(msgs) == 0 {
return err
}
return fmt.Errorf("%s", strings.Join(msgs, "; "))
}
type analyzedQuery struct {
Name string `json:"name"`
Cmd string `json:"cmd"`
Columns []analyzedColumn `json:"columns"`
Params []analyzedParam `json:"params"`
AST *ast.RawStmt `json:"ast,omitempty"`
}
type analyzedColumn struct {
Name string `json:"name"`
Type *core.TypeExpr `json:"type,omitempty"`
Table string `json:"table,omitempty"`
}
type analyzedParam struct {
Number int `json:"number"`
Column analyzedColumn `json:"column"`
}
func newAnalyzedQuery(q *compiler.Query, includeAST bool) analyzedQuery {
aq := analyzedQuery{
Name: q.Metadata.Name,
Cmd: q.Metadata.Cmd,
Columns: make([]analyzedColumn, 0, len(q.Columns)),
Params: make([]analyzedParam, 0, len(q.Params)),
}
for _, col := range q.Columns {
aq.Columns = append(aq.Columns, newAnalyzedColumn(col))
}
for _, p := range q.Params {
aq.Params = append(aq.Params, analyzedParam{
Number: p.Number,
Column: newAnalyzedColumn(p.Column),
})
}
if includeAST {
aq.AST = q.RawStmt
}
return aq
}
func newAnalyzedColumn(col *compiler.Column) analyzedColumn {
if col == nil {
return analyzedColumn{}
}
ac := analyzedColumn{
Name: col.Name,
Type: newAnalyzedType(col),
}
if col.Table != nil {
ac.Table = col.Table.Name
}
return ac
}
// newAnalyzedType is the column's type as an expression: the one the
// analysis core wrote when it did, otherwise the flat description the
// compiler holds, which is the data type wrapped in one array node per
// dimension with the column's nullability on the outermost node.
func newAnalyzedType(col *compiler.Column) *core.TypeExpr {
if col.TypeExpr != nil {
return col.TypeExpr
}
if col.DataType == "" {
return nil
}
t := core.ParseTypeExpr(col.DataType)
dims := col.ArrayDims
if col.IsArray && dims == 0 {
dims = 1
}
for i := 0; i < dims; i++ {
t = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: t}}}
}
t.Nullable = !col.NotNull
return t
}

277
internal/cmd/cmd.go Normal file
View File

@@ -0,0 +1,277 @@
package cmd
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime/trace"
"github.com/cubicdaiya/gonp"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"gopkg.in/yaml.v3"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/info"
"github.com/sqlc-dev/sqlc/internal/opts"
"github.com/sqlc-dev/sqlc/internal/sqlcdebug"
"github.com/sqlc-dev/sqlc/internal/tracer"
)
var debugProcessPlugins = sqlcdebug.New("processplugins")
func init() {
createDBCmd.Flags().StringP("queryset", "", "", "name of the queryset to use")
pushCmd.Flags().BoolP("dry-run", "", false, "dump push request (default: false)")
initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file")
initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file")
initCmd.MarkFlagsMutuallyExclusive("v1", "v2")
}
// Do runs the command logic.
func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int {
rootCmd := &cobra.Command{Use: "sqlc", SilenceUsage: true}
rootCmd.PersistentFlags().StringP("file", "f", "", "specify an alternate config file (default: sqlc.yaml)")
rootCmd.AddCommand(checkCmd)
rootCmd.AddCommand(createDBCmd)
rootCmd.AddCommand(diffCmd)
rootCmd.AddCommand(genCmd)
rootCmd.AddCommand(newFmtCmd())
rootCmd.AddCommand(initCmd)
rootCmd.AddCommand(newParseCmd())
rootCmd.AddCommand(newAnalyzeCmd())
rootCmd.AddCommand(versionCmd)
rootCmd.AddCommand(verifyCmd)
rootCmd.AddCommand(pushCmd)
rootCmd.AddCommand(NewCmdVet())
rootCmd.SetArgs(args)
rootCmd.SetIn(stdin)
rootCmd.SetOut(stdout)
rootCmd.SetErr(stderr)
ctx := context.Background()
if tracer.Path() != "" {
tracectx, cleanup, err := tracer.Start(ctx)
if err != nil {
fmt.Printf("failed to start trace: %v\n", err)
return 1
}
ctx = tracectx
defer cleanup()
}
if err := rootCmd.ExecuteContext(ctx); err != nil {
if exitError, ok := err.(*exec.ExitError); ok {
return exitError.ExitCode()
} else {
return 1
}
}
return 0
}
var version string
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print the sqlc version number",
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "version").End()
if version == "" {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n", info.Version)
} else {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n", version)
}
return nil
},
}
var initCmd = &cobra.Command{
Use: "init",
Short: "Create an empty sqlc.yaml settings file",
RunE: func(cmd *cobra.Command, args []string) error {
useV1, err := cmd.Flags().GetBool("v1")
if err != nil {
return err
}
var yamlConfig any
if useV1 {
yamlConfig = config.V1GenerateSettings{Version: "1"}
} else {
yamlConfig = config.Config{Version: "2"}
}
defer trace.StartRegion(cmd.Context(), "init").End()
file := "sqlc.yaml"
if f := cmd.Flag("file"); f != nil && f.Changed {
file = f.Value.String()
if file == "" {
return fmt.Errorf("file argument is empty")
}
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
fmt.Printf("%s is already created\n", file)
return nil
}
blob, err := yaml.Marshal(yamlConfig)
if err != nil {
return err
}
err = os.WriteFile(file, blob, 0644)
if err != nil {
return err
}
configDoc := "https://docs.sqlc.dev/en/stable/reference/config.html"
fmt.Printf(
"%s is added. Please visit %s to learn more about configuration\n",
file,
configDoc,
)
return nil
},
}
type Env struct {
DryRun bool
Experiment opts.Experiment
}
func ParseEnv(c *cobra.Command) Env {
dr := c.Flag("dry-run")
return Env{
DryRun: dr != nil && dr.Changed,
Experiment: opts.ExperimentFromEnv(),
}
}
var ErrPluginProcessDisabled = errors.New("plugin: process-based plugins disabled via SQLCDEBUG=processplugins=0")
func (e *Env) Validate(cfg *config.Config) error {
for _, plugin := range cfg.Plugins {
if plugin.Process != nil && debugProcessPlugins.Value() == "0" {
return ErrPluginProcessDisabled
}
}
return nil
}
func getConfigPath(stderr io.Writer, f *pflag.Flag) (string, string) {
if f != nil && f.Changed {
file := f.Value.String()
if file == "" {
fmt.Fprintln(stderr, "error parsing config: file argument is empty")
os.Exit(1)
}
abspath, err := filepath.Abs(file)
if err != nil {
fmt.Fprintf(stderr, "error parsing config: absolute file path lookup failed: %s\n", err)
os.Exit(1)
}
return filepath.Dir(abspath), filepath.Base(abspath)
} else {
wd, err := os.Getwd()
if err != nil {
fmt.Fprintln(stderr, "error parsing sqlc.json: file does not exist")
os.Exit(1)
}
return wd, ""
}
}
var genCmd = &cobra.Command{
Use: "generate",
Short: "Generate source code from SQL",
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "generate").End()
stderr := cmd.ErrOrStderr()
dir, name := getConfigPath(stderr, cmd.Flag("file"))
output, err := Generate(cmd.Context(), dir, name, &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
})
if err != nil {
os.Exit(1)
}
defer trace.StartRegion(cmd.Context(), "writefiles").End()
for filename, source := range output {
os.MkdirAll(filepath.Dir(filename), 0755)
if err := os.WriteFile(filename, []byte(source), 0644); err != nil {
fmt.Fprintf(stderr, "%s: %s\n", filename, err)
return err
}
}
return nil
},
}
var checkCmd = &cobra.Command{
Use: "compile",
Short: "Statically check SQL for syntax and type errors",
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "compile").End()
stderr := cmd.ErrOrStderr()
dir, name := getConfigPath(stderr, cmd.Flag("file"))
_, err := Generate(cmd.Context(), dir, name, &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
})
if err != nil {
os.Exit(1)
}
return nil
},
}
func getLines(f []byte) []string {
fp := bytes.NewReader(f)
scanner := bufio.NewScanner(fp)
lines := make([]string, 0)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines
}
func filterHunks[T gonp.Elem](uniHunks []gonp.UniHunk[T]) []gonp.UniHunk[T] {
var out []gonp.UniHunk[T]
for i, uniHunk := range uniHunks {
var changed bool
for _, e := range uniHunk.GetChanges() {
switch e.GetType() {
case gonp.SesDelete:
changed = true
case gonp.SesAdd:
changed = true
}
}
if changed {
out = append(out, uniHunks[i])
}
}
return out
}
var diffCmd = &cobra.Command{
Use: "diff",
Short: "Compare the generated files to the existing files",
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "diff").End()
stderr := cmd.ErrOrStderr()
dir, name := getConfigPath(stderr, cmd.Flag("file"))
opts := &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
}
if err := Diff(cmd.Context(), dir, name, opts); err != nil {
os.Exit(1)
}
return nil
},
}

102
internal/cmd/createdb.go Normal file
View File

@@ -0,0 +1,102 @@
package cmd
import (
"context"
"fmt"
"os"
"runtime/trace"
"time"
"github.com/spf13/cobra"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/dbmanager"
"github.com/sqlc-dev/sqlc/internal/migrations"
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
)
var createDBCmd = &cobra.Command{
Use: "createdb",
Short: "Create an ephemeral database",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "createdb").End()
stderr := cmd.ErrOrStderr()
dir, filename := getConfigPath(stderr, cmd.Flag("file"))
querySetName, err := cmd.Flags().GetString("queryset")
if err != nil {
return err
}
err = CreateDB(cmd.Context(), dir, filename, querySetName, &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
})
if err != nil {
fmt.Fprintln(stderr, err.Error())
os.Exit(1)
}
return nil
},
}
func CreateDB(ctx context.Context, dir, filename, querySetName string, o *Options) error {
_, conf, err := o.ReadConfig(dir, filename)
if err != nil {
return err
}
// Find the first queryset with a managed database
var queryset *config.SQL
var count int
for _, sql := range conf.SQL {
if querySetName != "" && sql.Name != querySetName {
continue
}
if sql.Database != nil && sql.Database.Managed {
queryset = &sql
count += 1
}
}
if queryset == nil && querySetName != "" {
return fmt.Errorf("no queryset found with name %q", querySetName)
}
if queryset == nil {
return fmt.Errorf("no querysets configured to use a managed database")
}
if count > 1 {
return fmt.Errorf("multiple querysets configured to use managed databases")
}
switch queryset.Engine {
case config.EngineMySQL:
// pass
case config.EnginePostgreSQL:
// pass
default:
return fmt.Errorf("createdb does not support the %s engine", queryset.Engine)
}
var ddl []string
files, err := sqlpath.Glob(queryset.Schema)
if err != nil {
return err
}
for _, schema := range files {
contents, err := os.ReadFile(schema)
if err != nil {
return fmt.Errorf("read file: %w", err)
}
ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
}
now := time.Now().UTC().UnixNano()
client := dbmanager.NewClient(conf.Servers)
resp, err := client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
Engine: string(queryset.Engine),
Migrations: ddl,
Prefix: fmt.Sprintf("sqlc_createdb_%d", now),
})
if err != nil {
return fmt.Errorf("managed: create database: %w", err)
}
fmt.Println(resp.Uri)
return nil
}

59
internal/cmd/diff.go Normal file
View File

@@ -0,0 +1,59 @@
package cmd
import (
"context"
"errors"
"fmt"
"os"
"runtime/trace"
"sort"
"strings"
"github.com/cubicdaiya/gonp"
)
func Diff(ctx context.Context, dir, name string, opts *Options) error {
stderr := opts.Stderr
output, err := Generate(ctx, dir, name, opts)
if err != nil {
return err
}
defer trace.StartRegion(ctx, "checkfiles").End()
var errored bool
keys := make([]string, 0, len(output))
for k, _ := range output {
kk := k
keys = append(keys, kk)
}
sort.Strings(keys)
for _, filename := range keys {
source := output[filename]
if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
errored = true
// stdout message
continue
}
existing, err := os.ReadFile(filename)
if err != nil {
errored = true
fmt.Fprintf(stderr, "%s: %s\n", filename, err)
continue
}
diff := gonp.New(getLines(existing), getLines([]byte(source)))
diff.Compose()
uniHunks := filterHunks(diff.UnifiedHunks())
if len(uniHunks) > 0 {
errored = true
fmt.Fprintf(stderr, "--- a%s\n", strings.TrimPrefix(filename, dir))
fmt.Fprintf(stderr, "+++ b%s\n", strings.TrimPrefix(filename, dir))
diff.FprintUniHunks(stderr, uniHunks)
}
}
if errored {
return errors.New("diff found")
}
return nil
}

481
internal/cmd/fmt.go Normal file
View File

@@ -0,0 +1,481 @@
package cmd
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"runtime/trace"
"sort"
"strings"
"github.com/cubicdaiya/gonp"
"github.com/spf13/cobra"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/engine/dolphin"
"github.com/sqlc-dev/sqlc/internal/engine/postgresql"
"github.com/sqlc-dev/sqlc/internal/engine/sqlite"
"github.com/sqlc-dev/sqlc/internal/sql/ast"
"github.com/sqlc-dev/sqlc/internal/sql/format"
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
)
// noLineLimit renders without a maximum line width: like gofmt, fmt never
// rewraps a line on its own. Breaks come from the author's own line breaks
// at the boundaries the printer models, and from comments, which cannot
// share a line with the code after them.
const noLineLimit = 1 << 30
// queryFormatter is what the fmt command needs from an engine: a parser
// that surfaces the comments its lexer already scans (so statements and
// comments come from one pass, and interior comments format along with
// their statements) plus dialect-aware formatting. Comments cannot be
// reprinted without parser support, so ParseFile is the price of admission.
type queryFormatter interface {
ParseFile(io.Reader) (*ast.File, error)
format.Dialect
}
// newQueryFormatter returns the formatter for engines fmt supports —
// SQLite, PostgreSQL and MySQL today. An engine joins by teaching its
// parser to surface comments (meyer's, oliphant's and marino's ParseFile
// are the templates) and adding its case here.
func newQueryFormatter(engine config.Engine) queryFormatter {
switch engine {
case config.EnginePostgreSQL:
return postgresql.NewParser()
case config.EngineSQLite:
return sqlite.NewParser()
case config.EngineMySQL:
// The format parser preserves identifier case and proves every
// statement via Fingerprint, the way oliphant does for PostgreSQL.
return dolphin.NewFormatParser()
default:
return nil
}
}
func newFmtCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "fmt",
Short: "Format SQL queries",
Long: `Format the SQL query files referenced by the configuration file.
Each query is parsed with the engine's parser and printed back in a canonical
form, with its comments kept where they were written. A statement that cannot
be proven to survive formatting unchanged is left exactly as written. Files
are rewritten in place; pass --diff to print the changes to stdout instead.`,
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "fmt").End()
stderr := cmd.ErrOrStderr()
stdout := cmd.OutOrStdout()
showDiff, err := cmd.Flags().GetBool("diff")
if err != nil {
return err
}
dir, name := getConfigPath(stderr, cmd.Flag("file"))
output, err := Format(cmd.Context(), dir, name, &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
})
if err != nil {
os.Exit(1)
}
filenames := make([]string, 0, len(output))
for filename := range output {
filenames = append(filenames, filename)
}
sort.Strings(filenames)
for _, filename := range filenames {
existing, err := os.ReadFile(filename)
if err != nil {
fmt.Fprintf(stderr, "%s: %s\n", filename, err)
return err
}
source := output[filename]
if string(existing) == source {
continue
}
if showDiff {
diff := gonp.New(getLines(existing), getLines([]byte(source)))
diff.Compose()
uniHunks := filterHunks(diff.UnifiedHunks())
if len(uniHunks) == 0 {
continue
}
rel := filepath.ToSlash(strings.TrimPrefix(filename, dir))
fmt.Fprintf(stdout, "--- a%s\n", rel)
fmt.Fprintf(stdout, "+++ b%s\n", rel)
diff.FprintUniHunks(stdout, uniHunks)
continue
}
if err := os.WriteFile(filename, []byte(source), 0644); err != nil {
fmt.Fprintf(stderr, "%s: %s\n", filename, err)
return err
}
}
return nil
},
}
cmd.Flags().Bool("diff", false, "display diffs instead of rewriting files")
return cmd
}
// Format formats every query file referenced by the configuration with its
// engine's SQL formatter. It returns the formatted contents of each file,
// keyed by absolute path. Query files for engines without a formatter are
// skipped.
func Format(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
defer trace.StartRegion(ctx, "format").End()
stderr := o.Stderr
configPath, conf, err := o.ReadConfig(dir, filename)
if err != nil {
return nil, err
}
base := filepath.Base(configPath)
if err := config.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return nil, err
}
engines := map[string]config.Engine{}
var files []string
for _, sql := range conf.SQL {
joined := make([]string, 0, len(sql.Queries))
for _, q := range sql.Queries {
joined = append(joined, filepath.Join(dir, q))
}
list, err := sqlpath.Glob(joined)
if err != nil {
fmt.Fprintf(stderr, "error listing queries: %s\n", err)
return nil, err
}
for _, file := range list {
if _, ok := engines[file]; ok {
continue
}
engines[file] = sql.Engine
files = append(files, file)
}
}
output := map[string]string{}
unsupported := map[config.Engine]bool{}
for _, file := range files {
f := newQueryFormatter(engines[file])
if f == nil {
if !unsupported[engines[file]] {
unsupported[engines[file]] = true
fmt.Fprintf(stderr, "sqlc fmt does not yet support the %s engine; query files left unchanged\n", engines[file])
}
continue
}
rel, err := filepath.Rel(dir, file)
if err != nil {
rel = file
}
contents, err := os.ReadFile(file)
if err != nil {
fmt.Fprintf(stderr, "%s: skipped: %s\n", rel, err)
continue
}
formatted, err := formatQueries(f, string(contents))
if err != nil {
// Not every query file parses without the compiler's help — for
// example, some engines only accept sqlc.slice() after
// preprocessing. Leave those files as they are.
fmt.Fprintf(stderr, "%s: skipped: %s\n", rel, err)
continue
}
// File-level belt for the reprinter path: no formatting result may
// change the file's comments or its statement count. A violation is
// a formatter bug; keep the file as written and say so.
before, err1 := f.ParseFile(strings.NewReader(string(contents)))
after, err2 := f.ParseFile(strings.NewReader(formatted))
if err1 == nil && (err2 != nil || len(before.Stmts) != len(after.Stmts) || !sameComments(before.Comments, after.Comments)) {
fmt.Fprintf(stderr, "%s: skipped: formatting would alter the file (this is a bug in sqlc fmt)\n", rel)
continue
}
output[file] = formatted
}
return output, nil
}
// formatQueries formats each statement in a query file. The comments above
// every statement (that's where the "-- name:" annotation lives) are kept
// as written; comments inside a statement are threaded through the printer
// by position.
func formatQueries(f queryFormatter, src string) (string, error) {
// The parser hands over statements and comments from one lexer pass;
// statement bodies slice the comment list by span.
file, err := f.ParseFile(strings.NewReader(src))
if err != nil {
return "", err
}
stmts, fileComments := file.Stmts, file.Comments
var blocks []string
prevEnd := 0
for _, stmt := range stmts {
if stmt.Raw == nil {
continue
}
start := stmt.Raw.StmtLocation
length := stmt.Raw.StmtLen
if length == 0 {
// An unterminated final statement is reported with a zero length.
length = len(src) - start
}
if start < 0 || start+length > len(src) {
return "", fmt.Errorf("statement location out of bounds")
}
// Engines differ on whether a statement's span covers the comments
// above it, so start each segment where the previous one ended: the
// gap holds nothing but the previous terminator, whitespace and
// comments.
segStart := prevEnd
if start < segStart {
segStart = start
}
if start+length <= prevEnd {
continue
}
prevEnd = start + length
seg := src[segStart : start+length]
segOff := segStart
// A comment on the same line as the previous statement's terminator
// belongs to that statement, not to this one's header.
if len(blocks) > 0 {
if trailing, remainder := splitTrailingComment(seg); trailing != "" {
blocks[len(blocks)-1] += " " + trailing
segOff += len(seg) - len(remainder)
seg = remainder
}
}
orig := strings.TrimSpace(strings.TrimLeft(seg, "; \t\r\n"))
if orig == "" {
continue
}
header, rest := splitLeadingComments(orig)
// The statement body's own file offset, so its interior comments
// can be sliced out of the file's comment list in the same
// coordinates the parser stamped on the nodes.
restOff := segOff + strings.Index(seg, orig) + len(orig) - len(rest)
var interior []ast.Comment
for _, c := range fileComments {
if c.Start >= restOff && c.End <= start+length {
interior = append(interior, c)
}
}
block := append([]string{}, header...)
block = append(block, formatStmt(f, stmt.Raw, rest, interior, src))
blocks = append(blocks, strings.Join(block, "\n"))
}
// Everything after the last statement is its terminator, whitespace and
// trailing comments; keep the comments.
tailSeg := src[prevEnd:]
if len(blocks) > 0 {
if trailing, remainder := splitTrailingComment(tailSeg); trailing != "" {
blocks[len(blocks)-1] += " " + trailing
tailSeg = remainder
}
}
if tail := strings.TrimSpace(strings.TrimLeft(tailSeg, "; \t\r\n")); tail != "" {
blocks = append(blocks, tail)
}
if len(blocks) == 0 {
return src, nil
}
return strings.Join(blocks, "\n\n") + "\n", nil
}
// splitTrailingComment splits a comment sitting on the same line as the
// previous statement's terminator off the front of the text between two
// statements, returning the comment (or "") and the remaining text.
func splitTrailingComment(seg string) (string, string) {
k := 0
for k < len(seg) && (seg[k] == ';' || seg[k] == ' ' || seg[k] == '\t' || seg[k] == '\r') {
k++
}
rest := seg[k:]
line, _, _ := strings.Cut(rest, "\n")
trimmed := strings.TrimSpace(line)
switch {
case strings.HasPrefix(trimmed, "--"), strings.HasPrefix(trimmed, "#"):
return trimmed, seg[k+len(line):]
case strings.HasPrefix(trimmed, "/*") &&
strings.HasSuffix(trimmed, "*/") && strings.Count(trimmed, "*/") == 1:
// A block comment contained on the terminator's line.
return trimmed, seg[k+len(line):]
}
return "", seg
}
// splitLeadingComments splits a statement's text into the comment lines above
// it and the statement itself. Comment lines are returned with surrounding
// whitespace trimmed (block comment continuation lines keep their leading
// whitespace); blank lines between comments are kept.
func splitLeadingComments(text string) ([]string, string) {
var header []string
rest := text
for rest != "" {
line, remainder, found := strings.Cut(rest, "\n")
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "/*") && !strings.Contains(trimmed, "*/") {
// A block comment spanning several lines: commit it to the
// header only when its closing line arrives with nothing after
// it. Otherwise leave the whole block with the statement.
block := []string{trimmed}
blockRest := remainder
for closed := false; !closed; {
if !found || blockRest == "" {
return header, rest
}
var blockLine string
blockLine, blockRest, found = strings.Cut(blockRest, "\n")
if end := strings.TrimSpace(blockLine); strings.Contains(end, "*/") {
if !strings.HasSuffix(end, "*/") || strings.Count(end, "*/") != 1 {
return header, rest
}
closed = true
}
block = append(block, strings.TrimRight(blockLine, " \t\r"))
}
header = append(header, block...)
rest = blockRest
continue
}
if !isCommentLine(trimmed) {
break
}
header = append(header, trimmed)
if !found {
rest = ""
break
}
rest = remainder
}
// A statement never ends with a comment-only line here (the parser found a
// statement in this segment), but drop any blank lines left at the end of
// the header so comments sit directly above their query.
for len(header) > 0 && header[len(header)-1] == "" {
header = header[:len(header)-1]
}
return header, rest
}
func isCommentLine(line string) bool {
switch {
case line == "":
// A blank line between comments.
return true
case strings.HasPrefix(line, "--"):
return true
case strings.HasPrefix(line, "#"):
// MySQL's line-comment syntax; text that reaches these helpers sits
// outside statements, where a # line in a file the engine parsed
// can only be a comment.
return true
case strings.HasPrefix(line, "/*") && strings.HasSuffix(line, "*/") && strings.Count(line, "*/") == 1:
// A block comment contained on a single line.
return true
default:
return false
}
}
// fingerprinter is implemented by engines that can reduce a query to a
// fingerprint that survives changes in whitespace, case and layout —
// PostgreSQL via oliphant's pg_query-compatible Fingerprint. Where it is
// available, fmt gets a proof the other checks cannot give: the formatted
// statement still parses to the same query as the original.
type fingerprinter interface {
Fingerprint(string) (string, error)
}
// formatStmt returns the canonical form of a single statement, or the
// original text when formatting cannot be proven to preserve the query.
func formatStmt(f queryFormatter, raw *ast.RawStmt, orig string, interior []ast.Comment, src string) string {
fallback := strings.TrimSuffix(strings.TrimSpace(orig), ";") + ";"
if out, ok := formatWithComments(f, raw, interior, src); ok {
if fp, ok := f.(fingerprinter); ok && !sameFingerprint(fp, orig, out) {
return fallback
}
return out
}
return fallback
}
// sameFingerprint reports that both texts fingerprint successfully to the
// same value. Anything less is not a proof, so the caller keeps the
// statement as written.
func sameFingerprint(fp fingerprinter, orig, out string) bool {
a, err1 := fp.Fingerprint(orig)
b, err2 := fp.Fingerprint(out)
return err1 == nil && err2 == nil && a == b
}
// formatWithComments pretty-prints a statement with its interior comments
// and proves the result faithful three ways before accepting it: every
// comment survives (multiset equality after reparsing the output), the SQL
// still parses as one statement, and the output is a fixed point (printing
// the reparsed statement with its reparsed comments reproduces it).
func formatWithComments(f queryFormatter, raw *ast.RawStmt, interior []ast.Comment, src string) (string, bool) {
out := prettyCommented(raw, f, interior, src)
if strings.TrimSpace(strings.TrimSuffix(out, ";")) == "" {
return "", false
}
file, err := f.ParseFile(strings.NewReader(out))
if err != nil || len(file.Stmts) != 1 || file.Stmts[0].Raw == nil {
return "", false
}
if !sameComments(interior, file.Comments) {
return "", false
}
again := prettyCommented(file.Stmts[0].Raw, f, file.Comments, out)
if !strings.EqualFold(again, out) {
return "", false
}
return out, true
}
// prettyCommented attaches a statement's comments to its nodes and renders
// it, returning "" when the printer panics or fails to place every comment.
func prettyCommented(raw *ast.RawStmt, f queryFormatter, comments []ast.Comment, src string) (out string) {
defer func() {
if r := recover(); r != nil {
out = ""
}
}()
ct := ast.AttachComments(raw, f, comments, src)
out = ast.PrettyWithComments(raw, f, noLineLimit, ct)
if !ct.Exhausted() {
return ""
}
return out
}
// sameComments reports whether two comment lists carry the same comment
// texts, as multisets, ignoring surrounding whitespace.
func sameComments(a, b []ast.Comment) bool {
if len(a) != len(b) {
return false
}
counts := make(map[string]int, len(a))
for _, c := range a {
counts[strings.TrimSpace(c.Text)]++
}
for _, c := range b {
counts[strings.TrimSpace(c.Text)]--
}
for _, n := range counts {
if n != 0 {
return false
}
}
return true
}

376
internal/cmd/generate.go Normal file
View File

@@ -0,0 +1,376 @@
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime/trace"
"strings"
"sync"
"google.golang.org/grpc"
"github.com/sqlc-dev/sqlc/internal/codegen/golang"
genjson "github.com/sqlc-dev/sqlc/internal/codegen/json"
"github.com/sqlc-dev/sqlc/internal/compiler"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/config/convert"
"github.com/sqlc-dev/sqlc/internal/debug"
"github.com/sqlc-dev/sqlc/internal/ext"
"github.com/sqlc-dev/sqlc/internal/ext/process"
"github.com/sqlc-dev/sqlc/internal/ext/wasm"
"github.com/sqlc-dev/sqlc/internal/multierr"
"github.com/sqlc-dev/sqlc/internal/opts"
"github.com/sqlc-dev/sqlc/internal/plugin"
"github.com/sqlc-dev/sqlc/internal/sqlcdebug"
)
var debugDumpCatalog = sqlcdebug.New("dumpcatalog")
const errMessageNoVersion = `The configuration file must have a version number.
Set the version to 1 or 2 at the top of sqlc.json:
{
"version": "1"
...
}
`
const errMessageUnknownVersion = `The configuration file has an invalid version number.
The supported version can only be "1" or "2".
`
const errMessageNoPackages = `No packages are configured`
func printFileErr(stderr io.Writer, dir string, fileErr *multierr.FileError) {
filename, err := filepath.Rel(dir, fileErr.Filename)
if err != nil {
filename = fileErr.Filename
}
fmt.Fprintf(stderr, "%s:%d:%d: %s\n", filename, fileErr.Line, fileErr.Column, fileErr.Err)
}
func findPlugin(conf config.Config, name string) (*config.Plugin, error) {
for _, plug := range conf.Plugins {
if plug.Name == name {
return &plug, nil
}
}
return nil, fmt.Errorf("plugin not found")
}
func readConfig(stderr io.Writer, dir, filename string) (string, *config.Config, error) {
configPath := ""
if filename != "" {
configPath = filepath.Join(dir, filename)
} else {
var yamlMissing, jsonMissing, ymlMissing bool
yamlPath := filepath.Join(dir, "sqlc.yaml")
ymlPath := filepath.Join(dir, "sqlc.yml")
jsonPath := filepath.Join(dir, "sqlc.json")
if _, err := os.Stat(yamlPath); os.IsNotExist(err) {
yamlMissing = true
}
if _, err := os.Stat(jsonPath); os.IsNotExist(err) {
jsonMissing = true
}
if _, err := os.Stat(ymlPath); os.IsNotExist(err) {
ymlMissing = true
}
if yamlMissing && ymlMissing && jsonMissing {
fmt.Fprintln(stderr, "error parsing configuration files. sqlc.(yaml|yml) or sqlc.json: file does not exist")
return "", nil, errors.New("config file missing")
}
if (!yamlMissing || !ymlMissing) && !jsonMissing {
fmt.Fprintln(stderr, "error: both sqlc.json and sqlc.(yaml|yml) files present")
return "", nil, errors.New("sqlc.json and sqlc.(yaml|yml) present")
}
if jsonMissing {
if yamlMissing {
configPath = ymlPath
} else {
configPath = yamlPath
}
} else {
configPath = jsonPath
}
}
base := filepath.Base(configPath)
file, err := os.Open(configPath)
if err != nil {
fmt.Fprintf(stderr, "error parsing %s: file does not exist\n", base)
return "", nil, err
}
defer file.Close()
conf, err := config.ParseConfig(file)
if err != nil {
switch err {
case config.ErrMissingVersion:
fmt.Fprint(stderr, errMessageNoVersion)
case config.ErrUnknownVersion:
fmt.Fprint(stderr, errMessageUnknownVersion)
case config.ErrNoPackages:
fmt.Fprint(stderr, errMessageNoPackages)
}
fmt.Fprintf(stderr, "error parsing %s: %s\n", base, err)
return "", nil, err
}
return configPath, &conf, nil
}
func Generate(ctx context.Context, dir, filename string, o *Options) (map[string]string, error) {
e := o.Env
stderr := o.Stderr
configPath, conf, err := o.ReadConfig(dir, filename)
if err != nil {
return nil, err
}
base := filepath.Base(configPath)
if err := config.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return nil, err
}
if err := e.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return nil, err
}
g := &generator{
dir: dir,
output: map[string]string{},
}
if err := processQuerySets(ctx, g, conf, dir, o); err != nil {
return nil, err
}
return g.output, nil
}
type generator struct {
m sync.Mutex
dir string
output map[string]string
}
func (g *generator) Pairs(ctx context.Context, conf *config.Config) []OutputPair {
var pairs []OutputPair
for _, sql := range conf.SQL {
if sql.Gen.Go != nil {
pairs = append(pairs, OutputPair{
SQL: sql,
Gen: config.SQLGen{Go: sql.Gen.Go},
})
}
if sql.Gen.JSON != nil {
pairs = append(pairs, OutputPair{
SQL: sql,
Gen: config.SQLGen{JSON: sql.Gen.JSON},
})
}
for i := range sql.Codegen {
pairs = append(pairs, OutputPair{
SQL: sql,
Plugin: &sql.Codegen[i],
})
}
}
return pairs
}
func (g *generator) ProcessResult(ctx context.Context, combo config.CombinedSettings, sql OutputPair, result *compiler.Result) error {
out, resp, err := codegen(ctx, combo, sql, result)
if err != nil {
return err
}
files := map[string]string{}
for _, file := range resp.Files {
files[file.Name] = string(file.Contents)
}
g.m.Lock()
// out is specified by the user, not a plugin
absout := filepath.Join(g.dir, out)
// When the Go codegen is configured to emit the models file into a
// separate package directory, route that file to its own absolute path.
// This is the only file allowed to live outside of `out`.
var (
modelsFileName string
modelsAbsout string
modelsAbsfile string
)
if sql.Gen.Go != nil && sql.Gen.Go.OutputModelsPath != "" && sql.Gen.Go.ModelsEmitEnabled() {
modelsFileName = sql.Gen.Go.OutputModelsFileName
if modelsFileName == "" {
modelsFileName = "models.go"
}
modelsAbsout = filepath.Join(g.dir, sql.Gen.Go.OutputModelsPath)
modelsAbsfile = filepath.Join(modelsAbsout, modelsFileName)
}
for n, source := range files {
if modelsFileName != "" && n == modelsFileName {
// Models file routed to a separate package directory.
if strings.Contains(modelsAbsfile, "..") {
return fmt.Errorf("invalid file output path: %s", modelsAbsfile)
}
if !strings.HasPrefix(modelsAbsfile, modelsAbsout) {
return fmt.Errorf("invalid file output path: %s", modelsAbsfile)
}
g.output[modelsAbsfile] = source
continue
}
filename := filepath.Join(g.dir, out, n)
// filepath.Join calls filepath.Clean which should remove all "..", but
// double check to make sure
if strings.Contains(filename, "..") {
return fmt.Errorf("invalid file output path: %s", filename)
}
// The output file must be contained inside the output directory
if !strings.HasPrefix(filename, absout) {
return fmt.Errorf("invalid file output path: %s", filename)
}
g.output[filename] = source
}
g.m.Unlock()
return nil
}
func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) {
defer trace.StartRegion(ctx, "parse").End()
var copts []compiler.Option
if parserOpts.Experiment.CoreAnalyzer {
copts = append(copts, compiler.WithCoreAnalysis())
}
c, err := compiler.NewCompiler(sql, combo, parserOpts, copts...)
defer func() {
if c != nil {
c.Close(ctx)
}
}()
if err != nil {
fmt.Fprintf(stderr, "error creating compiler: %s\n", err)
return nil, true
}
if err := c.ParseCatalog(sql.Schema); err != nil {
fmt.Fprintf(stderr, "# package %s\n", name)
if parserErr, ok := err.(*multierr.Error); ok {
for _, fileErr := range parserErr.Errs() {
printFileErr(stderr, dir, fileErr)
}
} else {
fmt.Fprintf(stderr, "error parsing schema: %s\n", err)
}
return nil, true
}
if debugDumpCatalog.Value() == "1" {
debug.Dump(c.Catalog())
}
if err := c.ParseQueries(sql.Queries, parserOpts); err != nil {
fmt.Fprintf(stderr, "# package %s\n", name)
if parserErr, ok := err.(*multierr.Error); ok {
for _, fileErr := range parserErr.Errs() {
printFileErr(stderr, dir, fileErr)
}
} else {
fmt.Fprintf(stderr, "error parsing queries: %s\n", err)
}
return nil, true
}
return c.Result(), false
}
func codegen(ctx context.Context, combo config.CombinedSettings, sql OutputPair, result *compiler.Result) (string, *plugin.GenerateResponse, error) {
defer trace.StartRegion(ctx, "codegen").End()
req := codeGenRequest(result, combo)
var handler grpc.ClientConnInterface
var out string
switch {
case sql.Plugin != nil:
out = sql.Plugin.Out
plug, err := findPlugin(combo.Global, sql.Plugin.Plugin)
if err != nil {
return "", nil, fmt.Errorf("plugin not found: %s", err)
}
switch {
case plug.Process != nil:
handler = &process.Runner{
Cmd: plug.Process.Cmd,
Env: plug.Env,
Format: plug.Process.Format,
}
case plug.WASM != nil:
handler = &wasm.Runner{
URL: plug.WASM.URL,
SHA256: plug.WASM.SHA256,
Env: plug.Env,
}
default:
return "", nil, fmt.Errorf("unsupported plugin type")
}
opts, err := convert.YAMLtoJSON(sql.Plugin.Options)
if err != nil {
return "", nil, fmt.Errorf("invalid plugin options: %w", err)
}
req.PluginOptions = opts
global, found := combo.Global.Options[plug.Name]
if found {
opts, err := convert.YAMLtoJSON(global)
if err != nil {
return "", nil, fmt.Errorf("invalid global options: %w", err)
}
req.GlobalOptions = opts
}
case sql.Gen.Go != nil:
out = combo.Go.Out
handler = ext.HandleFunc(golang.Generate)
opts, err := json.Marshal(sql.Gen.Go)
if err != nil {
return "", nil, fmt.Errorf("opts marshal failed: %w", err)
}
req.PluginOptions = opts
if combo.Global.Overrides.Go != nil {
opts, err := json.Marshal(combo.Global.Overrides.Go)
if err != nil {
return "", nil, fmt.Errorf("opts marshal failed: %w", err)
}
req.GlobalOptions = opts
}
case sql.Gen.JSON != nil:
out = combo.JSON.Out
handler = ext.HandleFunc(genjson.Generate)
opts, err := json.Marshal(sql.Gen.JSON)
if err != nil {
return "", nil, fmt.Errorf("opts marshal failed: %w", err)
}
req.PluginOptions = opts
default:
return "", nil, fmt.Errorf("missing language backend")
}
client := plugin.NewCodegenServiceClient(handler)
resp, err := client.Generate(ctx, req)
return out, resp, err
}

29
internal/cmd/options.go Normal file
View File

@@ -0,0 +1,29 @@
package cmd
import (
"io"
"github.com/sqlc-dev/sqlc/internal/config"
)
type Options struct {
Env Env
Stderr io.Writer
// TODO: Move these to a command-specific struct
Tags []string
Against string
// Testing only
MutateConfig func(*config.Config)
}
func (o *Options) ReadConfig(dir, filename string) (string, *config.Config, error) {
path, conf, err := readConfig(o.Stderr, dir, filename)
if err != nil {
return path, conf, err
}
if o.MutateConfig != nil {
o.MutateConfig(conf)
}
return path, conf, nil
}

166
internal/cmd/parse.go Normal file
View File

@@ -0,0 +1,166 @@
package cmd
import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"github.com/spf13/cobra"
"github.com/sqlc-dev/sqlc/internal/engine/clickhouse"
"github.com/sqlc-dev/sqlc/internal/engine/dolphin"
"github.com/sqlc-dev/sqlc/internal/engine/duckdb"
"github.com/sqlc-dev/sqlc/internal/engine/googlesql"
"github.com/sqlc-dev/sqlc/internal/engine/mssql"
"github.com/sqlc-dev/sqlc/internal/engine/postgresql"
"github.com/sqlc-dev/sqlc/internal/engine/sqlite"
"github.com/sqlc-dev/sqlc/internal/metadata"
"github.com/sqlc-dev/sqlc/internal/source"
"github.com/sqlc-dev/sqlc/internal/sql/ast"
)
// dialectParser is the subset of the engine parsers that the parse command
// needs: parsing SQL into statements and reporting the dialect's comment syntax
// (used to extract the sqlc query name and command).
type dialectParser interface {
Parse(io.Reader) ([]ast.Statement, error)
CommentSyntax() source.CommentSyntax
}
// parsedStatement is the JSON representation of a single parsed statement. The
// name and cmd are extracted from the sqlc query annotation (e.g.
// "-- name: GetAuthor :one") and are omitted when the statement has none.
type parsedStatement struct {
Name string `json:"name,omitempty"`
Cmd string `json:"cmd,omitempty"`
AST *ast.RawStmt `json:"ast"`
}
func newParseCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "parse [file]",
Short: "Parse SQL and output the AST as JSON",
Long: `Parse SQL from a file or stdin and output the abstract syntax tree as JSON.
Each statement is reported with its sqlc query name and command (when the
statement carries a "-- name:" annotation) alongside the AST.
Examples:
# Parse a SQL file with PostgreSQL dialect
sqlc parse --dialect postgresql schema.sql
# Parse from stdin with MySQL dialect
echo "SELECT * FROM users" | sqlc parse --dialect mysql
# Parse SQLite SQL
sqlc parse --dialect sqlite queries.sql
# Parse ClickHouse SQL
sqlc parse --dialect clickhouse queries.sql
# Parse GoogleSQL (BigQuery, Spanner)
sqlc parse --dialect googlesql queries.sql
# Parse SQL Server (T-SQL) SQL
sqlc parse --dialect mssql queries.sql
# Parse DuckDB SQL
sqlc parse --dialect duckdb queries.sql`,
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
dialect, err := cmd.Flags().GetString("dialect")
if err != nil {
return err
}
if dialect == "" {
return fmt.Errorf("--dialect flag is required (postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)")
}
// Determine input source
var input io.Reader
if len(args) == 1 {
file, err := os.Open(args[0])
if err != nil {
return fmt.Errorf("failed to open file: %w", err)
}
defer file.Close()
input = file
} else {
// Check if stdin has data
stat, err := os.Stdin.Stat()
if err != nil {
return fmt.Errorf("failed to stat stdin: %w", err)
}
if (stat.Mode() & os.ModeCharDevice) != 0 {
return fmt.Errorf("no input provided. Specify a file path or pipe SQL via stdin")
}
input = cmd.InOrStdin()
}
// Select the parser for the requested dialect
var parser dialectParser
switch dialect {
case "postgresql", "postgres", "pg":
parser = postgresql.NewParser()
case "mysql":
parser = dolphin.NewParser()
case "sqlite":
parser = sqlite.NewParser()
case "clickhouse":
parser = clickhouse.NewParser()
case "googlesql":
parser = googlesql.NewParser()
case "mssql", "sqlserver":
parser = mssql.NewParser()
case "duckdb":
parser = duckdb.NewParser()
default:
return fmt.Errorf("unsupported dialect: %s (use postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)", dialect)
}
// Read the full source so each statement's name and command can be
// extracted from its annotation comment.
src, err := io.ReadAll(input)
if err != nil {
return fmt.Errorf("failed to read input: %w", err)
}
stmts, err := parser.Parse(strings.NewReader(string(src)))
if err != nil {
return fmt.Errorf("parse error: %w", err)
}
commentSyntax := metadata.CommentSyntax(parser.CommentSyntax())
// Output the AST as a single JSON document
out := make([]parsedStatement, 0, len(stmts))
for _, stmt := range stmts {
ps := parsedStatement{AST: stmt.Raw}
rawSQL, err := source.Pluck(string(src), stmt.Raw.StmtLocation, stmt.Raw.StmtLen)
if err != nil {
return fmt.Errorf("failed to read statement source: %w", err)
}
name, cmd, err := metadata.ParseQueryNameAndType(rawSQL, commentSyntax)
if err != nil {
return fmt.Errorf("failed to parse query annotation: %w", err)
}
ps.Name = name
ps.Cmd = cmd
out = append(out, ps)
}
stdout := cmd.OutOrStdout()
encoder := json.NewEncoder(stdout)
encoder.SetIndent("", " ")
if err := encoder.Encode(out); err != nil {
return fmt.Errorf("failed to encode AST: %w", err)
}
return nil
},
}
cmd.Flags().StringP("dialect", "d", "", "SQL dialect to use (postgresql, mysql, sqlite, clickhouse, googlesql, mssql, or duckdb)")
return cmd
}

133
internal/cmd/process.go Normal file
View File

@@ -0,0 +1,133 @@
package cmd
import (
"bytes"
"context"
"fmt"
"io"
"path/filepath"
"runtime"
"runtime/trace"
"golang.org/x/sync/errgroup"
"github.com/sqlc-dev/sqlc/internal/compiler"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/opts"
)
type OutputPair struct {
Gen config.SQLGen
Plugin *config.Codegen
config.SQL
}
type ResultProcessor interface {
Pairs(context.Context, *config.Config) []OutputPair
ProcessResult(context.Context, config.CombinedSettings, OutputPair, *compiler.Result) error
}
func Process(ctx context.Context, rp ResultProcessor, dir, filename string, o *Options) error {
e := o.Env
stderr := o.Stderr
configPath, conf, err := o.ReadConfig(dir, filename)
if err != nil {
return err
}
base := filepath.Base(configPath)
if err := config.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return err
}
if err := e.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return err
}
return processQuerySets(ctx, rp, conf, dir, o)
}
func processQuerySets(ctx context.Context, rp ResultProcessor, conf *config.Config, dir string, o *Options) error {
stderr := o.Stderr
errored := false
pairs := rp.Pairs(ctx, conf)
grp, gctx := errgroup.WithContext(ctx)
grp.SetLimit(runtime.GOMAXPROCS(0))
stderrs := make([]bytes.Buffer, len(pairs))
for i, pair := range pairs {
sql := pair
errout := &stderrs[i]
grp.Go(func() error {
combo := config.Combine(*conf, sql.SQL)
if sql.Plugin != nil {
combo.Codegen = *sql.Plugin
}
// TODO: This feels like a hack that will bite us later
joined := make([]string, 0, len(sql.Schema))
for _, s := range sql.Schema {
joined = append(joined, filepath.Join(dir, s))
}
sql.Schema = joined
joined = make([]string, 0, len(sql.Queries))
for _, q := range sql.Queries {
joined = append(joined, filepath.Join(dir, q))
}
sql.Queries = joined
var name, lang string
parseOpts := opts.Parser{
Experiment: o.Env.Experiment,
}
switch {
case sql.Gen.Go != nil:
name = combo.Go.Package
lang = "golang"
case sql.Plugin != nil:
lang = fmt.Sprintf("process:%s", sql.Plugin.Plugin)
name = sql.Plugin.Plugin
}
packageRegion := trace.StartRegion(gctx, "package")
trace.Logf(gctx, "", "name=%s dir=%s plugin=%s", name, dir, lang)
result, failed := parse(gctx, name, dir, sql.SQL, combo, parseOpts, errout)
if failed {
packageRegion.End()
errored = true
return nil
}
if err := rp.ProcessResult(gctx, combo, sql, result); err != nil {
fmt.Fprintf(errout, "# package %s\n", name)
fmt.Fprintf(errout, "error generating code: %s\n", err)
errored = true
}
packageRegion.End()
return nil
})
}
if err := grp.Wait(); err != nil {
return err
}
if errored {
for i, _ := range stderrs {
if _, err := io.Copy(stderr, &stderrs[i]); err != nil {
return err
}
}
return fmt.Errorf("errored")
}
return nil
}

92
internal/cmd/push.go Normal file
View File

@@ -0,0 +1,92 @@
package cmd
import (
"context"
"fmt"
"os"
"sync"
"github.com/spf13/cobra"
"github.com/sqlc-dev/sqlc/internal/bundler"
"github.com/sqlc-dev/sqlc/internal/compiler"
"github.com/sqlc-dev/sqlc/internal/config"
)
func init() {
pushCmd.Flags().StringSliceP("tag", "t", nil, "tag this push with a value")
}
var pushCmd = &cobra.Command{
Use: "push",
Aliases: []string{"upload"},
Short: "Push the schema, queries, and configuration for this project",
RunE: func(cmd *cobra.Command, args []string) error {
stderr := cmd.ErrOrStderr()
dir, name := getConfigPath(stderr, cmd.Flag("file"))
tags, err := cmd.Flags().GetStringSlice("tag")
if err != nil {
return err
}
opts := &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
Tags: tags,
}
if err := Push(cmd.Context(), dir, name, opts); err != nil {
fmt.Fprintf(stderr, "error pushing: %s\n", err)
os.Exit(1)
}
return nil
},
}
type pusher struct {
m sync.Mutex
results []*bundler.QuerySetArchive
}
func (g *pusher) Pairs(ctx context.Context, conf *config.Config) []OutputPair {
var pairs []OutputPair
for _, sql := range conf.SQL {
pairs = append(pairs, OutputPair{
SQL: sql,
})
}
return pairs
}
func (g *pusher) ProcessResult(ctx context.Context, combo config.CombinedSettings, sql OutputPair, result *compiler.Result) error {
req := codeGenRequest(result, combo)
g.m.Lock()
g.results = append(g.results, &bundler.QuerySetArchive{
Name: sql.Name,
Schema: sql.Schema,
Queries: sql.Queries,
Request: req,
})
g.m.Unlock()
return nil
}
func Push(ctx context.Context, dir, filename string, opts *Options) error {
e := opts.Env
stderr := opts.Stderr
configPath, conf, err := readConfig(stderr, dir, filename)
if err != nil {
return err
}
up := bundler.NewUploader(configPath, dir, conf)
if err := up.Validate(); err != nil {
return err
}
p := &pusher{}
if err := Process(ctx, p, dir, filename, opts); err != nil {
return err
}
if e.DryRun {
return up.DumpRequestOut(ctx, p.results)
} else {
return up.Upload(ctx, p.results, opts.Tags)
}
}

233
internal/cmd/shim.go Normal file
View File

@@ -0,0 +1,233 @@
package cmd
import (
"github.com/sqlc-dev/sqlc/internal/compiler"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/config/convert"
"github.com/sqlc-dev/sqlc/internal/info"
"github.com/sqlc-dev/sqlc/internal/plugin"
"github.com/sqlc-dev/sqlc/internal/sql/catalog"
)
func pluginSettings(r *compiler.Result, cs config.CombinedSettings) *plugin.Settings {
return &plugin.Settings{
Version: cs.Global.Version,
Engine: string(cs.Package.Engine),
Schema: []string(cs.Package.Schema),
Queries: []string(cs.Package.Queries),
Codegen: pluginCodegen(cs, cs.Codegen),
}
}
func pluginCodegen(cs config.CombinedSettings, s config.Codegen) *plugin.Codegen {
opts, err := convert.YAMLtoJSON(s.Options)
if err != nil {
panic(err)
}
cg := &plugin.Codegen{
Out: s.Out,
Plugin: s.Plugin,
Options: opts,
}
for _, p := range cs.Global.Plugins {
if p.Name == s.Plugin {
cg.Env = p.Env
cg.Process = pluginProcess(p)
cg.Wasm = pluginWASM(p)
return cg
}
}
return cg
}
func pluginProcess(p config.Plugin) *plugin.Codegen_Process {
if p.Process != nil {
return &plugin.Codegen_Process{
Cmd: p.Process.Cmd,
}
}
return nil
}
func pluginWASM(p config.Plugin) *plugin.Codegen_WASM {
if p.WASM != nil {
return &plugin.Codegen_WASM{
Url: p.WASM.URL,
Sha256: p.WASM.SHA256,
}
}
return nil
}
func pluginCatalog(c *catalog.Catalog) *plugin.Catalog {
var schemas []*plugin.Schema
for _, s := range c.Schemas {
var enums []*plugin.Enum
var cts []*plugin.CompositeType
for _, typ := range s.Types {
switch typ := typ.(type) {
case *catalog.Enum:
enums = append(enums, &plugin.Enum{
Name: typ.Name,
Comment: typ.Comment,
Vals: typ.Vals,
})
case *catalog.CompositeType:
cts = append(cts, &plugin.CompositeType{
Name: typ.Name,
Comment: typ.Comment,
})
}
}
var tables []*plugin.Table
for _, t := range s.Tables {
var columns []*plugin.Column
for _, c := range t.Columns {
l := -1
if c.Length != nil {
l = *c.Length
}
columns = append(columns, &plugin.Column{
Name: c.Name,
Type: &plugin.Identifier{
Catalog: c.Type.Catalog,
Schema: c.Type.Schema,
Name: c.Type.Name,
},
Comment: c.Comment,
NotNull: c.IsNotNull,
Unsigned: c.IsUnsigned,
IsArray: c.IsArray,
ArrayDims: int32(c.ArrayDims),
Length: int32(l),
Table: &plugin.Identifier{
Catalog: t.Rel.Catalog,
Schema: t.Rel.Schema,
Name: t.Rel.Name,
},
})
}
tables = append(tables, &plugin.Table{
Rel: &plugin.Identifier{
Catalog: t.Rel.Catalog,
Schema: t.Rel.Schema,
Name: t.Rel.Name,
},
Columns: columns,
Comment: t.Comment,
})
}
schemas = append(schemas, &plugin.Schema{
Comment: s.Comment,
Name: s.Name,
Tables: tables,
Enums: enums,
CompositeTypes: cts,
})
}
return &plugin.Catalog{
Name: c.Name,
DefaultSchema: c.DefaultSchema,
Comment: c.Comment,
Schemas: schemas,
}
}
func pluginQueries(r *compiler.Result) []*plugin.Query {
var out []*plugin.Query
for _, q := range r.Queries {
var params []*plugin.Parameter
var columns []*plugin.Column
for _, c := range q.Columns {
columns = append(columns, pluginQueryColumn(c))
}
for _, p := range q.Params {
params = append(params, pluginQueryParam(p))
}
var iit *plugin.Identifier
if q.InsertIntoTable != nil {
iit = &plugin.Identifier{
Catalog: q.InsertIntoTable.Catalog,
Schema: q.InsertIntoTable.Schema,
Name: q.InsertIntoTable.Name,
}
}
out = append(out, &plugin.Query{
Name: q.Metadata.Name,
Cmd: q.Metadata.Cmd,
Text: q.SQL,
Comments: q.Metadata.Comments,
Columns: columns,
Params: params,
Filename: q.Metadata.Filename,
InsertIntoTable: iit,
})
}
return out
}
func pluginQueryColumn(c *compiler.Column) *plugin.Column {
l := -1
if c.Length != nil {
l = *c.Length
}
out := &plugin.Column{
Name: c.Name,
OriginalName: c.OriginalName,
Comment: c.Comment,
NotNull: c.NotNull,
Unsigned: c.Unsigned,
IsArray: c.IsArray,
ArrayDims: int32(c.ArrayDims),
Length: int32(l),
IsNamedParam: c.IsNamedParam,
IsFuncCall: c.IsFuncCall,
IsSqlcSlice: c.IsSqlcSlice,
}
if c.Type != nil {
out.Type = &plugin.Identifier{
Catalog: c.Type.Catalog,
Schema: c.Type.Schema,
Name: c.Type.Name,
}
} else {
out.Type = &plugin.Identifier{
Name: c.DataType,
}
}
if c.Table != nil {
out.Table = &plugin.Identifier{
Catalog: c.Table.Catalog,
Schema: c.Table.Schema,
Name: c.Table.Name,
}
}
if c.EmbedTable != nil {
out.EmbedTable = &plugin.Identifier{
Catalog: c.EmbedTable.Catalog,
Schema: c.EmbedTable.Schema,
Name: c.EmbedTable.Name,
}
}
return out
}
func pluginQueryParam(p compiler.Parameter) *plugin.Parameter {
return &plugin.Parameter{
Number: int32(p.Number),
Column: pluginQueryColumn(p.Column),
}
}
func codeGenRequest(r *compiler.Result, settings config.CombinedSettings) *plugin.GenerateRequest {
return &plugin.GenerateRequest{
Settings: pluginSettings(r, settings),
Catalog: pluginCatalog(r.Catalog),
Queries: pluginQueries(r),
SqlcVersion: info.Version,
}
}

156
internal/cmd/verify.go Normal file
View File

@@ -0,0 +1,156 @@
package cmd
import (
"context"
"database/sql"
"errors"
"fmt"
"log/slog"
"os"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/spf13/cobra"
"google.golang.org/protobuf/proto"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/dbmanager"
"github.com/sqlc-dev/sqlc/internal/migrations"
"github.com/sqlc-dev/sqlc/internal/plugin"
"github.com/sqlc-dev/sqlc/internal/quickdb"
pb "github.com/sqlc-dev/sqlc/internal/quickdb/v1"
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
)
func init() {
verifyCmd.Flags().String("against", "", "compare against this tag")
}
var verifyCmd = &cobra.Command{
Use: "verify",
Short: "Verify schema, queries, and configuration for this project",
RunE: func(cmd *cobra.Command, args []string) error {
stderr := cmd.ErrOrStderr()
dir, name := getConfigPath(stderr, cmd.Flag("file"))
against, err := cmd.Flags().GetString("against")
if err != nil {
return err
}
opts := &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
Against: against,
}
if err := Verify(cmd.Context(), dir, name, opts); err != nil {
fmt.Fprintf(stderr, "Error verifying queries: %s\n", err)
os.Exit(1)
}
return nil
},
}
func Verify(ctx context.Context, dir, filename string, opts *Options) error {
stderr := opts.Stderr
_, conf, err := readConfig(stderr, dir, filename)
if err != nil {
return err
}
client, err := quickdb.NewClientFromConfig(conf.Cloud)
if err != nil {
return fmt.Errorf("client init failed: %w", err)
}
manager := dbmanager.NewClient(conf.Servers)
// Get query sets from a previous archive by tag. If no tag is provided, get
// the latest query sets.
previous, err := client.GetQuerySets(ctx, &pb.GetQuerySetsRequest{
Tag: opts.Against,
})
if err != nil {
return err
}
// Create a mapping of name to query set
existing := map[string]config.SQL{}
for _, qs := range conf.SQL {
existing[qs.Name] = qs
}
var verr error
for _, qs := range previous.QuerySets {
// TODO: Create a function for this so that we can return early on errors
check := func() error {
if qs.Name == "" {
return fmt.Errorf("unnamed query set")
}
current, found := existing[qs.Name]
if !found {
return fmt.Errorf("unknown query set: %s", qs.Name)
}
// Read the schema files into memory, removing rollback statements
var ddl []string
files, err := sqlpath.Glob(current.Schema)
if err != nil {
return err
}
for _, schema := range files {
contents, err := os.ReadFile(schema)
if err != nil {
return fmt.Errorf("read file: %w", err)
}
ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
}
var codegen plugin.GenerateRequest
if err := proto.Unmarshal(qs.CodegenRequest.Contents, &codegen); err != nil {
return err
}
// Create (or re-use) a database to verify against
resp, err := manager.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
Engine: string(current.Engine),
Migrations: ddl,
})
if err != nil {
return err
}
db, err := sql.Open("pgx", resp.Uri)
if err != nil {
return err
}
defer db.Close()
var qerr error
for _, query := range codegen.Queries {
stmt, err := db.PrepareContext(ctx, query.Text)
if err != nil {
fmt.Fprintf(stderr, "Failed to prepare the following query:\n")
fmt.Fprintf(stderr, "%s\n", query.Text)
fmt.Fprintf(stderr, "Error was: %s\n", err)
qerr = err
continue
}
if err := stmt.Close(); err != nil {
slog.Error("stmt.Close failed", "err", err)
}
}
return qerr
}
if err := check(); err != nil {
verr = errors.New("errored")
fmt.Fprintf(stderr, "FAIL\t%s\n", qs.Name)
fmt.Fprintf(stderr, " ERROR\t%s\n", err)
} else {
fmt.Fprintf(stderr, "ok\t%s\n", qs.Name)
}
}
return verr
}

709
internal/cmd/vet.go Normal file
View File

@@ -0,0 +1,709 @@
package cmd
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"github.com/sqlc-dev/sqlc/internal/constants"
"io"
"log"
"os"
"path/filepath"
"runtime/trace"
"slices"
"strings"
"sync"
"time"
_ "github.com/go-sql-driver/mysql"
"cel.dev/cel-go/cel"
"cel.dev/cel-go/ext"
"github.com/jackc/pgx/v5"
"github.com/spf13/cobra"
"google.golang.org/protobuf/encoding/protojson"
"github.com/sqlc-dev/sqlc/internal/config"
"github.com/sqlc-dev/sqlc/internal/dbmanager"
"github.com/sqlc-dev/sqlc/internal/debug"
"github.com/sqlc-dev/sqlc/internal/migrations"
"github.com/sqlc-dev/sqlc/internal/opts"
"github.com/sqlc-dev/sqlc/internal/plugin"
"github.com/sqlc-dev/sqlc/internal/quickdb"
"github.com/sqlc-dev/sqlc/internal/shfmt"
"github.com/sqlc-dev/sqlc/internal/sql/sqlpath"
"github.com/sqlc-dev/sqlc/internal/sqlcdebug"
"github.com/sqlc-dev/sqlc/internal/vet"
)
var (
debugDumpExplain = sqlcdebug.New("dumpexplain")
debugDumpVetEnv = sqlcdebug.New("dumpvetenv")
debugDatabases = sqlcdebug.New("databases")
)
var ErrFailedChecks = errors.New("failed checks")
var pjson = protojson.UnmarshalOptions{AllowPartial: true, DiscardUnknown: true}
func NewCmdVet() *cobra.Command {
return &cobra.Command{
Use: "vet",
Short: "Vet examines queries",
RunE: func(cmd *cobra.Command, args []string) error {
defer trace.StartRegion(cmd.Context(), "vet").End()
stderr := cmd.ErrOrStderr()
opts := &Options{
Env: ParseEnv(cmd),
Stderr: stderr,
}
dir, name := getConfigPath(stderr, cmd.Flag("file"))
if err := Vet(cmd.Context(), dir, name, opts); err != nil {
if !errors.Is(err, ErrFailedChecks) {
fmt.Fprintf(stderr, "%s\n", err)
}
os.Exit(1)
}
return nil
},
}
}
func Vet(ctx context.Context, dir, filename string, opts *Options) error {
e := opts.Env
stderr := opts.Stderr
configPath, conf, err := readConfig(stderr, dir, filename)
if err != nil {
return err
}
base := filepath.Base(configPath)
if err := config.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return err
}
if err := e.Validate(conf); err != nil {
fmt.Fprintf(stderr, "error validating %s: %s\n", base, err)
return err
}
env, err := cel.NewEnv(
cel.StdLib(),
ext.Strings(ext.StringsVersion(1)),
cel.Types(
&vet.Config{},
&vet.Query{},
&vet.PostgreSQL{},
&vet.MySQL{},
),
cel.Variable("query",
cel.ObjectType("vet.Query"),
),
cel.Variable("config",
cel.ObjectType("vet.Config"),
),
cel.Variable("postgresql",
cel.ObjectType("vet.PostgreSQL"),
),
cel.Variable("mysql",
cel.ObjectType("vet.MySQL"),
),
)
if err != nil {
return fmt.Errorf("new CEL env error: %s", err)
}
rules := map[string]rule{
constants.QueryRuleDbPrepare: {NeedsPrepare: true},
}
for _, c := range conf.Rules {
if c.Name == "" {
return fmt.Errorf("rules require a name")
}
if _, found := rules[c.Name]; found {
return fmt.Errorf("type-check error: a rule with the name '%s' already exists", c.Name)
}
if c.Rule == "" {
return fmt.Errorf("type-check error: %s is empty", c.Name)
}
ast, issues := env.Compile(c.Rule)
if issues != nil && issues.Err() != nil {
return fmt.Errorf("type-check error: %s %s", c.Name, issues.Err())
}
prg, err := env.Program(ast)
if err != nil {
return fmt.Errorf("program construction error: %s %s", c.Name, err)
}
rule := rule{Program: &prg, Message: c.Msg}
// TODO There's probably a nicer way to do this from the ast
// https://pkg.go.dev/cel.dev/cel-go/common/ast#AllMatcher
if strings.Contains(c.Rule, "postgresql.explain") ||
strings.Contains(c.Rule, "mysql.explain") {
rule.NeedsExplain = true
}
rules[c.Name] = rule
}
c := checker{
Rules: rules,
Conf: conf,
Dir: dir,
Env: env,
Stderr: stderr,
OnlyManagedDB: debugDatabases.Value() == "managed",
Replacer: shfmt.NewReplacer(nil),
}
errored := false
for _, sql := range conf.SQL {
if err := c.checkSQL(ctx, sql); err != nil {
if !errors.Is(err, ErrFailedChecks) {
fmt.Fprintf(stderr, "%s\n", err)
}
errored = true
}
}
if errored {
return ErrFailedChecks
}
return nil
}
type preparer interface {
Prepare(context.Context, string, string) error
}
type pgxConn struct {
c *pgx.Conn
}
func (p *pgxConn) Prepare(ctx context.Context, name, query string) error {
_, err := p.c.Prepare(ctx, name, query)
return err
}
// Return a default value for a PostgreSQL column based on its type. Returns nil
// if the type is unknown.
func pgDefaultValue(col *plugin.Column) any {
if col == nil {
return nil
}
if col.Type == nil {
return nil
}
typname := strings.TrimPrefix(col.Type.Name, "pg_catalog.")
switch typname {
case "any", "void":
if col.IsArray {
return []any{}
} else {
return nil
}
case "anyarray":
return []any{}
case "bool", "boolean":
if col.IsArray {
return []bool{}
} else {
return false
}
case "double", "double precision", "real":
if col.IsArray {
return []float32{}
} else {
return 0.1
}
case "json", "jsonb":
if col.IsArray {
return []string{}
} else {
return "{}"
}
case "citext", "string", "text", "varchar":
if col.IsArray {
return []string{}
} else {
return ""
}
case "bigint", "bigserial", "integer", "int", "int2", "int4", "int8", "serial":
if col.IsArray {
return []int{}
} else {
return 1
}
case "date", "time", "timestamp", "timestamptz":
if col.IsArray {
return []time.Time{}
} else {
return time.Time{}
}
case "uuid":
if col.IsArray {
return []string{}
} else {
return "00000000-0000-0000-0000-000000000000"
}
case "numeric", "decimal":
if col.IsArray {
return []string{}
} else {
return "0.1"
}
case "inet":
if col.IsArray {
return []string{}
} else {
return "192.168.0.1/24"
}
case "cidr":
if col.IsArray {
return []string{}
} else {
return "192.168.1/24"
}
default:
return nil
}
}
// Return a default value for a MySQL column based on its type. Returns nil
// if the type is unknown.
func mysqlDefaultValue(col *plugin.Column) any {
if col == nil {
return nil
}
if col.Type == nil {
return nil
}
switch col.Type.Name {
case "any":
return nil
case "bool":
return false
case "int", "bigint", "mediumint", "smallint", "tinyint", "bit":
return 1
case "decimal": // "numeric", "dec", "fixed"
// No perfect choice here to avoid "Impossible WHERE" but I think
// 0.1 is decent. It works for all cases where `scale` > 0 which
// should be the majority. For more information refer to
// https://dev.mysql.com/doc/refman/8.1/en/fixed-point-types.html.
return 0.1
case "float", "double":
return 0.1
case "date":
return "0000-00-00"
case "datetime", "timestamp":
return "0000-00-00 00:00:00"
case "time":
return "00:00:00"
case "year":
return "0000"
case "char", "varchar", "binary", "varbinary", "tinyblob", "blob",
"mediumblob", "longblob", "tinytext", "text", "mediumtext", "longtext":
return ""
case "json":
return "{}"
default:
return nil
}
}
func (p *pgxConn) Explain(ctx context.Context, query string, args ...*plugin.Parameter) (*vetEngineOutput, error) {
eQuery := "EXPLAIN (ANALYZE false, VERBOSE, COSTS, SETTINGS, BUFFERS, FORMAT JSON) " + query
eArgs := make([]any, len(args))
for i, a := range args {
eArgs[i] = pgDefaultValue(a.Column)
}
row := p.c.QueryRow(ctx, eQuery, eArgs...)
var result []json.RawMessage
if err := row.Scan(&result); err != nil {
return nil, err
}
if debugDumpExplain.Value() == "1" {
fmt.Println(eQuery, "with args", eArgs)
fmt.Println(string(result[0]))
}
var explain vet.PostgreSQLExplain
if err := pjson.Unmarshal(result[0], &explain); err != nil {
return nil, err
}
return &vetEngineOutput{PostgreSQL: &vet.PostgreSQL{Explain: &explain}}, nil
}
type dbPreparer struct {
db *sql.DB
}
func (p *dbPreparer) Prepare(ctx context.Context, name, query string) error {
s, err := p.db.PrepareContext(ctx, query)
if s != nil {
s.Close()
}
return err
}
type explainer interface {
Explain(context.Context, string, ...*plugin.Parameter) (*vetEngineOutput, error)
}
type mysqlExplainer struct {
*sql.DB
}
func (me *mysqlExplainer) Explain(ctx context.Context, query string, args ...*plugin.Parameter) (*vetEngineOutput, error) {
eQuery := "EXPLAIN FORMAT=JSON " + query
eArgs := make([]any, len(args))
for i, a := range args {
eArgs[i] = mysqlDefaultValue(a.Column)
}
row := me.QueryRowContext(ctx, eQuery, eArgs...)
var result json.RawMessage
if err := row.Scan(&result); err != nil {
return nil, err
}
if debugDumpExplain.Value() == "1" {
fmt.Println(eQuery, "with args", eArgs)
fmt.Println(string(result))
}
var explain vet.MySQLExplain
if err := pjson.Unmarshal(result, &explain); err != nil {
return nil, err
}
if explain.QueryBlock.Message != "" {
return nil, fmt.Errorf("mysql explain: %s", explain.QueryBlock.Message)
}
return &vetEngineOutput{MySQL: &vet.MySQL{Explain: &explain}}, nil
}
type rule struct {
Program *cel.Program
Message string
NeedsPrepare bool
NeedsExplain bool
}
type checker struct {
Rules map[string]rule
Conf *config.Config
Dir string
Env *cel.Env
Stderr io.Writer
OnlyManagedDB bool
Client dbmanager.Client
clientOnce sync.Once
Replacer *shfmt.Replacer
}
// isInMemorySQLite checks if a SQLite URI refers to an in-memory database
func isInMemorySQLite(uri string) bool {
if uri == ":memory:" || uri == "" {
return true
}
// Check for file URI with mode=memory parameter
// e.g., "file:test?mode=memory&cache=shared"
if strings.Contains(uri, "mode=memory") {
return true
}
return false
}
func (c *checker) fetchDatabaseUri(ctx context.Context, s config.SQL) (string, func() error, error) {
cleanup := func() error {
return nil
}
if s.Database == nil {
panic("fetch database URI called with nil database")
}
if !s.Database.Managed {
uri, err := c.DSN(s.Database.URI)
return uri, cleanup, err
}
// Initialize the client exactly once, even if called concurrently
c.clientOnce.Do(func() {
c.Client = dbmanager.NewClient(c.Conf.Servers)
})
var ddl []string
files, err := sqlpath.Glob(s.Schema)
if err != nil {
return "", cleanup, err
}
for _, schema := range files {
contents, err := os.ReadFile(schema)
if err != nil {
return "", cleanup, fmt.Errorf("read file: %w", err)
}
ddl = append(ddl, migrations.RemoveRollbackStatements(string(contents)))
}
resp, err := c.Client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{
Engine: string(s.Engine),
Migrations: ddl,
})
if err != nil {
return "", cleanup, fmt.Errorf("managed: create database: %w", err)
}
var uri string
switch s.Engine {
case config.EngineMySQL:
dburi, err := quickdb.MySQLReformatURI(resp.Uri)
if err != nil {
return "", cleanup, fmt.Errorf("reformat uri: %w", err)
}
uri = dburi
default:
uri = resp.Uri
}
return uri, cleanup, nil
}
func (c *checker) DSN(dsn string) (string, error) {
return c.Replacer.Replace(dsn), nil
}
func (c *checker) checkSQL(ctx context.Context, s config.SQL) error {
// TODO: Create a separate function for this logic so we can
combo := config.Combine(*c.Conf, s)
// TODO: This feels like a hack that will bite us later
joined := make([]string, 0, len(s.Schema))
for _, s := range s.Schema {
joined = append(joined, filepath.Join(c.Dir, s))
}
s.Schema = joined
joined = make([]string, 0, len(s.Queries))
for _, q := range s.Queries {
joined = append(joined, filepath.Join(c.Dir, q))
}
s.Queries = joined
var name string
parseOpts := opts.Parser{}
result, failed := parse(ctx, name, c.Dir, s, combo, parseOpts, c.Stderr)
if failed {
return ErrFailedChecks
}
var prep preparer
var expl explainer
if s.Database != nil { // TODO only set up a database connection if a rule evaluation requires it
if s.Database.URI != "" && c.OnlyManagedDB {
return fmt.Errorf("database: connections disabled via SQLCDEBUG=databases=managed")
}
dburl, cleanup, err := c.fetchDatabaseUri(ctx, s)
if err != nil {
return err
}
defer func() {
if err := cleanup(); err != nil {
fmt.Fprintf(c.Stderr, "error cleaning up: %s\n", err)
}
}()
switch s.Engine {
case config.EnginePostgreSQL:
conn, err := pgx.Connect(ctx, dburl)
if err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
if err := conn.Ping(ctx); err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
defer conn.Close(ctx)
pConn := &pgxConn{conn}
prep = pConn
expl = pConn
case config.EngineMySQL:
db, err := sql.Open("mysql", dburl)
if err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
if err := db.PingContext(ctx); err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
defer db.Close()
prep = &dbPreparer{db}
expl = &mysqlExplainer{db}
case config.EngineSQLite:
db, err := sql.Open("sqlite3", dburl)
if err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
if err := db.PingContext(ctx); err != nil {
return fmt.Errorf("database: connection error: %s", err)
}
defer db.Close()
// For in-memory SQLite databases, apply migrations
if isInMemorySQLite(dburl) {
files, err := sqlpath.Glob(s.Schema)
if err != nil {
return fmt.Errorf("schema: %w", err)
}
for _, schema := range files {
contents, err := os.ReadFile(schema)
if err != nil {
return fmt.Errorf("read schema file: %w", err)
}
ddl := migrations.RemoveRollbackStatements(string(contents))
if _, err := db.ExecContext(ctx, ddl); err != nil {
return fmt.Errorf("apply schema %s: %w", schema, err)
}
}
}
prep = &dbPreparer{db}
// SQLite really doesn't want us to depend on the output of EXPLAIN
// QUERY PLAN: https://www.sqlite.org/eqp.html
expl = nil
default:
return fmt.Errorf("unsupported database uri: %s", s.Engine)
}
}
errored := false
req := codeGenRequest(result, combo)
cfg := vetConfig(req)
for i, query := range req.Queries {
md := result.Queries[i].Metadata
if md.Flags[constants.QueryFlagSqlcVetDisable] {
// If the vet disable flag is specified without any rules listed, all rules are ignored.
if len(md.RuleSkiplist) == 0 {
if debug.Active {
log.Printf("Skipping all vet rules for query: %s\n", query.Name)
}
continue
}
// Rules which are listed to be disabled but not declared in the config file are rejected.
for r := range md.RuleSkiplist {
if !slices.Contains(s.Rules, r) {
fmt.Fprintf(c.Stderr, "%s: %s: rule-check error: rule %q does not exist in the config file\n", query.Filename, query.Name, r)
errored = true
}
}
}
evalMap := map[string]any{
"query": vetQuery(query),
"config": cfg,
}
for _, name := range s.Rules {
if _, skip := md.RuleSkiplist[name]; skip {
if debug.Active {
log.Printf("Skipping vet rule %q for query: %s\n", name, query.Name)
}
} else {
rule, ok := c.Rules[name]
if !ok {
return fmt.Errorf("type-check error: a rule with the name '%s' does not exist", name)
}
if rule.NeedsPrepare {
if prep == nil {
fmt.Fprintf(c.Stderr, "%s: %s: %s: error preparing query: database connection required\n", query.Filename, query.Name, name)
errored = true
continue
}
prepName := fmt.Sprintf("sqlc_vet_%d_%d", time.Now().Unix(), i)
if err := prep.Prepare(ctx, prepName, query.Text); err != nil {
fmt.Fprintf(c.Stderr, "%s: %s: %s: error preparing query: %s\n", query.Filename, query.Name, name, err)
errored = true
continue
}
}
// short-circuit for "sqlc/db-prepare" rule which doesn't have a CEL program
if rule.Program == nil {
continue
}
// Get explain output for this query if we need it
_, pgsqlOK := evalMap["postgresql"]
_, mysqlOK := evalMap["mysql"]
if rule.NeedsExplain && !(pgsqlOK || mysqlOK) {
if expl == nil {
fmt.Fprintf(c.Stderr, "%s: %s: %s: error explaining query: database connection required\n", query.Filename, query.Name, name)
errored = true
continue
}
engineOutput, err := expl.Explain(ctx, query.Text, query.Params...)
if err != nil {
fmt.Fprintf(c.Stderr, "%s: %s: %s: error explaining query: %s\n", query.Filename, query.Name, name, err)
errored = true
continue
}
evalMap["postgresql"] = engineOutput.PostgreSQL
evalMap["mysql"] = engineOutput.MySQL
}
if debugDumpVetEnv.Value() == "1" {
fmt.Printf("vars for rule '%s' evaluating against query '%s':\n", name, query.Name)
debug.DumpAsJSON(evalMap)
}
out, _, err := (*rule.Program).Eval(evalMap)
if err != nil {
return err
}
tripped, ok := out.Value().(bool)
if !ok {
return fmt.Errorf("expression returned non-bool value: %v", out.Value())
}
if tripped {
// TODO: Get line numbers in the output
if rule.Message == "" {
fmt.Fprintf(c.Stderr, "%s: %s: %s\n", query.Filename, query.Name, name)
} else {
fmt.Fprintf(c.Stderr, "%s: %s: %s: %s\n", query.Filename, query.Name, name, rule.Message)
}
errored = true
}
}
}
}
if errored {
return ErrFailedChecks
}
return nil
}
func vetConfig(req *plugin.GenerateRequest) *vet.Config {
return &vet.Config{
Version: req.Settings.Version,
Engine: req.Settings.Engine,
Schema: req.Settings.Schema,
Queries: req.Settings.Queries,
}
}
func vetQuery(q *plugin.Query) *vet.Query {
var params []*vet.Parameter
for _, p := range q.Params {
params = append(params, &vet.Parameter{
Number: p.Number,
})
}
return &vet.Query{
Sql: q.Text,
Name: q.Name,
Cmd: strings.TrimPrefix(q.Cmd, ":"),
Params: params,
}
}
type vetEngineOutput struct {
PostgreSQL *vet.PostgreSQL
MySQL *vet.MySQL
}

View File

@@ -0,0 +1,7 @@
package cmd
import (
_ "github.com/ncruces/go-sqlite3/driver"
_ "github.com/sqlc-dev/sqlc/internal/sqlite3ext"
)