upstream-sqlc/internal/endtoend/endtoend_test.go

package main

import (
	"bytes"
	"context"
	"fmt"
	"os"
	osexec "os/exec"
	"path/filepath"
	"runtime"
	"slices"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	"github.com/google/go-cmp/cmp/cmpopts"

	"github.com/sqlc-dev/sqlc/internal/cmd"
	"github.com/sqlc-dev/sqlc/internal/config"
	"github.com/sqlc-dev/sqlc/internal/opts"
	"github.com/sqlc-dev/sqlc/internal/sqlcdebug"
	"github.com/sqlc-dev/sqlc/internal/sqltest/docker"
	"github.com/sqlc-dev/sqlc/internal/sqltest/native"
)

// withSQLCDEBUG installs the given SQLCDEBUG-formatted string for the
// duration of the test and restores the empty default afterwards.
//
// Callers in TestReplay are sequential, so this does not need a mutex:
// Go's test scheduler does not run TestReplay concurrently with the
// parallel top-level tests in this package.
func withSQLCDEBUG(t *testing.T, raw string) func() {
	t.Helper()
	if raw == "" {
		return func() {}
	}
	sqlcdebug.Update(raw)
	return func() { sqlcdebug.Update("") }
}

func lineEndings() cmp.Option {
	return cmp.Transformer("LineEndings", func(in string) string {
		// Replace Windows new lines with Unix newlines
		return strings.Replace(in, "\r\n", "\n", -1)
	})
}

func stderrTransformer() cmp.Option {
	return cmp.Transformer("Stderr", func(in string) string {
		s := strings.Replace(in, "\r", "", -1)
		return strings.Replace(s, "\\", "/", -1)
	})
}

func TestExamples(t *testing.T) {
	t.Parallel()
	ctx := context.Background()

	examples, err := filepath.Abs(filepath.Join("..", "..", "examples"))
	if err != nil {
		t.Fatal(err)
	}

	files, err := os.ReadDir(examples)
	if err != nil {
		t.Fatal(err)
	}

	for _, replay := range files {
		if !replay.IsDir() {
			continue
		}
		tc := replay.Name()
		t.Run(tc, func(t *testing.T) {
			t.Parallel()
			path := filepath.Join(examples, tc)
			var stderr bytes.Buffer
			opts := &cmd.Options{
				Env:    cmd.Env{},
				Stderr: &stderr,
			}
			output, err := cmd.Generate(ctx, path, "", opts)
			if err != nil {
				t.Fatalf("sqlc generate failed: %s", stderr.String())
			}
			cmpDirectory(t, path, output)
		})
	}
}

func BenchmarkExamples(b *testing.B) {
	ctx := context.Background()
	examples, err := filepath.Abs(filepath.Join("..", "..", "examples"))
	if err != nil {
		b.Fatal(err)
	}
	files, err := os.ReadDir(examples)
	if err != nil {
		b.Fatal(err)
	}
	for _, replay := range files {
		if !replay.IsDir() {
			continue
		}
		tc := replay.Name()
		b.Run(tc, func(b *testing.B) {
			path := filepath.Join(examples, tc)
			for i := 0; i < b.N; i++ {
				var stderr bytes.Buffer
				opts := &cmd.Options{
					Env:    cmd.Env{},
					Stderr: &stderr,
				}
				cmd.Generate(ctx, path, "", opts)
			}
		})
	}
}

type textContext struct {
	Mutate  func(*testing.T, string) func(*config.Config)
	Enabled func() bool
	// Experiments names the experiments every case in this context runs
	// with. A case's own SQLCEXPERIMENT is appended to these, so a case can
	// still turn one back off with the "no" prefix.
	Experiments func() []string
}

func TestReplay(t *testing.T) {
	// Ensure that this environment variable is always set to true when running
	// end-to-end tests
	os.Setenv("SQLC_DUMMY_VALUE", "true")

	// t.Parallel()
	ctx := context.Background()

	var mysqlURI, postgresURI string

	// First, check environment variables
	if uri := os.Getenv("POSTGRESQL_SERVER_URI"); uri != "" {
		postgresURI = uri
	}
	if uri := os.Getenv("MYSQL_SERVER_URI"); uri != "" {
		mysqlURI = uri
	}

	// Try Docker for any missing databases
	if postgresURI == "" || mysqlURI == "" {
		if err := docker.Installed(); err == nil {
			if postgresURI == "" {
				host, err := docker.StartPostgreSQLServer(ctx)
				if err != nil {
					t.Logf("docker postgresql startup failed: %s", err)
				} else {
					postgresURI = host
				}
			}
			if mysqlURI == "" {
				host, err := docker.StartMySQLServer(ctx)
				if err != nil {
					t.Logf("docker mysql startup failed: %s", err)
				} else {
					mysqlURI = host
				}
			}
		}
	}

	// Try native installation for any missing databases (Linux only)
	if postgresURI == "" || mysqlURI == "" {
		if err := native.Supported(); err == nil {
			if postgresURI == "" {
				host, err := native.StartPostgreSQLServer(ctx)
				if err != nil {
					t.Logf("native postgresql startup failed: %s", err)
				} else {
					postgresURI = host
				}
			}
			if mysqlURI == "" {
				host, err := native.StartMySQLServer(ctx)
				if err != nil {
					t.Logf("native mysql startup failed: %s", err)
				} else {
					mysqlURI = host
				}
			}
		}
	}

	// Log which databases are available
	t.Logf("PostgreSQL available: %v (URI: %s)", postgresURI != "", postgresURI)
	t.Logf("MySQL available: %v (URI: %s)", mysqlURI != "", mysqlURI)

	contexts := map[string]textContext{
		"base": {
			Mutate:  func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} },
			Enabled: func() bool { return true },
		},
		"managed-db": {
			Mutate: func(t *testing.T, path string) func(*config.Config) {
				return func(c *config.Config) {
					// Add all servers - tests will fail if database isn't available
					c.Servers = []config.Server{
						{
							Name:   "postgres",
							Engine: config.EnginePostgreSQL,
							URI:    postgresURI,
						},
						{
							Name:   "mysql",
							Engine: config.EngineMySQL,
							URI:    mysqlURI,
						},
					}

					for i := range c.SQL {
						switch c.SQL[i].Engine {
						case config.EnginePostgreSQL:
							c.SQL[i].Database = &config.Database{
								Managed: true,
							}
						case config.EngineMySQL:
							c.SQL[i].Database = &config.Database{
								Managed: true,
							}
						case config.EngineSQLite:
							c.SQL[i].Database = &config.Database{
								Managed: true,
							}
						default:
							// pass
						}
					}
				}
			},
			Enabled: func() bool {
				// Enabled if at least one database URI is available
				return postgresURI != "" || mysqlURI != ""
			},
		},
		"core": {
			Mutate:      func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} },
			Experiments: func() []string { return []string{"coreanalyzer"} },
			Enabled: func() bool {
				// Running the whole corpus through the analysis core is opt-in
				// while the two paths still disagree. The core needs no
				// database, so nothing else gates this.
				return os.Getenv("SQLC_TEST_CORE") != ""
			},
		},
	}

	for name, testctx := range contexts {

		if !testctx.Enabled() {
			continue
		}

		for _, replay := range FindTests(t, "testdata", name) {
			tc := replay
			t.Run(filepath.Join(name, tc.Name), func(t *testing.T) {
				var stderr bytes.Buffer
				var output map[string]string
				var err error

				path, _ := filepath.Abs(tc.Path)
				args := tc.Exec
				if args == nil {
					args = &Exec{Command: "generate"}
				}
				expected := string(tc.Stderr)

				if args.Process != "" {
					_, err := osexec.LookPath(args.Process)
					if err != nil {
						t.Skipf("executable not found: %s %s", args.Process, err)
					}
				}

				if len(args.Contexts) > 0 {
					if !slices.Contains(args.Contexts, name) {
						t.Skipf("unsupported context: %s", name)
					}
				}

				if len(args.OS) > 0 {
					if !slices.Contains(args.OS, runtime.GOOS) {
						t.Skipf("unsupported os: %s", runtime.GOOS)
					}
				}

				experiments := args.Env["SQLCEXPERIMENT"]
				if testctx.Experiments != nil {
					experiments = strings.Join(append(testctx.Experiments(), experiments), ",")
				}

				opts := cmd.Options{
					Env: cmd.Env{
						Experiment: opts.ExperimentFromString(experiments),
					},
					Stderr:       &stderr,
					MutateConfig: testctx.Mutate(t, path),
				}

				release := withSQLCDEBUG(t, args.Env["SQLCDEBUG"])
				defer release()

				switch args.Command {
				case "diff":
					err = cmd.Diff(ctx, path, "", &opts)
				case "generate":
					output, err = cmd.Generate(ctx, path, "", &opts)
					if err == nil {
						cmpDirectory(t, path, output)
					}
				case "vet":
					err = cmd.Vet(ctx, path, "", &opts)
				case "parse", "analyze", "fmt":
					// These commands are flag-driven (fmt cases pass --diff so
					// nothing is rewritten). Run them through the real CLI
					// entry point from inside the test directory so file
					// arguments resolve and the output stays independent of
					// the absolute path.
					var stdout bytes.Buffer
					wd, werr := os.Getwd()
					if werr != nil {
						t.Fatal(werr)
					}
					if cerr := os.Chdir(path); cerr != nil {
						t.Fatal(cerr)
					}
					code := cmd.Do(append([]string{args.Command}, args.Args...), nil, &stdout, &stderr)
					if cerr := os.Chdir(wd); cerr != nil {
						t.Fatal(cerr)
					}
					if code != 0 {
						err = fmt.Errorf("%s exited with code %d", args.Command, code)
					} else if diff := cmp.Diff(strings.TrimSpace(string(tc.Stdout)), strings.TrimSpace(stdout.String()), lineEndings()); diff != "" {
						t.Errorf("stdout differed (-want +got):\n%s", diff)
					}
				default:
					t.Fatalf("unknown command")
				}

				if len(expected) == 0 && err != nil {
					t.Fatalf("sqlc %s failed: %s", args.Command, stderr.String())
				}

				diff := cmp.Diff(
					strings.TrimSpace(expected),
					strings.TrimSpace(stderr.String()),
					stderrTransformer(),
				)
				if diff != "" {
					t.Fatalf("stderr differed (-want +got):\n%s", diff)
				}
			})
		}
	}
}

func cmpDirectory(t *testing.T, dir string, actual map[string]string) {
	expected := map[string]string{}
	var ff = func(path string, file os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if file.IsDir() {
			return nil
		}
		if !strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, ".kt") && !strings.HasSuffix(path, ".py") && !strings.HasSuffix(path, ".json") && !strings.HasSuffix(path, ".txt") {
			return nil
		}
		// TODO: Figure out a better way to ignore certain files
		if strings.HasSuffix(path, ".txt") && filepath.Base(path) != "hello.txt" {
			return nil
		}
		if filepath.Base(path) == "sqlc.json" {
			return nil
		}
		if filepath.Base(path) == "exec.json" {
			return nil
		}
		if strings.Contains(path, "/kotlin/build") {
			return nil
		}
		if strings.HasSuffix(path, "_test.go") || strings.Contains(path, "src/test/") {
			return nil
		}
		if strings.Contains(path, "/python/.venv") || strings.Contains(path, "/python/src/tests/") ||
			strings.HasSuffix(path, "__init__.py") || strings.Contains(path, "/python/src/dbtest/") ||
			strings.Contains(path, "/python/.mypy_cache") {
			return nil
		}
		blob, err := os.ReadFile(path)
		if err != nil {
			return err
		}
		expected[path] = string(blob)
		return nil
	}
	if err := filepath.Walk(dir, ff); err != nil {
		t.Fatal(err)
	}

	opts := []cmp.Option{
		cmpopts.EquateEmpty(),
		lineEndings(),
	}

	if !cmp.Equal(expected, actual, opts...) {
		t.Errorf("%s contents differ", dir)
		for name, contents := range expected {
			if actual[name] == "" {
				t.Errorf("%s is empty", name)
				return
			}
			if diff := cmp.Diff(contents, actual[name], opts...); diff != "" {
				t.Errorf("%s differed (-want +got):\n%s", name, diff)
			}
		}
	}
}

func BenchmarkReplay(b *testing.B) {
	ctx := context.Background()
	var dirs []string
	err := filepath.Walk("testdata", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.Name() == "sqlc.json" || info.Name() == "sqlc.yaml" || info.Name() == "sqlc.yml" {
			dirs = append(dirs, filepath.Dir(path))
			return filepath.SkipDir
		}
		return nil
	})
	if err != nil {
		b.Fatal(err)
	}
	for _, replay := range dirs {
		tc := replay
		b.Run(tc, func(b *testing.B) {
			path, _ := filepath.Abs(tc)
			for i := 0; i < b.N; i++ {
				var stderr bytes.Buffer
				opts := &cmd.Options{
					Env:    cmd.Env{},
					Stderr: &stderr,
				}
				cmd.Generate(ctx, path, "", opts)
			}
		})
	}
}