snapshot of https://github.com/sqlc-dev/sqlc.git (main, 2026-09-16; blobs > 131072B omitted)

internal/goldeneye/postgresql/postgresql.go

@@ -0,0 +1,364 @@
+// Package postgresql generates the PostgreSQL dialect seed under
+// internal/engine/postgresql/dialect from a live server: pg_catalog's
+// functions, the relations of pg_catalog and information_schema, and one
+// directory per contrib extension holding the types and functions CREATE
+// EXTENSION adds. types.jsonl and operators.jsonl at the top of the dialect
+// are written by hand and are not this package's business.
+//
+// The server is named by POSTGRESQL_SERVER_URI and has to be the major
+// release in Major, since each release adds to its catalogs.
+package postgresql
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/jackc/pgx/v5"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
+)
+
+// Engine is the name of the engine directory the dialect lives under.
+const Engine = "postgresql"
+
+// Major is the PostgreSQL major release the dialect is generated from.
+// Bumping it is a deliberate change: every release adds functions and
+// catalog columns, so regenerate and review the dialect after changing it.
+const Major = 16
+
+// https://dba.stackexchange.com/questions/255412/how-to-select-functions-that-belong-in-a-given-extension-in-postgresql
+//
+// Extension functions are added to the public schema
+const extensionFuncs = `
+WITH extension_funcs AS (
+  SELECT p.oid
+  FROM pg_catalog.pg_extension AS e
+      INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
+      INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
+      INNER JOIN pg_catalog.pg_namespace AS ne ON (ne.oid = e.extnamespace)
+      INNER JOIN pg_catalog.pg_namespace AS np ON (np.oid = p.pronamespace)
+  WHERE d.deptype = 'e' AND e.extname = $1
+)
+SELECT p.proname as name,
+  format_type(p.prorettype, NULL),
+  array(select format_type(unnest(p.proargtypes), NULL)),
+  p.proargnames,
+  p.proargnames[p.pronargs-p.pronargdefaults+1:p.pronargs],
+  p.proargmodes::text[]
+FROM pg_catalog.pg_proc p
+JOIN extension_funcs ef ON ef.oid = p.oid
+WHERE pg_function_is_visible(p.oid)
+-- simply order all columns to keep subsequent runs stable
+ORDER BY 1, 2, 3, 4, 5;
+`
+
+// The types an extension defines, in the order they were declared so that a
+// type appears after the types it is built on. Array types are created
+// implicitly rather than by the extension's script, so they carry an internal
+// dependency and never show up here.
+const extensionTypes = `
+SELECT t.typname, t.typcategory
+FROM pg_catalog.pg_extension AS e
+    INNER JOIN pg_catalog.pg_depend AS d ON (d.refobjid = e.oid)
+    INNER JOIN pg_catalog.pg_type AS t ON (t.oid = d.objid)
+WHERE d.deptype = 'e'
+  AND d.classid = 'pg_catalog.pg_type'::regclass
+  AND e.extname = $1
+  AND t.typtype <> 'p'
+ORDER BY t.oid;
+`
+
+// Locate returns the server to generate from, named by POSTGRESQL_SERVER_URI.
+func Locate() (string, error) {
+	if url := os.Getenv("POSTGRESQL_SERVER_URI"); url != "" {
+		return url, nil
+	}
+	return "", errors.New("POSTGRESQL_SERVER_URI is not set")
+}
+
+// Version reports the release a server is.
+func Version(ctx context.Context, url string) (string, error) {
+	conn, err := pgx.Connect(ctx, url)
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close(ctx)
+	var version string
+	if err := conn.QueryRow(ctx, "SELECT version()").Scan(&version); err != nil {
+		return "", err
+	}
+	return version, nil
+}
+
+// checkVersion refuses a server of another major release than the dialect
+// is generated from, whose catalogs would differ from the committed ones
+// without anything being wrong.
+func checkVersion(ctx context.Context, conn *pgx.Conn) error {
+	var num string
+	if err := conn.QueryRow(ctx, "SELECT current_setting('server_version_num')").Scan(&num); err != nil {
+		return err
+	}
+	n, err := strconv.Atoi(num)
+	if err != nil {
+		return fmt.Errorf("server_version_num %q: %w", num, err)
+	}
+	if major := n / 10000; major != Major {
+		return fmt.Errorf("the dialect is generated from PostgreSQL %d, but the server is PostgreSQL %d", Major, major)
+	}
+	return nil
+}
+
+func clean(arg string) string {
+	arg = strings.TrimSpace(arg)
+	arg = strings.Replace(arg, "\"any\"", "any", -1)
+	arg = strings.Replace(arg, "\"char\"", "char", -1)
+	arg = strings.Replace(arg, "\"timestamp\"", "char", -1)
+	return arg
+}
+
+// encodeFunctions encodes procs as JSONL, one function per line, in the form
+// the seed package reads. Both the analysis core and the catalog the legacy
+// compiler builds load the result.
+func encodeFunctions(procs []Proc) ([]byte, error) {
+	funcs := make([]dialect.Function, 0, len(procs))
+	for _, proc := range procs {
+		fn := dialect.Function{Name: proc.Name, Returns: proc.ReturnTypeName()}
+		for _, arg := range proc.Args() {
+			a := dialect.Arg{Name: arg.Name, Type: arg.TypeName(), HasDefault: arg.HasDefault}
+			if arg.Mode != "" && arg.Mode != "i" {
+				a.Mode = arg.Mode
+			}
+			fn.Args = append(fn.Args, a)
+		}
+		funcs = append(funcs, fn)
+	}
+	return dialect.JSONL(funcs)
+}
+
+// readExtensionTypes reads the types an extension defines.
+func readExtensionTypes(ctx context.Context, conn *pgx.Conn, extension string) ([]dialect.Type, error) {
+	rows, err := conn.Query(ctx, extensionTypes, extension)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var types []dialect.Type
+	for rows.Next() {
+		var t dialect.Type
+		if err := rows.Scan(&t.Name, &t.Category); err != nil {
+			return nil, err
+		}
+		types = append(types, t)
+	}
+	return types, rows.Err()
+}
+
+// writeRelations appends a schema's relations to out as JSONL, one relation
+// per line, in the form the seed package reads.
+func writeRelations(out *bytes.Buffer, schemaName string, relations []Relation) error {
+	enc := json.NewEncoder(out)
+	for _, relation := range relations {
+		rec := dialect.Relation{
+			Catalog: relation.Catalog,
+			Schema:  schemaName,
+			Name:    relation.Name,
+		}
+		for _, col := range relation.Columns {
+			c := dialect.Column{
+				Name:    col.Name,
+				Type:    col.Type,
+				NotNull: col.IsNotNull,
+				Array:   col.IsArray,
+			}
+			if col.Length != nil {
+				c.Length = *col.Length
+			}
+			rec.Columns = append(rec.Columns, c)
+		}
+		if err := enc.Encode(rec); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// preserveLegacyCatalogBehavior maintain previous ordering and filtering
+// that was manually done to the generated file pg_catalog.go.
+// Some of the test depend on this ordering - in particular, function lookups
+// where there might be multiple matching functions (due to type overloads)
+// Until sqlc supports "smarter" looking up of these functions,
+// preserveLegacyCatalogBehavior ensures there are no accidental test breakages
+func preserveLegacyCatalogBehavior(allProcs []Proc) []Proc {
+	// Preserve the legacy sort order of the end-to-end tests
+	sort.SliceStable(allProcs, func(i, j int) bool {
+		fnA := allProcs[i]
+		fnB := allProcs[j]
+
+		if fnA.Name == "lower" && fnB.Name == "lower" && len(fnA.ArgTypes) == 1 && fnA.ArgTypes[0] == "text" {
+			return true
+		}
+
+		if fnA.Name == "generate_series" && fnB.Name == "generate_series" && len(fnA.ArgTypes) == 2 && fnA.ArgTypes[0] == "numeric" {
+			return true
+		}
+
+		return false
+	})
+
+	procs := make([]Proc, 0, len(allProcs))
+	for _, p := range allProcs {
+		// Skip generating pg_catalog.concat to preserve legacy behavior
+		if p.Name == "concat" {
+			continue
+		}
+
+		procs = append(procs, p)
+	}
+
+	return procs
+}
+
+// Generate reads the dialect from the server. It creates every contrib
+// extension it describes, so the server's contrib package has to be
+// installed.
+func Generate(ctx context.Context, url string) (dialect.Files, error) {
+	conn, err := pgx.Connect(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close(ctx)
+	if err := checkVersion(ctx, conn); err != nil {
+		return nil, err
+	}
+
+	files := dialect.Files{}
+
+	// The two schemas sqlc knows PostgreSQL by. pg_catalog's functions are
+	// the dialect's standard library; the relations of both share one file,
+	// keyed by schema.
+	procs, err := readProcs(ctx, conn, "pg_catalog", extensions)
+	if err != nil {
+		return nil, err
+	}
+	if files[dialect.FunctionsFile], err = encodeFunctions(preserveLegacyCatalogBehavior(procs)); err != nil {
+		return nil, err
+	}
+	var relations bytes.Buffer
+	for _, schema := range []string{"pg_catalog", "information_schema"} {
+		rels, err := readRelations(ctx, conn, schema)
+		if err != nil {
+			return nil, err
+		}
+		if err := writeRelations(&relations, schema, rels); err != nil {
+			return nil, err
+		}
+	}
+	files[dialect.RelationsFile] = relations.Bytes()
+
+	// Each extension is a directory of its own under the dialect, holding the
+	// functions CREATE EXTENSION adds to the catalog.
+	for _, extension := range extensions {
+		if _, err := conn.Exec(ctx, fmt.Sprintf("CREATE EXTENSION IF NOT EXISTS %q", extension)); err != nil {
+			return nil, fmt.Errorf("error creating %s: %s", extension, err)
+		}
+
+		rows, err := conn.Query(ctx, extensionFuncs, extension)
+		if err != nil {
+			return nil, err
+		}
+		procs, err := scanProcs(rows)
+		if err != nil {
+			return nil, err
+		}
+		types, err := readExtensionTypes(ctx, conn, extension)
+		if err != nil {
+			return nil, err
+		}
+		if len(procs) == 0 && len(types) == 0 {
+			continue
+		}
+
+		// Preserve the legacy sort order of the end-to-end tests
+		sort.SliceStable(procs, func(i, j int) bool {
+			fnA := procs[i]
+			fnB := procs[j]
+
+			if extension == "pgcrypto" {
+				if fnA.Name == "digest" && fnB.Name == "digest" && len(fnA.ArgTypes) == 2 && fnA.ArgTypes[0] == "text" {
+					return true
+				}
+			}
+
+			return false
+		})
+
+		extensionDir := path.Join(dialect.ExtensionsDir, extension)
+		if len(types) > 0 {
+			if files[path.Join(extensionDir, dialect.TypesFile)], err = dialect.JSONL(types); err != nil {
+				return nil, fmt.Errorf("error generating extension %s: %w", extension, err)
+			}
+		}
+		if files[path.Join(extensionDir, dialect.FunctionsFile)], err = encodeFunctions(procs); err != nil {
+			return nil, fmt.Errorf("error generating extension %s: %w", extension, err)
+		}
+	}
+
+	return files, nil
+}
+
+// https://www.postgresql.org/docs/current/contrib.html
+var extensions = []string{
+	"adminpack",
+	"amcheck",
+	// "auth_delay",
+	// "auto_explain",
+	// "bloom",
+	"btree_gin",
+	"btree_gist",
+	"citext",
+	"cube",
+	"dblink",
+	// "dict_int",
+	// "dict_xsyn",
+	"earthdistance",
+	"file_fdw",
+	"fuzzystrmatch",
+	"hstore",
+	"intagg",
+	"intarray",
+	"isn",
+	"lo",
+	"ltree",
+	"pageinspect",
+	// "passwordcheck",
+	"pg_buffercache",
+	"pg_freespacemap",
+	"pg_prewarm",
+	"pg_stat_statements",
+	"pg_trgm",
+	"pg_visibility",
+	"pgcrypto",
+	"pgrowlocks",
+	"pgstattuple",
+	"postgres_fdw",
+	"seg",
+	// "sepgsql",
+	// "spi",
+	"sslinfo",
+	"tablefunc",
+	"tcn",
+	// "test_decoding",
+	// "tsm_system_rows",
+	// "tsm_system_time",
+	"unaccent",
+	"uuid-ossp",
+	"xml2",
+}

internal/goldeneye/postgresql/postgresql_test.go

@@ -0,0 +1,37 @@
+package postgresql
+
+import (
+	"context"
+	"testing"
+
+	"github.com/sqlc-dev/sqlc/internal/goldeneye/dialect"
+)
+
+// TestDialect verifies the committed PostgreSQL dialect against what the
+// server reports. It skips unless POSTGRESQL_SERVER_URI names a server.
+func TestDialect(t *testing.T) {
+	url, err := Locate()
+	if err != nil {
+		t.Skip(err)
+	}
+	ctx := context.Background()
+	version, err := Version(ctx, url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	files, err := Generate(ctx, url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dir, err := dialect.Dir(Engine)
+	if err != nil {
+		t.Fatal(err)
+	}
+	report, err := dialect.Check(dir, files)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report != "" {
+		t.Errorf("%s does not match what %s reports:\n%s", dir, version, report)
+	}
+}

internal/goldeneye/postgresql/proc.go

@@ -0,0 +1,163 @@
+package postgresql
+
+import (
+	"context"
+	"strings"
+
+	pgx "github.com/jackc/pgx/v5"
+)
+
+// https://stackoverflow.com/questions/25308765/postgresql-how-can-i-inspect-which-arguments-to-a-procedure-have-a-default-valu
+const catalogFuncs = `
+SELECT p.proname as name,
+  format_type(p.prorettype, NULL),
+  array(select format_type(unnest(p.proargtypes), NULL)),
+  p.proargnames,
+  p.proargnames[p.pronargs-p.pronargdefaults+1:p.pronargs],
+  p.proargmodes::text[]
+FROM pg_catalog.pg_proc p
+LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+WHERE n.nspname::text = $1
+  AND pg_function_is_visible(p.oid)
+  -- A function that one of the extensions the dialect describes separately
+  -- put in the schema, as adminpack puts its own in pg_catalog, belongs to
+  -- that extension's directory: leaving it out here keeps the output the
+  -- same whether or not the extension is installed.
+  AND NOT EXISTS (
+    SELECT 1
+    FROM pg_catalog.pg_depend d
+    INNER JOIN pg_catalog.pg_extension e ON e.oid = d.refobjid
+    WHERE d.classid = 'pg_catalog.pg_proc'::regclass
+      AND d.objid = p.oid
+      AND d.deptype = 'e'
+      AND e.extname = ANY($2))
+-- simply order all columns to keep subsequent runs stable
+ORDER BY 1, 2, 3, 4, 5;
+`
+
+type Proc struct {
+	Name       string
+	ReturnType string
+	ArgTypes   []string
+	ArgNames   []string
+	HasDefault []string
+	ArgModes   []string
+}
+
+func (p *Proc) ReturnTypeName() string {
+	return clean(p.ReturnType)
+}
+
+func (p *Proc) Args() []Arg {
+	var args []Arg
+	defaults := map[string]bool{}
+	for _, name := range p.HasDefault {
+		defaults[name] = true
+	}
+
+	for i, argType := range p.ArgTypes {
+		mode := "i"
+		name := ""
+		if i < len(p.ArgModes) {
+			mode = p.ArgModes[i]
+		}
+		if i < len(p.ArgNames) {
+			name = p.ArgNames[i]
+		}
+
+		args = append(args, Arg{
+			Name:       name,
+			Type:       argType,
+			Mode:       mode,
+			HasDefault: defaults[name],
+		})
+	}
+
+	// Some manual changes until https://github.com/sqlc-dev/sqlc/pull/1748
+	// can be completely implmented
+	if p.Name == "mode" {
+		return nil
+	}
+
+	if p.Name == "percentile_cont" && len(args) == 2 {
+		args = args[:1]
+	}
+
+	if p.Name == "percentile_disc" && len(args) == 2 {
+		args = args[:1]
+	}
+
+	return args
+}
+
+type Arg struct {
+	Name       string
+	Mode       string
+	Type       string
+	HasDefault bool
+}
+
+func (a *Arg) TypeName() string {
+	return clean(a.Type)
+}
+
+func scanProcs(rows pgx.Rows) ([]Proc, error) {
+	defer rows.Close()
+	// Iterate through the result set
+	var procs []Proc
+	for rows.Next() {
+		var p Proc
+		err := rows.Scan(
+			&p.Name,
+			&p.ReturnType,
+			&p.ArgTypes,
+			&p.ArgNames,
+			&p.HasDefault,
+			&p.ArgModes,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		// TODO: Filter these out in SQL
+		if strings.HasPrefix(p.ReturnType, "SETOF") {
+			continue
+		}
+
+		// The internal pseudo-type is used to declare functions that are meant
+		// only to be called internally by the database system, and not by
+		// direct invocation in an SQL query. If a function has at least one
+		// internal-type argument then it cannot be called from SQL. To
+		// preserve the type safety of this restriction it is important to
+		// follow this coding rule: do not create any function that is declared
+		// to return internal unless it has at least one internal argument
+		//
+		// https://www.postgresql.org/docs/current/datatype-pseudo.html
+		var skip bool
+		for i := range p.ArgTypes {
+			if p.ArgTypes[i] == "internal" {
+				skip = true
+			}
+		}
+		if skip {
+			continue
+		}
+		if p.ReturnType == "internal" {
+			continue
+		}
+
+		procs = append(procs, p)
+	}
+	return procs, rows.Err()
+}
+
+// readProcs reads the functions of a schema, leaving out the ones that
+// belong to any of the named extensions.
+func readProcs(ctx context.Context, conn *pgx.Conn, schemaName string, extensions []string) ([]Proc, error) {
+	rows, err := conn.Query(ctx, catalogFuncs, schemaName, extensions)
+	if err != nil {
+		return nil, err
+	}
+
+	return scanProcs(rows)
+}

internal/goldeneye/postgresql/relation.go

@@ -0,0 +1,118 @@
+package postgresql
+
+import (
+	"context"
+
+	pgx "github.com/jackc/pgx/v5"
+)
+
+// Relations are the relations available in pg_tables and pg_views
+// such as pg_catalog.pg_timezone_names
+const relationQuery = `
+with relations as (
+	select schemaname, tablename as name from pg_catalog.pg_tables
+	UNION ALL
+	select schemaname, viewname as name from pg_catalog.pg_views
+)
+select
+	relations.schemaname,
+	relations.name as tablename,
+	pg_attribute.attname as column_name,
+	attnotnull as column_notnull,
+	-- An array column's type is its element's, with the array flag set,
+	-- rather than pg_type's own _text spelling of the array type. Only the
+	-- _-prefixed array types count: int2vector and oidvector are in the
+	-- array category and carry an element, but are types of their own.
+	coalesce(element_type.typname, column_type.typname) as column_type,
+	nullif(coalesce(element_type.typlen, column_type.typlen), -1) as column_length,
+	element_type.oid is not null as column_isarray
+from relations
+inner join pg_catalog.pg_class on pg_class.relname = relations.name
+left join pg_catalog.pg_attribute on pg_attribute.attrelid = pg_class.oid
+inner join pg_catalog.pg_type column_type on pg_attribute.atttypid = column_type.oid
+left join pg_catalog.pg_type element_type
+	on column_type.typcategory = 'A' and column_type.typname like '\_%'
+	and element_type.oid = column_type.typelem
+where relations.schemaname = $1
+-- Make sure these columns are always generated in the same order
+-- so that the output is stable
+order by
+ relations.schemaname ASC,
+ relations.name ASC,
+ pg_attribute.attnum ASC
+`
+
+type Relation struct {
+	Catalog    string
+	SchemaName string
+	Name       string
+	Columns    []RelationColumn
+}
+
+type RelationColumn struct {
+	Name      string
+	Type      string
+	IsNotNull bool
+	IsArray   bool
+	Length    *int
+}
+
+func scanRelations(rows pgx.Rows) ([]Relation, error) {
+	defer rows.Close()
+	// Iterate through the result set
+	var relations []Relation
+	var prevRel *Relation
+
+	for rows.Next() {
+		var schemaName string
+		var tableName string
+		var columnName string
+		var columnNotNull bool
+		var columnType string
+		var columnLength *int
+		var columnIsArray bool
+		err := rows.Scan(
+			&schemaName,
+			&tableName,
+			&columnName,
+			&columnNotNull,
+			&columnType,
+			&columnLength,
+			&columnIsArray,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		if prevRel == nil || tableName != prevRel.Name {
+			// We are on the same table, just keep adding columns
+			r := Relation{
+				Catalog:    "pg_catalog",
+				SchemaName: schemaName,
+				Name:       tableName,
+			}
+
+			relations = append(relations, r)
+			prevRel = &relations[len(relations)-1]
+		}
+
+		prevRel.Columns = append(prevRel.Columns, RelationColumn{
+			Name:      columnName,
+			Type:      columnType,
+			IsNotNull: columnNotNull,
+			IsArray:   columnIsArray,
+			Length:    columnLength,
+		})
+	}
+
+	return relations, rows.Err()
+}
+
+func readRelations(ctx context.Context, conn *pgx.Conn, schemaName string) ([]Relation, error) {
+	rows, err := conn.Query(ctx, relationQuery, schemaName)
+	if err != nil {
+		return nil, err
+	}
+
+	return scanRelations(rows)
+}