snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/postgres/README.txt

@@ -0,0 +1,17 @@
+
+1. create "seaweedfs" database
+
+export PGHOME=/Library/PostgreSQL/10
+$PGHOME/bin/createdb --username=postgres --password seaweedfs
+
+2. create "filemeta" table
+$PGHOME/bin/psql --username=postgres --password seaweedfs
+
+CREATE TABLE IF NOT EXISTS filemeta (
+  dirhash     BIGINT,
+  name        VARCHAR(65535),
+  directory   VARCHAR(65535),
+  meta        bytea,
+  PRIMARY KEY (dirhash, name)
+);
+

weed/filer/postgres/pgx_conn.go

@@ -0,0 +1,14 @@
+package postgres
+
+import (
+	"database/sql"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/pgxutil"
+)
+
+// OpenPGXDB is a thin alias kept for callers (notably the postgres2 filer
+// store) that already depend on this entry point. New code should use
+// pgxutil.OpenDB directly.
+func OpenPGXDB(sqlUrl, adaptedSqlUrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (*sql.DB, error) {
+	return pgxutil.OpenDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
+}

weed/filer/postgres/postgres_collation.go

@@ -0,0 +1,69 @@
+package postgres
+
+import (
+	"database/sql"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// ConfigureListOrdering switches the list queries to COLLATE "C" when the live
+// filemeta.name collation is locale-aware, so S3 ListObjectsV2 stays
+// lexicographic. The fallback costs a sort, so warn the operator to fix it.
+func ConfigureListOrdering(db *sql.DB, gen *SqlGenPostgres) {
+	if isCockroachDB(db) {
+		// CockroachDB string comparison is already byte-ordered, so the
+		// fallback is redundant; older versions also reject COLLATE "C".
+		return
+	}
+	collation, isBinary, err := nameColumnCollation(db)
+	if err != nil {
+		glog.V(1).Infof("postgres: skip filemeta.name collation check: %v", err)
+		return
+	}
+	if isBinary {
+		return
+	}
+	gen.ForceBinaryCollation = true
+	glog.Warningf(`postgres: filemeta.name collation %q is not byte-ordered, so S3 list order is not byte-lexicographic and clients that merge sorted listings may report spurious diffs. Falling back to a slower COLLATE "C" sort; declare the name column COLLATE "C" (or use a C/C.UTF-8 database collation) for correct, indexed ordering.`, collation)
+}
+
+// isCockroachDB reports whether db is a CockroachDB backend, whose version()
+// string carries "CockroachDB". On any error we assume real Postgres.
+func isCockroachDB(db *sql.DB) bool {
+	var version string
+	if err := db.QueryRow("SELECT version()").Scan(&version); err != nil {
+		glog.V(1).Infof("postgres: skip CockroachDB detection: %v", err)
+		return false
+	}
+	return strings.Contains(version, "CockroachDB")
+}
+
+func nameColumnCollation(db *sql.DB) (collation string, isBinary bool, err error) {
+	// information_schema reports NULL for a column on the database default, so
+	// fall back to datcollate for the effective ordering.
+	row := db.QueryRow(`SELECT
+		(SELECT collation_name FROM information_schema.columns
+			WHERE table_schema = current_schema() AND table_name = $1 AND column_name = 'name'),
+		(SELECT datcollate FROM pg_database WHERE datname = current_database())`, abstract_sql.DEFAULT_TABLE)
+	var columnCollation, dbCollate sql.NullString
+	if err = row.Scan(&columnCollation, &dbCollate); err != nil {
+		return "", false, err
+	}
+	effective := columnCollation.String
+	if !columnCollation.Valid || effective == "" {
+		effective = dbCollate.String
+	}
+	return effective, isByteOrderedCollation(effective), nil
+}
+
+// isByteOrderedCollation reports whether a Postgres locale sorts by byte value
+// (C, POSIX, C.UTF-8) rather than locale order (en_US.UTF-8, ICU, ...).
+func isByteOrderedCollation(name string) bool {
+	switch strings.ToLower(strings.TrimSpace(name)) {
+	case "c", "posix", "c.utf-8", "c.utf8":
+		return true
+	}
+	return false
+}

weed/filer/postgres/postgres_collation_test.go

@@ -0,0 +1,49 @@
+package postgres
+
+import (
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+)
+
+func TestConfigureListOrderingSkipsCockroachDB(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock: %v", err)
+	}
+	defer db.Close()
+
+	mock.ExpectQuery("SELECT version\\(\\)").WillReturnRows(
+		sqlmock.NewRows([]string{"version"}).AddRow("CockroachDB CCL v22.1.22 (x86_64-pc-linux-gnu)"))
+
+	gen := &SqlGenPostgres{}
+	ConfigureListOrdering(db, gen)
+	if gen.ForceBinaryCollation {
+		t.Fatal("CockroachDB default ordering is byte-ordered; COLLATE \"C\" must not be forced")
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet expectations: %v", err)
+	}
+}
+
+func TestConfigureListOrderingForcesBinaryOnLocalePostgres(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock: %v", err)
+	}
+	defer db.Close()
+
+	mock.ExpectQuery("SELECT version\\(\\)").WillReturnRows(
+		sqlmock.NewRows([]string{"version"}).AddRow("PostgreSQL 16.2 on x86_64-pc-linux-gnu"))
+	mock.ExpectQuery("datcollate").WillReturnRows(
+		sqlmock.NewRows([]string{"collation_name", "datcollate"}).AddRow(nil, "en_US.UTF-8"))
+
+	gen := &SqlGenPostgres{}
+	ConfigureListOrdering(db, gen)
+	if !gen.ForceBinaryCollation {
+		t.Fatal("locale-aware Postgres collation must force byte ordering")
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("unmet expectations: %v", err)
+	}
+}

weed/filer/postgres/postgres_sql_gen.go

@@ -0,0 +1,97 @@
+package postgres
+
+import (
+	"fmt"
+
+	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+)
+
+type SqlGenPostgres struct {
+	CreateTableSqlTemplate string
+	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
+	// Force byte ordering on a locale-aware name column; see ConfigureListOrdering.
+	ForceBinaryCollation bool
+}
+
+// DefaultUpsertQuery keeps INSERTs idempotent so a duplicate-key failure
+// (23505) cannot poison the surrounding transaction (25P02). Used when the
+// user enables upsert but does not provide their own template.
+const DefaultUpsertQuery = `INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash, name) DO UPDATE SET directory=EXCLUDED.directory, meta=EXCLUDED.meta`
+
+// DefaultCreateTableQuery is used when createTable is left unset; an empty
+// template would otherwise render as %!(EXTRA ...) garbage SQL.
+const DefaultCreateTableQuery = `CREATE TABLE IF NOT EXISTS "%s" (dirhash BIGINT, name VARCHAR(65535), directory VARCHAR(65535), meta bytea, PRIMARY KEY (dirhash, name))`
+
+var (
+	_ = abstract_sql.SqlGenerator(&SqlGenPostgres{})
+)
+
+// ResolveCreateTableQuery normalizes the createTable config value. A boolean
+// true (read by viper as the string "true") selects the default template.
+// An empty or false value returns an empty string so the caller can skip
+// table creation. Any other value is treated as a custom SQL template.
+func ResolveCreateTableQuery(createTable string) string {
+	switch createTable {
+	case "true":
+		return DefaultCreateTableQuery
+	case "false", "":
+		return ""
+	default:
+		return createTable
+	}
+}
+
+func (gen *SqlGenPostgres) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf(`INSERT INTO "%s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)`, tableName)
+	}
+}
+
+func (gen *SqlGenPostgres) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf(`UPDATE "%s" SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4`, tableName)
+}
+
+func (gen *SqlGenPostgres) GetSqlFind(tableName string) string {
+	return fmt.Sprintf(`SELECT meta FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
+}
+
+func (gen *SqlGenPostgres) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND name=$2 AND directory=$3`, tableName)
+}
+
+func (gen *SqlGenPostgres) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf(`DELETE FROM "%s" WHERE dirhash=$1 AND directory=$2`, tableName)
+}
+
+// nameExpr forces byte ordering on a locale-aware column via COLLATE "C".
+func (gen *SqlGenPostgres) nameExpr() string {
+	if gen.ForceBinaryCollation {
+		return `name COLLATE "C"`
+	}
+	return "name"
+}
+
+func (gen *SqlGenPostgres) GetSqlListExclusive(tableName string) string {
+	name := gen.nameExpr()
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
+}
+
+func (gen *SqlGenPostgres) GetSqlListInclusive(tableName string) string {
+	name := gen.nameExpr()
+	return fmt.Sprintf(`SELECT NAME, meta FROM "%s" WHERE dirhash=$1 AND %s>=$2 AND directory=$3 AND %s like $4 ORDER BY %s ASC LIMIT $5`, tableName, name, name, name)
+}
+
+func (gen *SqlGenPostgres) GetSqlCreateTable(tableName string) string {
+	if gen.CreateTableSqlTemplate == "" {
+		return ""
+	}
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
+}
+
+func (gen *SqlGenPostgres) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
+}

weed/filer/postgres/postgres_sql_gen_test.go

@@ -0,0 +1,94 @@
+package postgres
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDefaultUpsertQueryIsConflictSafe(t *testing.T) {
+	gen := &SqlGenPostgres{UpsertQueryTemplate: DefaultUpsertQuery}
+	got := gen.GetSqlInsert("filemeta")
+	if !strings.Contains(got, "ON CONFLICT") {
+		t.Fatalf("expected ON CONFLICT in default upsert, got: %s", got)
+	}
+	if !strings.Contains(got, `"filemeta"`) {
+		t.Fatalf("expected quoted table name, got: %s", got)
+	}
+}
+
+func TestDefaultCreateTableQueryRendersValidSql(t *testing.T) {
+	gen := &SqlGenPostgres{CreateTableSqlTemplate: DefaultCreateTableQuery}
+	got := gen.GetSqlCreateTable("filemeta")
+	if strings.Contains(got, "%!") {
+		t.Fatalf("template rendered to malformed SQL, got: %s", got)
+	}
+	if !strings.Contains(got, `"filemeta"`) {
+		t.Fatalf("expected quoted table name, got: %s", got)
+	}
+}
+
+func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
+	gen := &SqlGenPostgres{}
+	got := gen.GetSqlInsert("filemeta")
+	if strings.Contains(got, "ON CONFLICT") {
+		t.Fatalf("plain INSERT path should not contain ON CONFLICT, got: %s", got)
+	}
+}
+
+func TestListSqlDefaultOrderingFollowsColumn(t *testing.T) {
+	gen := &SqlGenPostgres{}
+	for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+		if strings.Contains(got, "COLLATE") {
+			t.Fatalf("default list query should not force a collation, got: %s", got)
+		}
+		if !strings.Contains(got, "ORDER BY name ASC") {
+			t.Fatalf("expected plain name ordering, got: %s", got)
+		}
+	}
+}
+
+func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
+	gen := &SqlGenPostgres{ForceBinaryCollation: true}
+	for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+		if !strings.Contains(got, `ORDER BY name COLLATE "C" ASC`) {
+			t.Fatalf(`expected COLLATE "C" ordering, got: %s`, got)
+		}
+		if !strings.Contains(got, `name COLLATE "C" like $4`) {
+			t.Fatalf(`expected COLLATE "C" prefix filter, got: %s`, got)
+		}
+		if strings.Contains(got, "AND name>$2") || strings.Contains(got, "AND name>=$2") {
+			t.Fatalf("pagination comparison must also be byte-ordered, got: %s", got)
+		}
+	}
+}
+
+func TestIsByteOrderedCollation(t *testing.T) {
+	byteOrdered := []string{"C", "POSIX", "c", "C.UTF-8", "C.utf8"}
+	for _, c := range byteOrdered {
+		if !isByteOrderedCollation(c) {
+			t.Fatalf("expected %q to be treated as byte-ordered", c)
+		}
+	}
+	locale := []string{"en_US.UTF-8", "en_US.utf8", "en-US-x-icu", "und-x-icu", ""}
+	for _, c := range locale {
+		if isByteOrderedCollation(c) {
+			t.Fatalf("expected %q to be treated as locale-aware", c)
+		}
+	}
+}
+
+func TestResolveCreateTableQuery(t *testing.T) {
+	cases := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"true", DefaultCreateTableQuery},
+		{"false", ""},
+		{"CREATE TABLE custom", "CREATE TABLE custom"},
+	}
+	for _, c := range cases {
+		if got := ResolveCreateTableQuery(c.in); got != c.want {
+			t.Fatalf("ResolveCreateTableQuery(%q) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}

weed/filer/postgres/postgres_store.go

@@ -0,0 +1,142 @@
+// Package postgres provides PostgreSQL filer store implementation
+// Migrated from github.com/lib/pq to github.com/jackc/pgx for:
+// - Active development and support
+// - Better performance and PostgreSQL-specific features
+// - Improved error handling (no more panics)
+// - Built-in logging capabilities
+// - Superior SSL certificate support
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"strconv"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &PostgresStore{})
+}
+
+type PostgresStore struct {
+	abstract_sql.AbstractSqlStore
+}
+
+func (store *PostgresStore) GetName() string {
+	return "postgres"
+}
+
+func (store *PostgresStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	// Fewer idle slots than concurrent operations means a fresh connection per
+	// operation, until the filer runs out of ephemeral ports. connection_max_open
+	// stays unset: a listing runs a second query from its own callback, so a
+	// bounded pool deadlocks once the concurrency reaches it.
+	configuration.SetDefault(prefix+"connection_max_idle", 50)
+	configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300)
+	// Default on so minimal configs are not exposed to duplicate-key tx
+	// poisoning on Postgres; an explicit false still disables it.
+	configuration.SetDefault(prefix+"enableUpsert", true)
+	return store.initialize(
+		configuration.GetString(prefix+"createTable"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"hostname"),
+		configuration.GetInt(prefix+"port"),
+		configuration.GetString(prefix+"database"),
+		configuration.GetString(prefix+"schema"),
+		configuration.GetString(prefix+"sslmode"),
+		configuration.GetString(prefix+"sslcert"),
+		configuration.GetString(prefix+"sslkey"),
+		configuration.GetString(prefix+"sslrootcert"),
+		configuration.GetString(prefix+"sslcrl"),
+		configuration.GetBool(prefix+"pgbouncer_compatible"),
+		configuration.GetInt(prefix+"connection_max_idle"),
+		configuration.GetInt(prefix+"connection_max_open"),
+		configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
+	)
+}
+
+func (store *PostgresStore) initialize(createTable, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database, schema, sslmode, sslcert, sslkey, sslrootcert, sslcrl string, pgbouncerCompatible bool, maxIdle, maxOpen, maxLifetimeSeconds int) (err error) {
+
+	store.SupportBucketTable = false
+	createTable = ResolveCreateTableQuery(createTable)
+	if !enableUpsert {
+		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = DefaultUpsertQuery
+	}
+	gen := &SqlGenPostgres{
+		CreateTableSqlTemplate: createTable,
+		DropTableSqlTemplate:   `drop table if exists "%s"`,
+		UpsertQueryTemplate:    upsertQuery,
+	}
+	store.SqlGenerator = gen
+
+	// pgx-optimized connection string with better timeouts and connection handling
+	sqlUrl := "connect_timeout=30"
+
+	if hostname != "" {
+		sqlUrl += " host=" + hostname
+	}
+	if port != 0 {
+		sqlUrl += " port=" + strconv.Itoa(port)
+	}
+
+	// SSL configuration - pgx provides better SSL support than lib/pq
+	if sslmode != "" {
+		sqlUrl += " sslmode=" + sslmode
+	}
+	if sslcert != "" {
+		sqlUrl += " sslcert=" + sslcert
+	}
+	if sslkey != "" {
+		sqlUrl += " sslkey=" + sslkey
+	}
+	if sslrootcert != "" {
+		sqlUrl += " sslrootcert=" + sslrootcert
+	}
+	if sslcrl != "" {
+		sqlUrl += " sslcrl=" + sslcrl
+	}
+	if user != "" {
+		sqlUrl += " user=" + user
+	}
+	adaptedSqlUrl := sqlUrl
+	if password != "" {
+		sqlUrl += " password=" + password
+		adaptedSqlUrl += " password=ADAPTED"
+	}
+	if database != "" {
+		sqlUrl += " dbname=" + database
+		adaptedSqlUrl += " dbname=" + database
+	}
+	if schema != "" && !pgbouncerCompatible {
+		sqlUrl += " search_path=" + schema
+		adaptedSqlUrl += " search_path=" + schema
+	}
+	db, openErr := OpenPGXDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
+	if openErr != nil {
+		return openErr
+	}
+	if err = store.UseConnectionPools(db, func() (*sql.DB, error) {
+		return OpenPGXDB(sqlUrl, adaptedSqlUrl, pgbouncerCompatible, maxIdle, maxOpen, maxLifetimeSeconds)
+	}, maxIdle, maxOpen, maxLifetimeSeconds); err != nil {
+		return err
+	}
+
+	if createTable != "" {
+		if _, err = store.DB.ExecContext(context.Background(), gen.GetSqlCreateTable(abstract_sql.DEFAULT_TABLE)); err != nil {
+			return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
+		}
+	}
+
+	ConfigureListOrdering(store.DB, gen)
+
+	return nil
+}