snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/mysql/mysql_collation.go

@@ -0,0 +1,41 @@
+package mysql
+
+import (
+	"database/sql"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// ConfigureListOrdering switches the list queries to BINARY byte ordering when
+// the live filemeta.name collation is not binary, so S3 ListObjectsV2 stays
+// lexicographic. The fallback costs a filesort, so warn the operator to fix it.
+func ConfigureListOrdering(db *sql.DB, gen *SqlGenMysql) {
+	collation, isBinary, err := nameColumnCollation(db)
+	if err != nil {
+		glog.V(1).Infof("mysql: skip filemeta.name collation check: %v", err)
+		return
+	}
+	if isBinary {
+		return
+	}
+	gen.ForceBinaryCollation = true
+	glog.Warningf("mysql: filemeta.name collation %q is not binary, so S3 list order is not byte-lexicographic and clients that merge sorted listings may report spurious diffs. Falling back to a slower BINARY sort; convert the name column to a *_bin collation (e.g. utf8mb4_bin) for correct, indexed ordering.", collation)
+}
+
+func nameColumnCollation(db *sql.DB) (collation string, isBinary bool, err error) {
+	row := db.QueryRow("SELECT COLLATION_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = 'name'", abstract_sql.DEFAULT_TABLE)
+	var c sql.NullString
+	if err = row.Scan(&c); err != nil {
+		return "", false, err
+	}
+	return c.String, isBinaryCollation(c.String), nil
+}
+
+// isBinaryCollation reports whether a collation orders by byte value: a NULL
+// collation (BINARY column) or any *_bin collation.
+func isBinaryCollation(collation string) bool {
+	c := strings.ToLower(strings.TrimSpace(collation))
+	return c == "" || c == "binary" || strings.HasSuffix(c, "_bin")
+}

weed/filer/mysql/mysql_sql_gen.go

@@ -0,0 +1,84 @@
+package mysql
+
+import (
+	"fmt"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+)
+
+type SqlGenMysql struct {
+	CreateTableSqlTemplate string
+	DropTableSqlTemplate   string
+	UpsertQueryTemplate    string
+	// Force byte ordering on a non-binary name column; see ConfigureListOrdering.
+	ForceBinaryCollation bool
+}
+
+// DefaultUpsertQuery keeps INSERTs idempotent so the inode-index KvPut
+// after every entry write does not emit a duplicate-key roundtrip on
+// every mutation. The VALUES() form works on MariaDB and MySQL >=5.7;
+// the newer "AS new" alias from MySQL 8.0.19 errors on MariaDB, so it
+// is left for users to opt into via an explicit upsertQuery.
+const DefaultUpsertQuery = "INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE `meta` = VALUES(`meta`)"
+
+// DefaultCreateTableQuery is used when createTable is left unset; an empty
+// template would otherwise render as %!(EXTRA ...) garbage SQL.
+const DefaultCreateTableQuery = "CREATE TABLE IF NOT EXISTS `%s` (`dirhash` BIGINT NOT NULL, `name` VARCHAR(766) NOT NULL, `directory` TEXT NOT NULL, `meta` LONGBLOB, PRIMARY KEY (`dirhash`, `name`)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin"
+
+var (
+	_ = abstract_sql.SqlGenerator(&SqlGenMysql{})
+)
+
+func (gen *SqlGenMysql) GetSqlInsert(tableName string) string {
+	if gen.UpsertQueryTemplate != "" {
+		return fmt.Sprintf(gen.UpsertQueryTemplate, tableName)
+	} else {
+		return fmt.Sprintf("INSERT INTO `%s` (`dirhash`,`name`,`directory`,`meta`) VALUES(?,?,?,?)", tableName)
+	}
+}
+
+func (gen *SqlGenMysql) GetSqlUpdate(tableName string) string {
+	return fmt.Sprintf("UPDATE `%s` SET `meta` = ? WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+}
+
+func (gen *SqlGenMysql) GetSqlFind(tableName string) string {
+	return fmt.Sprintf("SELECT `meta` FROM `%s` WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+}
+
+func (gen *SqlGenMysql) GetSqlDelete(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE `dirhash` = ? AND `name` = ? AND `directory` = ?", tableName)
+}
+
+func (gen *SqlGenMysql) GetSqlDeleteFolderChildren(tableName string) string {
+	return fmt.Sprintf("DELETE FROM `%s` WHERE `dirhash` = ? AND `directory` = ?", tableName)
+}
+
+// nameExpr forces byte ordering on a non-binary column at the cost of a filesort.
+func (gen *SqlGenMysql) nameExpr() string {
+	if gen.ForceBinaryCollation {
+		return "BINARY `name`"
+	}
+	return "`name`"
+}
+
+func (gen *SqlGenMysql) GetSqlListExclusive(tableName string) string {
+	name := gen.nameExpr()
+	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s > ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
+}
+
+func (gen *SqlGenMysql) GetSqlListInclusive(tableName string) string {
+	name := gen.nameExpr()
+	return fmt.Sprintf("SELECT `name`, `meta` FROM `%s` WHERE `dirhash` = ? AND %s >= ? AND `directory` = ? AND %s LIKE ? ORDER BY %s ASC LIMIT ?", tableName, name, name, name)
+}
+
+func (gen *SqlGenMysql) GetSqlCreateTable(tableName string) string {
+	if gen.CreateTableSqlTemplate == "" {
+		return ""
+	}
+	return fmt.Sprintf(gen.CreateTableSqlTemplate, tableName)
+}
+
+func (gen *SqlGenMysql) GetSqlDropTable(tableName string) string {
+	return fmt.Sprintf(gen.DropTableSqlTemplate, tableName)
+}

weed/filer/mysql/mysql_sql_gen_test.go

@@ -0,0 +1,81 @@
+package mysql
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDefaultUpsertQueryUsesOnDuplicateKey(t *testing.T) {
+	gen := &SqlGenMysql{UpsertQueryTemplate: DefaultUpsertQuery}
+	got := gen.GetSqlInsert("filemeta")
+	if !strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
+		t.Fatalf("expected ON DUPLICATE KEY UPDATE in default upsert, got: %s", got)
+	}
+	if strings.Contains(got, "AS `new`") {
+		t.Fatalf("default should avoid MySQL 8.0.19 row-alias syntax for MariaDB compat, got: %s", got)
+	}
+	if !strings.Contains(got, "`filemeta`") {
+		t.Fatalf("expected backticked table name, got: %s", got)
+	}
+}
+
+func TestDefaultCreateTableQueryRendersValidSql(t *testing.T) {
+	gen := &SqlGenMysql{CreateTableSqlTemplate: DefaultCreateTableQuery}
+	got := gen.GetSqlCreateTable("filemeta")
+	if strings.Contains(got, "%!") {
+		t.Fatalf("template rendered to malformed SQL, got: %s", got)
+	}
+	if !strings.Contains(got, "`filemeta`") {
+		t.Fatalf("expected backticked table name, got: %s", got)
+	}
+}
+
+func TestEmptyUpsertTemplateFallsBackToPlainInsert(t *testing.T) {
+	gen := &SqlGenMysql{}
+	got := gen.GetSqlInsert("filemeta")
+	if strings.Contains(got, "ON DUPLICATE KEY UPDATE") {
+		t.Fatalf("plain INSERT path should not contain ON DUPLICATE KEY UPDATE, got: %s", got)
+	}
+}
+
+func TestListSqlDefaultOrderingFollowsColumn(t *testing.T) {
+	gen := &SqlGenMysql{}
+	for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+		if strings.Contains(got, "BINARY") {
+			t.Fatalf("default list query should not force BINARY, got: %s", got)
+		}
+		if !strings.Contains(got, "ORDER BY `name` ASC") {
+			t.Fatalf("expected plain name ordering, got: %s", got)
+		}
+	}
+}
+
+func TestListSqlBinaryOrderingOnNonBinaryColumn(t *testing.T) {
+	gen := &SqlGenMysql{ForceBinaryCollation: true}
+	for _, got := range []string{gen.GetSqlListExclusive("filemeta"), gen.GetSqlListInclusive("filemeta")} {
+		if !strings.Contains(got, "ORDER BY BINARY `name` ASC") {
+			t.Fatalf("expected BINARY ordering, got: %s", got)
+		}
+		if !strings.Contains(got, "BINARY `name` LIKE ?") {
+			t.Fatalf("expected BINARY prefix filter, got: %s", got)
+		}
+		if strings.Contains(got, "AND `name` > ?") || strings.Contains(got, "AND `name` >= ?") {
+			t.Fatalf("pagination comparison must also be BINARY, got: %s", got)
+		}
+	}
+}
+
+func TestIsBinaryCollation(t *testing.T) {
+	binary := []string{"", "binary", "utf8mb4_bin", "utf8mb3_bin", "latin1_bin", "UTF8MB4_BIN"}
+	for _, c := range binary {
+		if !isBinaryCollation(c) {
+			t.Fatalf("expected %q to be treated as binary", c)
+		}
+	}
+	ci := []string{"utf8mb4_general_ci", "utf8mb3_general_ci", "utf8mb4_0900_ai_ci", "latin1_swedish_ci"}
+	for _, c := range ci {
+		if isBinaryCollation(c) {
+			t.Fatalf("expected %q to be treated as non-binary", c)
+		}
+	}
+}

weed/filer/mysql/mysql_store.go

@@ -0,0 +1,173 @@
+package mysql
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"database/sql"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?collation=utf8mb4_bin"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &MysqlStore{})
+}
+
+type MysqlStore struct {
+	abstract_sql.AbstractSqlStore
+}
+
+func (store *MysqlStore) GetName() string {
+	return "mysql"
+}
+
+func (store *MysqlStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	// Fewer idle slots than concurrent operations means a fresh connection per
+	// operation, until the filer runs out of ephemeral ports. connection_max_open
+	// stays unset: a listing runs a second query from its own callback, so a
+	// bounded pool deadlocks once the concurrency reaches it.
+	configuration.SetDefault(prefix+"connection_max_idle", 50)
+	configuration.SetDefault(prefix+"connection_max_lifetime_seconds", 300)
+	// Default on so minimal configs avoid the duplicate-key roundtrip the
+	// inode-index KvPut would otherwise emit on every write.
+	configuration.SetDefault(prefix+"enableUpsert", true)
+	return store.initialize(
+		configuration.GetString(prefix+"dsn"),
+		configuration.GetString(prefix+"upsertQuery"),
+		configuration.GetBool(prefix+"enableUpsert"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"hostname"),
+		configuration.GetInt(prefix+"port"),
+		configuration.GetString(prefix+"database"),
+		configuration.GetInt(prefix+"connection_max_idle"),
+		configuration.GetInt(prefix+"connection_max_open"),
+		configuration.GetInt(prefix+"connection_max_lifetime_seconds"),
+		configuration.GetBool(prefix+"interpolateParams"),
+		configuration.GetBool(prefix+"enable_tls"),
+		configuration.GetString(prefix+"ca_crt"),
+		configuration.GetString(prefix+"client_crt"),
+		configuration.GetString(prefix+"client_key"),
+		configuration.GetBool(prefix+"tls_insecure_skip_verify"),
+		configuration.GetString(prefix+"tls_server_name"),
+	)
+}
+
+func (store *MysqlStore) initialize(dsn string, upsertQuery string, enableUpsert bool, user, password, hostname string, port int, database string, maxIdle, maxOpen,
+	maxLifetimeSeconds int, interpolateParams bool, enableTls bool, caCrtDir string, clientCrtDir string, clientKeyDir string,
+	tlsInsecureSkipVerify bool, tlsServerName string) (err error) {
+
+	store.SupportBucketTable = false
+	if !enableUpsert {
+		upsertQuery = ""
+	} else if upsertQuery == "" {
+		upsertQuery = DefaultUpsertQuery
+	}
+	gen := &SqlGenMysql{
+		CreateTableSqlTemplate: "",
+		DropTableSqlTemplate:   "DROP TABLE IF EXISTS `%s`",
+		UpsertQueryTemplate:    upsertQuery,
+	}
+	store.SqlGenerator = gen
+
+	store.RetryableErrorCallback = func(err error) bool {
+		var mysqlError *mysql.MySQLError
+		if errors.As(err, &mysqlError) {
+			if mysqlError.Number == 1213 { // ER_LOCK_DEADLOCK
+				return true
+			}
+			if mysqlError.Number == 1205 { // ER_LOCK_WAIT_TIMEOUT
+				return true
+			}
+		}
+		return false
+	}
+
+	if dsn == "" {
+		dsn = fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database)
+		if interpolateParams {
+			dsn += "&interpolateParams=true"
+		}
+	}
+	cfg, err := mysql.ParseDSN(dsn)
+	if err != nil {
+		return fmt.Errorf("can not parse DSN error:%w", err)
+	}
+
+	if enableTls {
+		tlsConfig := &tls.Config{
+			MinVersion:         tls.VersionTLS12,
+			InsecureSkipVerify: tlsInsecureSkipVerify,
+			ServerName:         tlsServerName,
+		}
+
+		// When ca_crt is empty, leave RootCAs nil so Go falls back to the
+		// system trust store. This is the common case for managed databases
+		// (RDS, Aiven, ...) whose certs chain to a public CA already on the host.
+		if caCrtDir != "" {
+			rootCertPool := x509.NewCertPool()
+			pem, err := os.ReadFile(caCrtDir)
+			if err != nil {
+				return fmt.Errorf("read ca_crt %s: %w", caCrtDir, err)
+			}
+			if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
+				return fmt.Errorf("failed to append root certificate from %s", caCrtDir)
+			}
+			tlsConfig.RootCAs = rootCertPool
+		}
+
+		// Only attempt to load a client keypair when at least one of the paths is
+		// set. If either is set, both must load successfully — silently skipping
+		// a typo'd path used to mask broken mTLS setups as confusing handshake
+		// failures.
+		if clientCrtDir != "" || clientKeyDir != "" {
+			cert, err := tls.LoadX509KeyPair(clientCrtDir, clientKeyDir)
+			if err != nil {
+				return fmt.Errorf("load mysql client keypair (crt=%s key=%s): %w", clientCrtDir, clientKeyDir, err)
+			}
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+
+		// Set TLS directly on the parsed Config rather than registering a global
+		// "mysql-tls" entry — the global registry is process-wide and would be
+		// overwritten if a second MysqlStore is initialized with different TLS
+		// settings.
+		cfg.TLS = tlsConfig
+	}
+
+	connector, err := mysql.NewConnector(cfg)
+	if err != nil {
+		return fmt.Errorf("can not create mysql connector for %s error:%w", maskedDSN(cfg), err)
+	}
+
+	if err = store.UseConnectionPools(sql.OpenDB(connector), func() (*sql.DB, error) {
+		return sql.OpenDB(connector), nil
+	}, maxIdle, maxOpen, maxLifetimeSeconds); err != nil {
+		return err
+	}
+
+	if err = store.DB.Ping(); err != nil {
+		return fmt.Errorf("connect to %s error:%v", maskedDSN(cfg), err)
+	}
+
+	ConfigureListOrdering(store.DB, gen)
+
+	return nil
+}
+
+func maskedDSN(cfg *mysql.Config) string {
+	if cfg.Passwd == "" {
+		return cfg.FormatDSN()
+	}
+	return strings.ReplaceAll(cfg.FormatDSN(), cfg.Passwd, "<ADAPTED>")
+}