snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/ydb/doc.go

@@ -0,0 +1,7 @@
+/*
+Package ydb is for YDB filer store.
+
+The referenced "github.com/ydb-platform/ydb-go-sdk/v3" library is too big when compiled.
+So this is only compiled in "make full_install".
+*/
+package ydb

weed/filer/ydb/readme.md

@@ -0,0 +1,27 @@
+## YDB
+
+database: https://github.com/ydb-platform/ydb
+
+go driver: https://github.com/ydb-platform/ydb-go-sdk
+
+options:
+
+```
+[ydb]
+enabled=true
+dsn=grpcs://ydb-ru.yandex.net:2135/?database=/ru/home/username/db
+prefix="seaweedfs"
+useBucketPrefix=true
+poolSizeLimit=50
+dialTimeOut = 10
+```
+
+Authenticate produced with one of next environment variables:
+ * `YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=<path/to/sa_key_file>` — used service account key file by path
+ * `YDB_ANONYMOUS_CREDENTIALS="1"` — used for authenticate with anonymous access. Anonymous access needs for connect to testing YDB installation
+ * `YDB_METADATA_CREDENTIALS="1"` — used metadata service for authenticate to YDB from yandex cloud virtual machine or from yandex function
+ * `YDB_ACCESS_TOKEN_CREDENTIALS=<access_token>` — used for authenticate to YDB with short-life access token. For example, access token may be IAM token
+ * `YDB_CONNECTION_STRING="grpcs://endpoint/?database=database"`
+
+ * i test using this dev database:
+`make dev_ydb`

weed/filer/ydb/ydb_queries.go

@@ -0,0 +1,73 @@
+//go:build ydb
+
+package ydb
+
+import asql "github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+
+const (
+	upsertQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+		DECLARE $name AS Utf8;
+		DECLARE $meta AS String;
+		DECLARE $expire_at AS Optional<uint32>;
+
+		UPSERT INTO ` + asql.DEFAULT_TABLE + `
+			(dir_hash, name, directory, meta, expire_at)
+		VALUES
+			($dir_hash, $name, $directory, $meta, $expire_at);`
+
+	deleteQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+		DECLARE $name AS Utf8;
+
+		DELETE FROM ` + asql.DEFAULT_TABLE + ` 
+		WHERE dir_hash = $dir_hash AND directory = $directory AND name = $name;`
+
+	findQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+		DECLARE $name AS Utf8;
+		
+		SELECT meta
+		FROM ` + asql.DEFAULT_TABLE + ` 
+		WHERE dir_hash = $dir_hash AND directory = $directory AND name = $name;`
+
+	deleteFolderChildrenQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+
+		DELETE FROM ` + asql.DEFAULT_TABLE + ` 
+		WHERE dir_hash = $dir_hash AND directory = $directory;`
+
+	listDirectoryQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+		DECLARE $start_name AS Utf8;
+		DECLARE $prefix AS Utf8;
+		DECLARE $limit AS Uint64;
+		
+		SELECT name, meta
+		FROM ` + asql.DEFAULT_TABLE + `
+		WHERE dir_hash = $dir_hash AND directory = $directory and name > $start_name and name LIKE $prefix
+		ORDER BY name ASC LIMIT $limit;`
+
+	listInclusiveDirectoryQuery = `
+		PRAGMA TablePathPrefix("%v");
+		DECLARE $dir_hash AS int64;
+		DECLARE $directory AS Utf8;
+		DECLARE $start_name AS Utf8;
+		DECLARE $prefix AS Utf8;
+		DECLARE $limit AS Uint64;
+		
+		SELECT name, meta
+		FROM ` + asql.DEFAULT_TABLE + `
+		WHERE dir_hash = $dir_hash AND directory = $directory and name >= $start_name and name LIKE $prefix
+		ORDER BY name ASC LIMIT $limit;`
+)

weed/filer/ydb/ydb_store.go

@@ -0,0 +1,526 @@
+//go:build ydb
+
+package ydb
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	environ "github.com/ydb-platform/ydb-go-sdk-auth-environ"
+	"github.com/ydb-platform/ydb-go-sdk/v3"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
+)
+
+const (
+	defaultDialTimeOut            = 10
+	defaultPartitionBySizeEnabled = true
+	defaultPartitionSizeMb        = 200
+	defaultPartitionByLoadEnabled = true
+	defaultMinPartitionsCount     = 5
+	defaultMaxPartitionsCount     = 1000
+	defaultMaxListChunk           = 2000
+	defaultTablePathPrefix        = "seaweedfs"
+)
+
+var (
+	roQC = query.WithTxControl(query.OnlineReadOnlyTxControl())
+	rwQC = query.WithTxControl(query.DefaultTxControl())
+)
+
+type YdbStore struct {
+	DB                     *ydb.Driver
+	dirBuckets             string
+	tablePathPrefix        string
+	SupportBucketTable     bool
+	partitionBySizeEnabled options.FeatureFlag
+	partitionSizeMb        uint64
+	partitionByLoadEnabled options.FeatureFlag
+	minPartitionsCount     uint64
+	maxPartitionsCount     uint64
+	maxListChunk           int
+	dbs                    map[string]bool
+	dbsLock                sync.Mutex
+}
+
+func init() {
+	filer.Stores = append(filer.Stores, &YdbStore{})
+}
+
+func (store *YdbStore) GetName() string {
+	return "ydb"
+}
+
+func (store *YdbStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	configuration.SetDefault(prefix+"prefix", defaultTablePathPrefix)
+	configuration.SetDefault(prefix+"partitionBySizeEnabled", defaultPartitionBySizeEnabled)
+	configuration.SetDefault(prefix+"partitionSizeMb", defaultPartitionSizeMb)
+	configuration.SetDefault(prefix+"partitionByLoadEnabled", defaultPartitionByLoadEnabled)
+	configuration.SetDefault(prefix+"minPartitionsCount", defaultMinPartitionsCount)
+	configuration.SetDefault(prefix+"maxPartitionsCount", defaultMaxPartitionsCount)
+	configuration.SetDefault(prefix+"maxListChunk", defaultMaxListChunk)
+	return store.initialize(
+		configuration.GetString("filer.options.buckets_folder"),
+		configuration.GetString(prefix+"dsn"),
+		configuration.GetString(prefix+"prefix"),
+		configuration.GetBool(prefix+"useBucketPrefix"),
+		configuration.GetInt(prefix+"dialTimeOut"),
+		configuration.GetInt(prefix+"poolSizeLimit"),
+		configuration.GetBool(prefix+"partitionBySizeEnabled"),
+		uint64(configuration.GetInt(prefix+"partitionSizeMb")),
+		configuration.GetBool(prefix+"partitionByLoadEnabled"),
+		uint64(configuration.GetInt(prefix+"minPartitionsCount")),
+		uint64(configuration.GetInt(prefix+"maxPartitionsCount")),
+		configuration.GetInt(prefix+"maxListChunk"),
+	)
+}
+
+func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix string, useBucketPrefix bool, dialTimeOut int, poolSizeLimit int, partitionBySizeEnabled bool, partitionSizeMb uint64, partitionByLoadEnabled bool, minPartitionsCount uint64, maxPartitionsCount uint64, maxListChunk int) (err error) {
+	store.dirBuckets = dirBuckets
+	store.SupportBucketTable = useBucketPrefix
+	if partitionBySizeEnabled {
+		store.partitionBySizeEnabled = options.FeatureEnabled
+	} else {
+		store.partitionBySizeEnabled = options.FeatureDisabled
+	}
+	if partitionByLoadEnabled {
+		store.partitionByLoadEnabled = options.FeatureEnabled
+	} else {
+		store.partitionByLoadEnabled = options.FeatureDisabled
+	}
+	store.partitionSizeMb = partitionSizeMb
+	store.minPartitionsCount = minPartitionsCount
+	store.maxPartitionsCount = maxPartitionsCount
+	store.maxListChunk = maxListChunk
+	if store.SupportBucketTable {
+		glog.V(0).Infof("enabled BucketPrefix")
+	}
+	store.dbs = make(map[string]bool)
+	ctx := context.Background()
+	if dialTimeOut == 0 {
+		dialTimeOut = defaultDialTimeOut
+	}
+	opts := []ydb.Option{
+		ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second),
+		environ.WithEnvironCredentials(),
+	}
+	if poolSizeLimit > 0 {
+		opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit))
+	}
+	if dsn == "" {
+		dsn = os.Getenv("YDB_CONNECTION_STRING")
+	}
+	store.DB, err = ydb.Open(ctx, dsn, opts...)
+	if err != nil {
+		return fmt.Errorf("can not connect to %s: %w", dsn, err)
+	}
+
+	store.tablePathPrefix = path.Join(store.DB.Name(), tablePathPrefix)
+
+	if err := store.ensureTables(ctx); err != nil {
+		return err
+	}
+	return err
+}
+
+func (store *YdbStore) doTxOrDB(ctx context.Context, q *string, params *table.QueryParameters, ts query.ExecuteOption, processResultFunc func(res query.Result) error) (err error) {
+	var res query.Result
+	if tx, ok := ctx.Value("tx").(query.Transaction); ok {
+		res, err = tx.Query(ctx, *q, query.WithParameters(params))
+		if err != nil {
+			return fmt.Errorf("execute transaction: %w", err)
+		}
+	} else {
+		err = store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+			res, err = s.Query(ctx, *q, query.WithParameters(params), ts)
+			if err != nil {
+				return fmt.Errorf("execute statement: %w", err)
+			}
+			return nil
+		}, query.WithIdempotent())
+	}
+	if err != nil {
+		return err
+	}
+	if res != nil {
+		defer func() { _ = res.Close(ctx) }()
+		if processResultFunc != nil {
+			if err = processResultFunc(res); err != nil {
+				return fmt.Errorf("process result: %w", err)
+			}
+		}
+	}
+	return err
+}
+
+func (store *YdbStore) insertOrUpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	dir, name := entry.FullPath.DirAndName()
+	meta, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encode %s: %s", entry.FullPath, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		meta = util.MaybeGzipData(meta)
+	}
+	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
+	fileMeta := FileMeta{util.HashStringToLong(dir), name, *shortDir, meta}
+	return store.doTxOrDB(ctx, withPragma(tablePathPrefix, upsertQuery), fileMeta.queryParameters(entry.TtlSec), rwQC, nil)
+}
+
+func (store *YdbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.insertOrUpdateEntry(ctx, entry)
+}
+
+func (store *YdbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+	return store.insertOrUpdateEntry(ctx, entry)
+}
+
+func (store *YdbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+	dir, name := fullpath.DirAndName()
+	var data []byte
+	entryFound := false
+	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
+	q := withPragma(tablePathPrefix, findQuery)
+	queryParams := table.NewQueryParameters(
+		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+		table.ValueParam("$directory", types.UTF8Value(*shortDir)),
+		table.ValueParam("$name", types.UTF8Value(name)))
+
+	err = store.doTxOrDB(ctx, q, queryParams, roQC, func(res query.Result) error {
+		for rs, err := range res.ResultSets(ctx) {
+			if err != nil {
+				return err
+			}
+			for row, err := range rs.Rows(ctx) {
+				if err != nil {
+					return err
+				}
+				if scanErr := row.Scan(&data); scanErr != nil {
+					return fmt.Errorf("scan %s: %v", fullpath, scanErr)
+				}
+				entryFound = true
+				return nil
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if !entryFound {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	entry = &filer.Entry{FullPath: fullpath}
+	if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+		return nil, fmt.Errorf("decode %s: %v", fullpath, decodeErr)
+	}
+	return entry, nil
+}
+
+func (store *YdbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+	dir, name := fullpath.DirAndName()
+	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
+	q := withPragma(tablePathPrefix, deleteQuery)
+	glog.V(4).InfofCtx(ctx, "DeleteEntry %s, tablePathPrefix %s, shortDir %s", fullpath, *tablePathPrefix, *shortDir)
+	queryParams := table.NewQueryParameters(
+		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+		table.ValueParam("$directory", types.UTF8Value(*shortDir)),
+		table.ValueParam("$name", types.UTF8Value(name)))
+
+	return store.doTxOrDB(ctx, q, queryParams, rwQC, nil)
+}
+
+func (store *YdbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+	dir := string(fullpath)
+	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
+	q := withPragma(tablePathPrefix, deleteFolderChildrenQuery)
+	queryParams := table.NewQueryParameters(
+		table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+		table.ValueParam("$directory", types.UTF8Value(*shortDir)))
+
+	return store.doTxOrDB(ctx, q, queryParams, rwQC, nil)
+}
+
+func (store *YdbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return store.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, "", eachEntryFunc)
+}
+
+func (store *YdbStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	dir := string(dirPath)
+	tablePathPrefix, shortDir := store.getPrefix(ctx, &dir)
+	baseInclusive := withPragma(tablePathPrefix, listInclusiveDirectoryQuery)
+	baseExclusive := withPragma(tablePathPrefix, listDirectoryQuery)
+	var entryCount int64
+	var prevFetchedLessThanChunk bool
+	for entryCount < limit {
+		if prevFetchedLessThanChunk {
+			break
+		}
+		var q *string
+		if entryCount == 0 && includeStartFile {
+			q = baseInclusive
+		} else {
+			q = baseExclusive
+		}
+		rest := limit - entryCount
+		chunkLimit := rest
+		if chunkLimit > int64(store.maxListChunk) {
+			chunkLimit = int64(store.maxListChunk)
+		}
+		var rowCount int64
+
+		params := table.NewQueryParameters(
+			table.ValueParam("$dir_hash", types.Int64Value(util.HashStringToLong(*shortDir))),
+			table.ValueParam("$directory", types.UTF8Value(*shortDir)),
+			table.ValueParam("$start_name", types.UTF8Value(startFileName)),
+			table.ValueParam("$prefix", types.UTF8Value(prefix+"%")),
+			table.ValueParam("$limit", types.Uint64Value(uint64(chunkLimit))),
+		)
+
+		err := store.doTxOrDB(ctx, q, params, roQC, func(res query.Result) error {
+			for rs, err := range res.ResultSets(ctx) {
+				if err != nil {
+					return err
+				}
+				for row, err := range rs.Rows(ctx) {
+					if err != nil {
+						return err
+					}
+
+					var name string
+					var data []byte
+					if scanErr := row.Scan(&name, &data); scanErr != nil {
+						return fmt.Errorf("scan %s: %w", dir, scanErr)
+					}
+
+					lastFileName = name
+					entry := &filer.Entry{FullPath: util.NewFullPath(dir, name)}
+					if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(data)); decodeErr != nil {
+						return fmt.Errorf("decode entry %s: %w", entry.FullPath, decodeErr)
+					}
+
+					resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+					if resEachEntryFuncErr != nil {
+						return fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.FullPath, resEachEntryFuncErr)
+					}
+
+					if !resEachEntryFunc {
+						return nil
+					}
+
+					rowCount++
+					entryCount++
+					startFileName = lastFileName
+
+					if entryCount >= limit {
+						return nil
+					}
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			return lastFileName, err
+		}
+
+		if rowCount < chunkLimit {
+			prevFetchedLessThanChunk = true
+		}
+	}
+	return lastFileName, nil
+}
+
+func (store *YdbStore) BeginTransaction(ctx context.Context) (context.Context, error) {
+	session, err := store.DB.Table().CreateSession(ctx)
+	if err != nil {
+		return ctx, err
+	}
+	tx, err := session.BeginTransaction(ctx, table.TxSettings(table.WithSerializableReadWrite()))
+	if err != nil {
+		return ctx, err
+	}
+	return context.WithValue(ctx, "tx", tx), nil
+}
+
+func (store *YdbStore) CommitTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(table.Transaction); ok {
+		_, err := tx.CommitTx(ctx)
+		return err
+	}
+	return nil
+}
+
+func (store *YdbStore) RollbackTransaction(ctx context.Context) error {
+	if tx, ok := ctx.Value("tx").(table.Transaction); ok {
+		return tx.Rollback(ctx)
+	}
+	return nil
+}
+
+func (store *YdbStore) Shutdown() {
+	_ = store.DB.Close(context.Background())
+}
+
+var _ filer.BucketAware = (*YdbStore)(nil)
+
+func (store *YdbStore) CanDropWholeBucket() bool {
+	return store.SupportBucketTable
+}
+
+func (store *YdbStore) OnBucketCreation(bucket string) {
+	if !store.SupportBucketTable {
+		return
+	}
+	prefix := path.Join(store.tablePathPrefix, bucket)
+
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	if err := store.createTable(context.Background(), prefix); err != nil {
+		glog.Errorf("createTable %s: %v", prefix, err)
+	}
+
+	if store.dbs == nil {
+		return
+	}
+	store.dbs[bucket] = true
+}
+
+func (store *YdbStore) OnBucketDeletion(bucket string) {
+	if !store.SupportBucketTable {
+		return
+	}
+	store.dbsLock.Lock()
+	defer store.dbsLock.Unlock()
+
+	prefix := path.Join(store.tablePathPrefix, bucket)
+	glog.V(4).Infof("deleting table %s", prefix)
+
+	if err := store.deleteTable(context.Background(), prefix); err != nil {
+		glog.Errorf("deleteTable %s: %v", prefix, err)
+	}
+
+	if err := store.DB.Scheme().RemoveDirectory(context.Background(), prefix); err != nil {
+		glog.Errorf("remove directory %s: %v", prefix, err)
+	}
+
+	if store.dbs == nil {
+		return
+	}
+	delete(store.dbs, bucket)
+}
+
+func (store *YdbStore) createTable(ctx context.Context, prefix string) error {
+	return store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+		return s.CreateTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE), store.createTableOptions()...)
+	})
+}
+
+func (store *YdbStore) deleteTable(ctx context.Context, prefix string) error {
+	if !store.SupportBucketTable {
+		return nil
+	}
+	if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+		return s.DropTable(ctx, path.Join(prefix, abstract_sql.DEFAULT_TABLE))
+	}); err != nil {
+		return err
+	}
+	glog.V(4).InfofCtx(ctx, "deleted table %s", prefix)
+
+	return nil
+}
+
+func (store *YdbStore) getPrefix(ctx context.Context, dir *string) (tablePathPrefix *string, shortDir *string) {
+	tablePathPrefix = &store.tablePathPrefix
+	shortDir = dir
+	if !store.SupportBucketTable {
+		return
+	}
+
+	prefixBuckets := store.dirBuckets + "/"
+	glog.V(4).InfofCtx(ctx, "dir: %s, prefixBuckets: %s", *dir, prefixBuckets)
+	if strings.HasPrefix(*dir, prefixBuckets) {
+		// detect bucket
+		bucketAndDir := (*dir)[len(prefixBuckets):]
+		glog.V(4).InfofCtx(ctx, "bucketAndDir: %s", bucketAndDir)
+		var bucket string
+		if t := strings.Index(bucketAndDir, "/"); t > 0 {
+			bucket = bucketAndDir[:t]
+		} else if t < 0 {
+			bucket = bucketAndDir
+		}
+		if bucket == "" {
+			return
+		}
+
+		// Dot-prefixed entries directly under /buckets (e.g. .system) are internal
+		// folders, not S3 buckets; keep them under the default prefix.
+		if strings.HasPrefix(bucket, ".") {
+			return
+		}
+
+		store.dbsLock.Lock()
+		defer store.dbsLock.Unlock()
+
+		if _, found := store.dbs[bucket]; !found {
+			glog.V(4).InfofCtx(ctx, "bucket %q not in cache, verifying existence via DescribeTable", bucket)
+			tablePath := path.Join(store.tablePathPrefix, bucket, abstract_sql.DEFAULT_TABLE)
+			err2 := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+				_, err3 := s.DescribeTable(ctx, tablePath)
+				return err3
+			})
+			if err2 != nil {
+				glog.V(4).InfofCtx(ctx, "bucket %q not found (DescribeTable %s failed)", bucket, tablePath)
+				return
+			}
+			glog.V(4).InfofCtx(ctx, "bucket %q exists, adding to cache", bucket)
+			store.dbs[bucket] = true
+		}
+		bucketPrefix := path.Join(store.tablePathPrefix, bucket)
+		tablePathPrefix = &bucketPrefix
+	}
+	return
+}
+
+func (store *YdbStore) ensureTables(ctx context.Context) error {
+	prefixFull := store.tablePathPrefix
+
+	glog.V(4).InfofCtx(ctx, "creating base table %s", prefixFull)
+	baseTable := path.Join(prefixFull, abstract_sql.DEFAULT_TABLE)
+	if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+		return s.CreateTable(ctx, baseTable, store.createTableOptions()...)
+	}); err != nil {
+		return fmt.Errorf("failed to create base table %s: %v", baseTable, err)
+	}
+
+	glog.V(4).InfofCtx(ctx, "creating bucket tables")
+	if store.SupportBucketTable {
+		store.dbsLock.Lock()
+		defer store.dbsLock.Unlock()
+		for bucket := range store.dbs {
+			glog.V(4).InfofCtx(ctx, "creating bucket table %s", bucket)
+			bucketTable := path.Join(prefixFull, bucket, abstract_sql.DEFAULT_TABLE)
+			if err := store.DB.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+				return s.CreateTable(ctx, bucketTable, store.createTableOptions()...)
+			}); err != nil {
+				glog.ErrorfCtx(ctx, "failed to create bucket table %s: %v", bucketTable, err)
+			}
+		}
+	}
+	return nil
+}

weed/filer/ydb/ydb_store_kv.go

@@ -0,0 +1,82 @@
+//go:build ydb
+
+package ydb
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/ydb-platform/ydb-go-sdk/v3/query"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
+)
+
+func (store *YdbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
+	fileMeta := FileMeta{dirHash, name, dirStr, value}
+	return store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+		_, err = s.Query(ctx, *withPragma(&store.tablePathPrefix, upsertQuery),
+			query.WithParameters(fileMeta.queryParameters(0)), rwQC)
+		if err != nil {
+			return fmt.Errorf("kv put execute %s: %v", util.NewFullPath(dirStr, name).Name(), err)
+		}
+		return nil
+	}, query.WithIdempotent())
+}
+
+func (store *YdbStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
+	valueFound := false
+	err = store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) error {
+		res, err := s.Query(ctx, *withPragma(&store.tablePathPrefix, findQuery),
+			query.WithParameters(table.NewQueryParameters(
+				table.ValueParam("$dir_hash", types.Int64Value(dirHash)),
+				table.ValueParam("$directory", types.UTF8Value(dirStr)),
+				table.ValueParam("$name", types.UTF8Value(name)))), roQC)
+		if err != nil {
+			return fmt.Errorf("kv get execute %s: %v", util.NewFullPath(dirStr, name).Name(), err)
+		}
+		defer func() { _ = res.Close(ctx) }()
+		for rs, err := range res.ResultSets(ctx) {
+			if err != nil {
+				return err
+			}
+			for row, err := range rs.Rows(ctx) {
+				if err != nil {
+					return err
+				}
+				if err := row.Scan(&value); err != nil {
+					return fmt.Errorf("scan %s : %v", util.NewFullPath(dirStr, name).Name(), err)
+				}
+				valueFound = true
+				return nil
+			}
+		}
+		return nil
+	}, query.WithIdempotent())
+
+	if !valueFound {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return value, nil
+}
+
+func (store *YdbStore) KvDelete(ctx context.Context, key []byte) (err error) {
+	dirStr, dirHash, name := abstract_sql.GenDirAndName(key)
+	return store.DB.Query().Do(ctx, func(ctx context.Context, s query.Session) (err error) {
+		_, err = s.Query(ctx, *withPragma(&store.tablePathPrefix, deleteQuery),
+			query.WithParameters(table.NewQueryParameters(
+				table.ValueParam("$dir_hash", types.Int64Value(dirHash)),
+				table.ValueParam("$directory", types.UTF8Value(dirStr)),
+				table.ValueParam("$name", types.UTF8Value(name)))), rwQC)
+		if err != nil {
+			return fmt.Errorf("kv delete %s: %v", util.NewFullPath(dirStr, name).Name(), err)
+		}
+		return nil
+	}, query.WithIdempotent())
+
+}

weed/filer/ydb/ydb_store_test.go

@@ -0,0 +1,20 @@
+//go:build ydb
+
+package ydb
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer/store_test"
+)
+
+func TestStore(t *testing.T) {
+	// run "make test_ydb" under docker folder.
+	// to set up local env
+	if false {
+		store := &YdbStore{}
+		store.initialize("/buckets", "grpc://localhost:2136/?database=local", "seaweedfs", true, 10, 50,
+			true, 200, true, 5, 1000, 2000)
+		store_test.TestFilerStore(t, store)
+	}
+}

weed/filer/ydb/ydb_types.go

@@ -0,0 +1,65 @@
+//go:build ydb
+
+package ydb
+
+import (
+	"fmt"
+
+	"github.com/ydb-platform/ydb-go-sdk/v3/table"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
+	"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
+)
+
+type FileMeta struct {
+	DirHash   int64  `ydb:"type:int64"`
+	Name      string `ydb:"type:utf8"`
+	Directory string `ydb:"type:utf8"`
+	Meta      []byte `ydb:"type:string"`
+}
+
+type FileMetas []FileMeta
+
+func (fm *FileMeta) queryParameters(ttlSec int32) *table.QueryParameters {
+	var expireAtValue types.Value
+	if ttlSec > 0 {
+		expireAtValue = types.OptionalValue(types.Uint32Value(uint32(ttlSec)))
+	} else {
+		expireAtValue = types.NullValue(types.TypeUint32)
+	}
+	return table.NewQueryParameters(
+		table.ValueParam("$dir_hash", types.Int64Value(fm.DirHash)),
+		table.ValueParam("$directory", types.UTF8Value(fm.Directory)),
+		table.ValueParam("$name", types.UTF8Value(fm.Name)),
+		table.ValueParam("$meta", types.BytesValue(fm.Meta)),
+		table.ValueParam("$expire_at", expireAtValue))
+}
+
+func (store *YdbStore) createTableOptions() []options.CreateTableOption {
+	columnUnit := options.TimeToLiveUnitSeconds
+	return []options.CreateTableOption{
+		options.WithColumn("dir_hash", types.TypeInt64),
+		options.WithColumn("directory", types.TypeUTF8),
+		options.WithColumn("name", types.TypeUTF8),
+		options.WithColumn("meta", types.TypeString),
+		options.WithColumn("expire_at", types.Optional(types.TypeUint32)),
+		options.WithPrimaryKeyColumn("dir_hash", "directory", "name"),
+		options.WithTimeToLiveSettings(options.TimeToLiveSettings{
+			ColumnName: "expire_at",
+			ColumnUnit: &columnUnit,
+			Mode:       options.TimeToLiveModeValueSinceUnixEpoch},
+		),
+		options.WithPartitioningSettings(
+			options.WithPartitioningBy([]string{"dir_hash", "name"}),
+			options.WithPartitioningBySize(store.partitionBySizeEnabled),
+			options.WithPartitionSizeMb(store.partitionSizeMb),
+			options.WithPartitioningByLoad(store.partitionByLoadEnabled),
+			options.WithMinPartitionsCount(store.minPartitionsCount),
+			options.WithMaxPartitionsCount(store.maxPartitionsCount),
+		),
+	}
+}
+
+func withPragma(prefix *string, query string) *string {
+	queryWithPragma := fmt.Sprintf(query, *prefix)
+	return &queryWithPragma
+}