snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/redis3/ItemList.go

@@ -0,0 +1,510 @@
+package redis3
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/util/skiplist"
+)
+
+type ItemList struct {
+	skipList  *skiplist.SkipList
+	batchSize int
+	client    redis.UniversalClient
+	prefix    string
+}
+
+/*
+Be reluctant to create new nodes. Try to fit into either previous node or next node.
+Prefer to add to previous node.
+
+There are multiple cases after finding the name for greater or equal node
+ 1. found and node.Key == name
+    The node contains a batch with leading key the same as the name
+    nothing to do
+ 2. no such node found or node.Key > name
+
+    if no such node found
+      prevNode = list.LargestNode
+
+	// case 2.1
+    if previousNode contains name
+      nothing to do
+
+    // prefer to add to previous node
+    if prevNode != nil {
+		// case 2.2
+		if prevNode has capacity
+			prevNode.add name, and save
+			return
+		// case 2.3
+		split prevNode by name
+    }
+
+	// case 2.4
+	// merge into next node. Avoid too many nodes if adding data in reverse order.
+	if nextNode is not nil and nextNode has capacity
+	  delete nextNode.Key
+      nextNode.Key = name
+      nextNode.batch.add name
+      insert nodeNode.Key
+	  return
+
+	// case 2.5
+    if prevNode is nil
+      insert new node with key = name, value = batch{name}
+      return
+
+*/
+
+func (nl *ItemList) canAddMember(node *skiplist.SkipListElementReference, name string) (alreadyContains bool, nodeSize int, err error) {
+	ctx := context.Background()
+	pipe := nl.client.TxPipeline()
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	countOperation := pipe.ZLexCount(ctx, key, "-", "+")
+	scoreOperationt := pipe.ZScore(ctx, key, name)
+	if _, err = pipe.Exec(ctx); err != nil && err != redis.Nil {
+		return false, 0, err
+	}
+	if err == redis.Nil {
+		err = nil
+	}
+	alreadyContains = scoreOperationt.Err() == nil
+	nodeSize = int(countOperation.Val())
+	return
+}
+
+func (nl *ItemList) WriteName(name string) error {
+
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	// case 1: the name already exists as one leading key in the batch
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		return nil
+	}
+
+	var prevNodeReference *skiplist.SkipListElementReference
+	if !found {
+		prevNodeReference = nl.skipList.GetLargestNodeReference()
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNodeReference = nextNode.Prev
+	}
+
+	if prevNodeReference != nil {
+		alreadyContains, nodeSize, err := nl.canAddMember(prevNodeReference, name)
+		if err != nil {
+			return err
+		}
+		if alreadyContains {
+			// case 2.1
+			return nil
+		}
+
+		// case 2.2
+		if nodeSize < nl.batchSize {
+			return nl.NodeAddMember(prevNodeReference, name)
+		}
+
+		// case 2.3
+		x := nl.NodeInnerPosition(prevNodeReference, name)
+		y := nodeSize - x
+		addToX := x <= y
+		// add to a new node
+		if x == 0 || y == 0 {
+			if err := nl.ItemAdd(lookupKey, 0, name); err != nil {
+				return err
+			}
+			return nil
+		}
+		if addToX {
+			// collect names before name, add them to X
+			namesToX, err := nl.NodeRangeBeforeExclusive(prevNodeReference, name)
+			if err != nil {
+				return err
+			}
+			// delete skiplist reference to old node
+			if _, err := nl.skipList.DeleteByKey(prevNodeReference.Key); err != nil {
+				return err
+			}
+			// add namesToY and name to a new X
+			namesToX = append(namesToX, name)
+			if err := nl.ItemAdd([]byte(namesToX[0]), 0, namesToX...); err != nil {
+				return err
+			}
+			// remove names less than name from current Y
+			if err := nl.NodeDeleteBeforeExclusive(prevNodeReference, name); err != nil {
+				return err
+			}
+
+			// point skip list to current Y
+			if err := nl.ItemAdd(lookupKey, prevNodeReference.ElementPointer); err != nil {
+				return err
+			}
+			return nil
+		} else {
+			// collect names after name, add them to Y
+			namesToY, err := nl.NodeRangeAfterExclusive(prevNodeReference, name)
+			if err != nil {
+				return err
+			}
+			// add namesToY and name to a new Y
+			namesToY = append(namesToY, name)
+			if err := nl.ItemAdd(lookupKey, 0, namesToY...); err != nil {
+				return err
+			}
+			// remove names after name from current X
+			if err := nl.NodeDeleteAfterExclusive(prevNodeReference, name); err != nil {
+				return err
+			}
+			return nil
+		}
+
+	}
+
+	// case 2.4
+	if nextNode != nil {
+		nodeSize := nl.NodeSize(nextNode.Reference())
+		if nodeSize < nl.batchSize {
+			if id, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+				return err
+			} else {
+				if err := nl.ItemAdd(lookupKey, id, name); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+
+	// case 2.5
+	// now prevNode is nil
+	return nl.ItemAdd(lookupKey, 0, name)
+}
+
+/*
+// case 1: exists in nextNode
+
+	if nextNode != nil && nextNode.Key == name {
+		remove from nextNode, update nextNode
+		// TODO: merge with prevNode if possible?
+		return
+	}
+
+if nextNode is nil
+
+	prevNode = list.Largestnode
+
+if prevNode == nil and nextNode.Prev != nil
+
+	prevNode = load(nextNode.Prev)
+
+// case 2: does not exist
+// case 2.1
+
+	if prevNode == nil {
+		return
+	}
+
+// case 2.2
+
+	if prevNameBatch does not contain name {
+		return
+	}
+
+// case 3
+delete from prevNameBatch
+if prevNameBatch + nextNode < capacityList
+
+	// case 3.1
+	merge
+
+else
+
+	// case 3.2
+	update prevNode
+*/
+func (nl *ItemList) DeleteName(name string) error {
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+
+	// case 1
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		if err := nl.NodeDeleteMember(nextNode.Reference(), name); err != nil {
+			return err
+		}
+		minName := nl.NodeMin(nextNode.Reference())
+		if minName == "" {
+			return nl.NodeDelete(nextNode.Reference())
+		}
+		return nl.ItemAdd([]byte(minName), nextNode.Id)
+	}
+
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNode, err = nl.skipList.LoadElement(nextNode.Prev)
+		if err != nil {
+			return err
+		}
+	}
+
+	// case 2
+	if prevNode == nil {
+		// case 2.1
+		return nil
+	}
+	if !nl.NodeContainsItem(prevNode.Reference(), name) {
+		return nil
+	}
+
+	// case 3
+	if err := nl.NodeDeleteMember(prevNode.Reference(), name); err != nil {
+		return err
+	}
+	prevSize := nl.NodeSize(prevNode.Reference())
+	if prevSize == 0 {
+		if _, err := nl.skipList.DeleteByKey(prevNode.Key); err != nil {
+			return err
+		}
+		return nil
+	}
+	nextSize := nl.NodeSize(nextNode.Reference())
+	if nextSize > 0 && prevSize+nextSize < nl.batchSize {
+		// case 3.1 merge nextNode and prevNode
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		nextNames, err := nl.NodeRangeBeforeExclusive(nextNode.Reference(), "")
+		if err != nil {
+			return err
+		}
+		if err := nl.NodeAddMember(prevNode.Reference(), nextNames...); err != nil {
+			return err
+		}
+		return nl.NodeDelete(nextNode.Reference())
+	} else {
+		// case 3.2 update prevNode
+		// no action to take
+		return nil
+	}
+}
+
+func (nl *ItemList) ListNames(startFrom string, visitNamesFn func(name string) bool) error {
+	lookupKey := []byte(startFrom)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		prevNode = nil
+	}
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if prevNode != nil {
+		if !nl.NodeScanInclusiveAfter(prevNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+	}
+
+	for nextNode != nil {
+		if !nl.NodeScanInclusiveAfter(nextNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+		nextNode, err = nl.skipList.LoadElement(nextNode.Next[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (nl *ItemList) IsEmpty() bool {
+	return nl.skipList.StartLevels[0] == nil
+}
+
+func (nl *ItemList) RemoteAllListElement() error {
+
+	t := nl.skipList
+
+	nodeRef := t.StartLevels[0]
+	for nodeRef != nil {
+		node, err := t.LoadElement(nodeRef)
+		if err != nil {
+			return err
+		}
+		if node == nil {
+			return nil
+		}
+		if err := t.DeleteElement(node); err != nil {
+			return err
+		}
+		if err := nl.NodeDelete(node.Reference()); err != nil {
+			return err
+		}
+		nodeRef = node.Next[0]
+	}
+	return nil
+
+}
+
+func (nl *ItemList) NodeContainsItem(node *skiplist.SkipListElementReference, item string) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	_, err := nl.client.ZScore(context.Background(), key, item).Result()
+	if err == redis.Nil {
+		return false
+	}
+	if err == nil {
+		return true
+	}
+	return false
+}
+
+func (nl *ItemList) NodeSize(node *skiplist.SkipListElementReference) int {
+	if node == nil {
+		return 0
+	}
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "+").Val())
+}
+
+func (nl *ItemList) NodeAddMember(node *skiplist.SkipListElementReference, names ...string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	var members []redis.Z
+	for _, name := range names {
+		members = append(members, redis.Z{
+			Score:  0,
+			Member: name,
+		})
+	}
+	return nl.client.ZAddNX(context.Background(), key, members...).Err()
+}
+func (nl *ItemList) NodeDeleteMember(node *skiplist.SkipListElementReference, name string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.ZRem(context.Background(), key, name).Err()
+}
+
+func (nl *ItemList) NodeDelete(node *skiplist.SkipListElementReference) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.Del(context.Background(), key).Err()
+}
+
+func (nl *ItemList) NodeInnerPosition(node *skiplist.SkipListElementReference, name string) int {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "("+name).Val())
+}
+
+func (nl *ItemList) NodeMin(node *skiplist.SkipListElementReference) string {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	slice := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min:    "-",
+		Max:    "+",
+		Offset: 0,
+		Count:  1,
+	}).Val()
+	if len(slice) > 0 {
+		s := slice[0]
+		return s
+	}
+	return ""
+}
+
+func (nl *ItemList) NodeScanInclusiveAfter(node *skiplist.SkipListElementReference, startFrom string, visitNamesFn func(name string) bool) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "[" + startFrom
+	}
+	names := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Val()
+	for _, n := range names {
+		if !visitNamesFn(n) {
+			return false
+		}
+	}
+	return true
+}
+
+func (nl *ItemList) NodeRangeBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: "-",
+		Max: stopAt,
+	}).Result()
+}
+func (nl *ItemList) NodeRangeAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Result()
+}
+
+func (nl *ItemList) NodeDeleteBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, "-", stopAt).Err()
+}
+func (nl *ItemList) NodeDeleteAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, startFrom, "+").Err()
+}
+
+func (nl *ItemList) ItemAdd(lookupKey []byte, idIfKnown int64, names ...string) error {
+	if id, err := nl.skipList.InsertByKey(lookupKey, idIfKnown, nil); err != nil {
+		return err
+	} else {
+		if len(names) > 0 {
+			return nl.NodeAddMember(&skiplist.SkipListElementReference{
+				ElementPointer: id,
+				Key:            lookupKey,
+			}, names...)
+		}
+	}
+	return nil
+}

weed/filer/redis3/README.md

@@ -0,0 +1,4 @@
+Desuppported.
+
+This implementation attempts to use skip list.
+Did not get any report on actual benefits.

weed/filer/redis3/item_list_serde.go

@@ -0,0 +1,75 @@
+package redis3
+
+import (
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/skiplist"
+	"google.golang.org/protobuf/proto"
+)
+
+func LoadItemList(data []byte, prefix string, client redis.UniversalClient, store skiplist.ListStore, batchSize int) *ItemList {
+
+	nl := &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+
+	if len(data) == 0 {
+		return nl
+	}
+
+	message := &skiplist.SkipListProto{}
+	if err := proto.Unmarshal(data, message); err != nil {
+		glog.Errorf("loading skiplist: %v", err)
+	}
+	nl.skipList.MaxNewLevel = int(message.MaxNewLevel)
+	nl.skipList.MaxLevel = int(message.MaxLevel)
+	for i, ref := range message.StartLevels {
+		nl.skipList.StartLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	for i, ref := range message.EndLevels {
+		nl.skipList.EndLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	return nl
+}
+
+func (nl *ItemList) HasChanges() bool {
+	return nl.skipList.HasChanges
+}
+
+func (nl *ItemList) ToBytes() []byte {
+	message := &skiplist.SkipListProto{}
+	message.MaxNewLevel = int32(nl.skipList.MaxNewLevel)
+	message.MaxLevel = int32(nl.skipList.MaxLevel)
+	for _, ref := range nl.skipList.StartLevels {
+		if ref == nil {
+			break
+		}
+		message.StartLevels = append(message.StartLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	for _, ref := range nl.skipList.EndLevels {
+		if ref == nil {
+			break
+		}
+		message.EndLevels = append(message.EndLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	data, err := proto.Marshal(message)
+	if err != nil {
+		glog.Errorf("marshal skiplist: %v", err)
+	}
+	return data
+}

weed/filer/redis3/kv_directory_children.go

@@ -0,0 +1,151 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+const maxNameBatchSizeLimit = 1000000
+
+func insertChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer func() {
+		mutex.Unlock()
+	}()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.WriteName(name); err != nil {
+		glog.ErrorfCtx(ctx, "add %s %s: %v", key, name, err)
+		return err
+	}
+
+	if !nameList.HasChanges() {
+		return nil
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.DeleteName(name); err != nil {
+		return err
+	}
+	if !nameList.HasChanges() {
+		// Nothing to remove. If the list is empty anyway, its header outlived the
+		// removal of the last name - a delete that failed here before - so take it
+		// now rather than leaving the key behind for good.
+		if nameList.IsEmpty() {
+			return client.Del(ctx, key).Err()
+		}
+		return nil
+	}
+
+	// An emptied list reads the same as no list at all, and keeping the header would
+	// leave a key behind for every directory that is emptied.
+	if nameList.IsEmpty() {
+		return client.Del(ctx, key).Err()
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, onDeleteFn func(name string) error) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames("", func(name string) bool {
+		if err := onDeleteFn(name); err != nil {
+			glog.ErrorfCtx(ctx, "delete %s child %s: %v", key, name, err)
+			return false
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	if err = nameList.RemoteAllListElement(); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func listChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, startFileName string, eachFn func(name string) bool) error {
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames(startFileName, func(name string) bool {
+		return eachFn(name)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+
+}

weed/filer/redis3/kv_directory_children_test.go

@@ -0,0 +1,27 @@
+package redis3
+
+import (
+	"context"
+	"strconv"
+	"testing"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/stvp/tempredis"
+)
+
+func BenchmarkRedis(b *testing.B) {
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	for i := 0; i < b.N; i++ {
+		client.ZAddNX(context.Background(), "/yyy/bin", redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"})
+	}
+}

weed/filer/redis3/redis_cluster_store.go

@@ -0,0 +1,59 @@
+package redis3
+
+import (
+	"crypto/tls"
+
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v9"
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_conf"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_tls"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisCluster3Store{})
+}
+
+type RedisCluster3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *RedisCluster3Store) GetName() string {
+	return "redis_cluster3"
+}
+
+func (store *RedisCluster3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", false)
+	configuration.SetDefault(prefix+"routeByLatency", false)
+
+	tlsConfig, err := redis_tls.Config(configuration, prefix)
+	if err != nil {
+		return err
+	}
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+		tlsConfig,
+		redis_conf.Read(configuration, prefix),
+	)
+}
+
+func (store *RedisCluster3Store) initialize(addresses []string, password string, readOnly, routeByLatency bool, tlsConfig *tls.Config, settings redis_conf.Settings) (err error) {
+	options := &redis.ClusterOptions{
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+		TLSConfig:      tlsConfig,
+	}
+	settings.ApplyToCluster(options)
+	store.Client = redis.NewClusterClient(options)
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/redis_sentinel_store.go

@@ -0,0 +1,60 @@
+package redis3
+
+import (
+	"crypto/tls"
+
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v9"
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_conf"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_tls"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis3SentinelStore{})
+}
+
+type Redis3SentinelStore struct {
+	UniversalRedis3Store
+}
+
+func (store *Redis3SentinelStore) GetName() string {
+	return "redis3_sentinel"
+}
+
+func (store *Redis3SentinelStore) Initialize(configuration util.Configuration, prefix string) (err error) {
+	tlsConfig, err := redis_tls.Config(configuration, prefix)
+	if err != nil {
+		return err
+	}
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"masterName"),
+		configuration.GetString(prefix+"username"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetString(prefix+"sentinel_username"),
+		configuration.GetString(prefix+"sentinel_password"),
+		configuration.GetInt(prefix+"database"),
+		tlsConfig,
+		redis_conf.Read(configuration, prefix),
+	)
+}
+
+func (store *Redis3SentinelStore) initialize(addresses []string, masterName string, username string, password string, sentinelUsername string, sentinelPassword string, database int, tlsConfig *tls.Config, settings redis_conf.Settings) (err error) {
+	options := &redis.FailoverOptions{
+		MasterName:       masterName,
+		SentinelAddrs:    addresses,
+		Username:         username,
+		Password:         password,
+		SentinelUsername: sentinelUsername,
+		SentinelPassword: sentinelPassword,
+		DB:               database,
+		TLSConfig:        tlsConfig,
+	}
+	settings.ApplyToFailover(options)
+	store.Client = redis.NewFailoverClient(options)
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/redis_store.go

@@ -0,0 +1,52 @@
+package redis3
+
+import (
+	"crypto/tls"
+
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v9"
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_conf"
+	"github.com/seaweedfs/seaweedfs/weed/filer/redis_tls"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis3Store{})
+}
+
+type Redis3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *Redis3Store) GetName() string {
+	return "redis3"
+}
+
+func (store *Redis3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	tlsConfig, err := redis_tls.Config(configuration, prefix)
+	if err != nil {
+		return err
+	}
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+		tlsConfig,
+		redis_conf.Read(configuration, prefix),
+	)
+}
+
+func (store *Redis3Store) initialize(hostPort string, password string, database int, tlsConfig *tls.Config, settings redis_conf.Settings) (err error) {
+	options := &redis.Options{
+		Addr:      hostPort,
+		Password:  password,
+		DB:        database,
+		TLSConfig: tlsConfig,
+	}
+	settings.ApplyTo(options)
+	store.Client = redis.NewClient(options)
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/skiplist_element_store.go

@@ -0,0 +1,63 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util/skiplist"
+	"google.golang.org/protobuf/proto"
+)
+
+type SkipListElementStore struct {
+	Prefix string
+	client redis.UniversalClient
+}
+
+var _ = skiplist.ListStore(&SkipListElementStore{})
+
+func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore {
+	return &SkipListElementStore{
+		Prefix: prefix,
+		client: client,
+	}
+}
+
+func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := proto.Marshal(element)
+	if err != nil {
+		glog.Errorf("marshal %s: %v", key, err)
+	}
+	return m.client.Set(context.Background(), key, data, 0).Err()
+}
+
+func (m *SkipListElementStore) DeleteElement(id int64) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	return m.client.Del(context.Background(), key).Err()
+}
+
+func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := m.client.Get(context.Background(), key).Result()
+	if err != nil {
+		if err == redis.Nil {
+			return nil, nil
+		}
+		return nil, err
+	}
+	t := &skiplist.SkipListElement{}
+	err = proto.Unmarshal([]byte(data), t)
+	if err == nil {
+		for i := 0; i < len(t.Next); i++ {
+			if t.Next[i].IsNil() {
+				t.Next[i] = nil
+			}
+		}
+		if t.Prev.IsNil() {
+			t.Prev = nil
+		}
+	}
+	return t, err
+}

weed/filer/redis3/universal_redis_store.go

@@ -0,0 +1,211 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+
+	redsync "github.com/go-redsync/redsync/v4"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedis3Store struct {
+	Client  redis.UniversalClient
+	redsync *redsync.Redsync
+}
+
+func (store *UniversalRedis3Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedis3Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	if err = store.doInsertEntry(ctx, entry); err != nil {
+		return err
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+
+	if name != "" {
+		if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
+		value = util.MaybeGzipData(value)
+	}
+
+	if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+	return nil
+}
+
+func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.doInsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedis3Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	// The child listing is dropped by DeleteFolderChildren, together with the
+	// children it describes. Dropping it here would also discard an entry that
+	// arrived after the caller judged this directory empty, and nothing else
+	// records that the entry is there.
+	_, err = store.Client.Del(ctx, string(fullpath)).Result()
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	dir, name := fullpath.DirAndName()
+
+	if name != "" {
+		if err = removeChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	dirListKey := genDirectoryListKey(string(fullpath))
+	if err = removeChildren(ctx, store, dirListKey, func(name string) error {
+		path := util.NewFullPath(string(fullpath), name)
+		_, err = store.Client.Del(ctx, string(path)).Result()
+		if err != nil {
+			return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
+		}
+		// not efficient, but need to remove if it is a directory
+		store.Client.Del(ctx, genDirectoryListKey(string(path)))
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// removeChildren clears the skip list nodes but leaves the list itself behind
+	if _, err = store.Client.Del(ctx, dirListKey).Result(); err != nil {
+		return fmt.Errorf("DeleteFolderChildren %s list: %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *UniversalRedis3Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	dirListKey := genDirectoryListKey(string(dirPath))
+	counter := int64(0)
+
+	var callbackErr error
+	err = listChildren(ctx, store, dirListKey, startFileName, func(fileName string) bool {
+		if startFileName != "" {
+			if !includeStartFile && startFileName == fileName {
+				return true
+			}
+		}
+
+		path := util.NewFullPath(string(dirPath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		lastFileName = fileName
+		if err != nil {
+			glog.V(0).InfofCtx(ctx, "list %s : %v", path, err)
+			if err == filer_pb.ErrNotFound {
+				return true
+			}
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.Client.Del(ctx, string(path)).Result()
+					store.Client.ZRem(ctx, dirListKey, fileName).Result()
+					return true
+				}
+			}
+			counter++
+
+			resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
+			if resEachEntryFuncErr != nil {
+				glog.V(0).InfofCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", fileName, resEachEntryFuncErr)
+				callbackErr = resEachEntryFuncErr
+				return false
+			}
+
+			if !resEachEntryFunc {
+				return false
+			}
+
+			if counter >= limit {
+				return false
+			}
+		}
+		return true
+	})
+
+	if callbackErr != nil {
+		return lastFileName, fmt.Errorf(
+			"failed to process eachEntryFunc for dir %q, entry %q: %w",
+			dirPath, lastFileName, callbackErr,
+		)
+	}
+
+	return lastFileName, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedis3Store) Shutdown() {
+	store.Client.Close()
+}

weed/filer/redis3/universal_redis_store_kv.go

@@ -0,0 +1,42 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/redis/go-redis/v9"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+)
+
+func (store *UniversalRedis3Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.Client.Set(ctx, string(key), value, 0).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv put: %w", err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	data, err := store.Client.Get(ctx, string(key)).Result()
+
+	if err == redis.Nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return []byte(data), err
+}
+
+func (store *UniversalRedis3Store) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.Client.Del(ctx, string(key)).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %w", err)
+	}
+
+	return nil
+}

weed/filer/redis3/universal_redis_store_test.go

@@ -0,0 +1,129 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/go-redsync/redsync/v4"
+	goredis "github.com/go-redsync/redsync/v4/redis/goredis/v9"
+	"github.com/redis/go-redis/v9"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newTestStore(t *testing.T) (*UniversalRedis3Store, util.FullPath) {
+	t.Helper()
+
+	if os.Getenv("RUN_REDIS_TESTS") != "1" {
+		t.Skip("redis3 tests are disabled. Start a redis-server and set RUN_REDIS_TESTS=1 to enable, REDIS_ADDR defaults to 127.0.0.1:6379.")
+	}
+
+	addr := os.Getenv("REDIS_ADDR")
+	if addr == "" {
+		addr = "127.0.0.1:6379"
+	}
+
+	ctx := context.Background()
+	client := redis.NewClient(&redis.Options{Addr: addr})
+	t.Cleanup(func() {
+		if err := client.Close(); err != nil {
+			t.Errorf("close redis client: %v", err)
+		}
+	})
+	if err := client.Ping(ctx).Err(); err != nil {
+		t.Fatalf("connect to redis at %s: %v", addr, err)
+	}
+
+	store := &UniversalRedis3Store{Client: client, redsync: redsync.New(goredis.NewPool(client))}
+
+	dir := util.FullPath(fmt.Sprintf("/redis3_test_%d", time.Now().UnixNano()))
+	t.Cleanup(func() {
+		if err := store.DeleteFolderChildren(ctx, dir); err != nil {
+			t.Errorf("cleanup %s children: %v", dir, err)
+		}
+		if err := store.DeleteEntry(ctx, dir); err != nil {
+			t.Errorf("cleanup %s: %v", dir, err)
+		}
+	})
+
+	return store, dir
+}
+
+func listNames(t *testing.T, store *UniversalRedis3Store, dir util.FullPath) []string {
+	t.Helper()
+
+	names := []string{}
+	if _, err := store.ListDirectoryEntries(context.Background(), dir, "", true, 100, func(entry *filer.Entry) (bool, error) {
+		names = append(names, entry.Name())
+		return true, nil
+	}); err != nil {
+		t.Fatalf("list %s: %v", dir, err)
+	}
+	return names
+}
+
+func TestDeleteEntryKeepsChildListing(t *testing.T) {
+	store, dir := newTestStore(t)
+	ctx := context.Background()
+
+	now := time.Now()
+	child := dir.Child("obj")
+	if err := store.InsertEntry(ctx, &filer.Entry{
+		FullPath: child,
+		Attr:     filer.Attr{Crtime: now, Mtime: now, Mode: 0644},
+	}); err != nil {
+		t.Fatalf("InsertEntry %s: %v", child, err)
+	}
+
+	// The listing is the only record that the child sits under this directory, so
+	// removing the directory entry must leave it alone. An entry that arrived after
+	// the caller judged the directory empty would otherwise become unreachable.
+	if err := store.DeleteEntry(ctx, dir); err != nil {
+		t.Fatalf("DeleteEntry %s: %v", dir, err)
+	}
+	if names := listNames(t, store, dir); len(names) != 1 || names[0] != "obj" {
+		t.Errorf("child should still be listed after the directory entry is deleted, got %v", names)
+	}
+
+	// DeleteFolderChildren owns that cleanup, and takes the listing with it
+	if err := store.DeleteFolderChildren(ctx, dir); err != nil {
+		t.Fatalf("DeleteFolderChildren %s: %v", dir, err)
+	}
+	if names := listNames(t, store, dir); len(names) != 0 {
+		t.Errorf("listing should be empty once the children are deleted, got %v", names)
+	}
+	if n, err := store.Client.Exists(ctx, genDirectoryListKey(string(dir))).Result(); err != nil {
+		t.Fatalf("exists %s: %v", dir, err)
+	} else if n != 0 {
+		t.Errorf("listing key should be removed with the children, still present")
+	}
+}
+
+func TestRemovingTheLastChildDropsTheListing(t *testing.T) {
+	store, dir := newTestStore(t)
+	ctx := context.Background()
+
+	now := time.Now()
+	child := dir.Child("obj")
+	if err := store.InsertEntry(ctx, &filer.Entry{
+		FullPath: child,
+		Attr:     filer.Attr{Crtime: now, Mtime: now, Mode: 0644},
+	}); err != nil {
+		t.Fatalf("InsertEntry %s: %v", child, err)
+	}
+
+	// An emptied listing must not leave its header behind: cleanup deletes such a
+	// folder without touching the listing, so the key would never be collected.
+	if err := store.DeleteEntry(ctx, child); err != nil {
+		t.Fatalf("DeleteEntry %s: %v", child, err)
+	}
+	if n, err := store.Client.Exists(ctx, genDirectoryListKey(string(dir))).Result(); err != nil {
+		t.Fatalf("exists %s: %v", dir, err)
+	} else if n != 0 {
+		t.Errorf("listing key should be gone once the last child is removed, still present")
+	}
+}