snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/util/skiplist/Makefile

@@ -0,0 +1,6 @@
+all: gen
+
+.PHONY : gen
+
+gen:
+	protoc skiplist.proto --go_out=. --go-grpc_out=. --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative

weed/util/skiplist/list_store.go

@@ -0,0 +1,32 @@
+package skiplist
+
+type ListStore interface {
+	SaveElement(id int64, element *SkipListElement) error
+	DeleteElement(id int64) error
+	LoadElement(id int64) (*SkipListElement, error)
+}
+
+type MemStore struct {
+	m map[int64]*SkipListElement
+}
+
+func newMemStore() *MemStore {
+	return &MemStore{
+		m: make(map[int64]*SkipListElement),
+	}
+}
+
+func (m *MemStore) SaveElement(id int64, element *SkipListElement) error {
+	m.m[id] = element
+	return nil
+}
+
+func (m *MemStore) DeleteElement(id int64) error {
+	delete(m.m, id)
+	return nil
+}
+
+func (m *MemStore) LoadElement(id int64) (*SkipListElement, error) {
+	element := m.m[id]
+	return element, nil
+}

weed/util/skiplist/skiplist.go

@@ -0,0 +1,578 @@
+package skiplist
+
+// adapted from https://github.com/MauriceGit/skiplist/blob/master/skiplist.go
+
+import (
+	"bytes"
+	"fmt"
+	"math/bits"
+	"math/rand"
+	"time"
+)
+
+const (
+	// maxLevel denotes the maximum height of the skiplist. This height will keep the skiplist
+	// efficient for up to 34m entries. If there is a need for much more, please adjust this constant accordingly.
+	maxLevel = 25
+)
+
+type SkipList struct {
+	StartLevels [maxLevel]*SkipListElementReference
+	EndLevels   [maxLevel]*SkipListElementReference
+	MaxNewLevel int
+	MaxLevel    int
+	ListStore   ListStore
+	HasChanges  bool
+	// elementCount int
+}
+
+// NewSeed returns a new empty, initialized Skiplist.
+// Given a seed, a deterministic height/list behaviour can be achieved.
+// Eps is used to compare keys given by the ExtractKey() function on equality.
+func NewSeed(seed int64, listStore ListStore) *SkipList {
+
+	// Initialize random number generator.
+	rand.Seed(seed)
+	//fmt.Printf("SkipList seed: %v\n", seed)
+
+	list := &SkipList{
+		MaxNewLevel: maxLevel,
+		MaxLevel:    0,
+		ListStore:   listStore,
+		// elementCount: 0,
+	}
+
+	return list
+}
+
+// New returns a new empty, initialized Skiplist.
+func New(listStore ListStore) *SkipList {
+	return NewSeed(time.Now().UTC().UnixNano(), listStore)
+}
+
+// IsEmpty checks, if the skiplist is empty.
+func (t *SkipList) IsEmpty() bool {
+	return t.StartLevels[0] == nil
+}
+
+func (t *SkipList) generateLevel(maxLevel int) int {
+	level := maxLevel - 1
+	// First we apply some mask which makes sure that we don't get a level
+	// above our desired level. Then we find the first set bit.
+	var x = rand.Uint64() & ((1 << uint(maxLevel-1)) - 1)
+	zeroes := bits.TrailingZeros64(x)
+	if zeroes <= maxLevel {
+		level = zeroes
+	}
+
+	return level
+}
+
+func (t *SkipList) findEntryIndex(key []byte, minLevel int) int {
+	// Find good entry point so we don't accidentally skip half the list.
+	for i := t.MaxLevel; i >= 0; i-- {
+		if t.StartLevels[i] != nil && bytes.Compare(t.StartLevels[i].Key, key) < 0 || i <= minLevel {
+			return i
+		}
+	}
+	return 0
+}
+
+func (t *SkipList) findExtended(key []byte, findGreaterOrEqual bool) (prevElementIfVisited *SkipListElement, foundElem *SkipListElement, ok bool, err error) {
+
+	foundElem = nil
+	ok = false
+
+	if t.IsEmpty() {
+		return
+	}
+
+	index := t.findEntryIndex(key, 0)
+	var currentNode *SkipListElement
+
+	currentNode, err = t.LoadElement(t.StartLevels[index])
+	if err != nil {
+		return
+	}
+	if currentNode == nil {
+		return
+	}
+
+	// In case, that our first element is already greater-or-equal!
+	if findGreaterOrEqual && compareElement(currentNode, key) > 0 {
+		foundElem = currentNode
+		ok = true
+		return
+	}
+
+	for {
+		if compareElement(currentNode, key) == 0 {
+			foundElem = currentNode
+			ok = true
+			return
+		}
+
+		// Which direction are we continuing next time?
+		if currentNode.Next[index] != nil && bytes.Compare(currentNode.Next[index].Key, key) <= 0 {
+			// Go right
+			currentNode, err = t.LoadElement(currentNode.Next[index])
+			if err != nil {
+				return
+			}
+			if currentNode == nil {
+				return
+			}
+		} else {
+			if index > 0 {
+
+				// Early exit
+				if currentNode.Next[0] != nil && bytes.Compare(currentNode.Next[0].Key, key) == 0 {
+					prevElementIfVisited = currentNode
+					var currentNodeNext *SkipListElement
+					currentNodeNext, err = t.LoadElement(currentNode.Next[0])
+					if err != nil {
+						return
+					}
+					if currentNodeNext == nil {
+						return
+					}
+					foundElem = currentNodeNext
+					ok = true
+					return
+				}
+				// Go down
+				index--
+			} else {
+				// Element is not found and we reached the bottom.
+				if findGreaterOrEqual {
+					foundElem, err = t.LoadElement(currentNode.Next[index])
+					if err != nil {
+						return
+					}
+					ok = foundElem != nil
+				}
+
+				return
+			}
+		}
+	}
+}
+
+// Find tries to find an element in the skiplist based on the key from the given ListElement.
+// elem can be used, if ok is true.
+// Find runs in approx. O(log(n))
+func (t *SkipList) Find(key []byte) (prevIfVisited *SkipListElement, elem *SkipListElement, ok bool, err error) {
+
+	if t == nil || key == nil {
+		return
+	}
+
+	prevIfVisited, elem, ok, err = t.findExtended(key, false)
+	return
+}
+
+// FindGreaterOrEqual finds the first element, that is greater or equal to the given ListElement e.
+// The comparison is done on the keys (So on ExtractKey()).
+// FindGreaterOrEqual runs in approx. O(log(n))
+func (t *SkipList) FindGreaterOrEqual(key []byte) (prevIfVisited *SkipListElement, elem *SkipListElement, ok bool, err error) {
+
+	if t == nil || key == nil {
+		return
+	}
+
+	prevIfVisited, elem, ok, err = t.findExtended(key, true)
+	return
+}
+
+// Delete removes an element equal to e from the skiplist, if there is one.
+// If there are multiple entries with the same value, Delete will remove one of them
+// (Which one will change based on the actual skiplist layout)
+// Delete runs in approx. O(log(n))
+func (t *SkipList) DeleteByKey(key []byte) (id int64, err error) {
+
+	if t == nil || t.IsEmpty() || key == nil {
+		return
+	}
+
+	index := t.findEntryIndex(key, t.MaxLevel)
+
+	var currentNode *SkipListElement
+	var nextNode *SkipListElement
+
+	for {
+
+		if currentNode == nil {
+			nextNode, err = t.LoadElement(t.StartLevels[index])
+		} else {
+			nextNode, err = t.LoadElement(currentNode.Next[index])
+		}
+		if err != nil {
+			return id, err
+		}
+
+		// Found and remove!
+		if nextNode != nil && compareElement(nextNode, key) == 0 {
+
+			if currentNode != nil {
+				currentNode.Next[index] = nextNode.Next[index]
+				if err = t.SaveElement(currentNode); err != nil {
+					return id, err
+				}
+			}
+
+			if index == 0 {
+				if nextNode.Next[index] != nil {
+					nextNextNode, err := t.LoadElement(nextNode.Next[index])
+					if err != nil {
+						return id, err
+					}
+					if nextNextNode != nil {
+						nextNextNode.Prev = currentNode.Reference()
+						if err = t.SaveElement(nextNextNode); err != nil {
+							return id, err
+						}
+					}
+				}
+				// t.elementCount--
+				id = nextNode.Id
+				if err = t.DeleteElement(nextNode); err != nil {
+					return id, err
+				}
+			}
+
+			// Match by id, not the cached key: redis3 re-keys nodes in place,
+			// so StartLevels[index].Key can be stale and skip this update while
+			// the end readjustment below still clears EndLevels[index].
+			if t.StartLevels[index] != nil && t.StartLevels[index].ElementPointer == nextNode.Id {
+				t.HasChanges = true
+				t.StartLevels[index] = nextNode.Next[index]
+				// This was our currently highest node!
+				if t.StartLevels[index] == nil {
+					t.MaxLevel = index - 1
+				}
+			}
+
+			// Link from end needs readjustments.
+			if nextNode.Next[index] == nil {
+				t.EndLevels[index] = currentNode.Reference()
+				t.HasChanges = true
+			}
+			nextNode.Next[index] = nil
+		}
+
+		if nextNode != nil && compareElement(nextNode, key) < 0 {
+			// Go right
+			currentNode = nextNode
+		} else {
+			// Go down
+			index--
+			if index < 0 {
+				break
+			}
+		}
+	}
+	return
+}
+
+// Insert inserts the given ListElement into the skiplist.
+// Insert runs in approx. O(log(n))
+func (t *SkipList) InsertByKey(key []byte, idIfKnown int64, value []byte) (id int64, err error) {
+
+	if t == nil || key == nil {
+		return
+	}
+
+	level := t.generateLevel(t.MaxNewLevel)
+
+	// Only grow the height of the skiplist by one at a time!
+	if level > t.MaxLevel {
+		level = t.MaxLevel + 1
+		t.MaxLevel = level
+		t.HasChanges = true
+	}
+
+	id = idIfKnown
+	if id == 0 {
+		id = rand.Int63()
+	}
+	elem := &SkipListElement{
+		Id:    id,
+		Next:  make([]*SkipListElementReference, t.MaxNewLevel, t.MaxNewLevel),
+		Level: int32(level),
+		Key:   key,
+		Value: value,
+	}
+
+	// t.elementCount++
+
+	newFirst := true
+	newLast := true
+	if !t.IsEmpty() {
+		// Guard each end: a persisted skiplist may have one end nil, so
+		// self-heal instead of dereferencing it.
+		if t.StartLevels[0] != nil {
+			newFirst = compareElement(elem, t.StartLevels[0].Key) < 0
+		}
+		if t.EndLevels[0] != nil {
+			newLast = compareElement(elem, t.EndLevels[0].Key) > 0
+		}
+	}
+
+	normallyInserted := false
+	if !newFirst && !newLast {
+
+		normallyInserted = true
+
+		index := t.findEntryIndex(key, level)
+
+		var currentNode *SkipListElement
+		var nextNodeRef *SkipListElementReference
+
+		for {
+
+			if currentNode == nil {
+				nextNodeRef = t.StartLevels[index]
+			} else {
+				nextNodeRef = currentNode.Next[index]
+			}
+
+			var nextNode *SkipListElement
+
+			// Connect node to next
+			if index <= level && (nextNodeRef == nil || bytes.Compare(nextNodeRef.Key, key) > 0) {
+				elem.Next[index] = nextNodeRef
+				if currentNode != nil {
+					currentNode.Next[index] = elem.Reference()
+					if err = t.SaveElement(currentNode); err != nil {
+						return
+					}
+				}
+				if index == 0 {
+					elem.Prev = currentNode.Reference()
+					if nextNodeRef != nil {
+						if nextNode, err = t.LoadElement(nextNodeRef); err != nil {
+							return
+						}
+						if nextNode != nil {
+							nextNode.Prev = elem.Reference()
+							if err = t.SaveElement(nextNode); err != nil {
+								return
+							}
+						}
+					}
+				}
+			}
+
+			if nextNodeRef != nil && bytes.Compare(nextNodeRef.Key, key) <= 0 {
+				// Go right
+				if nextNode == nil {
+					// reuse nextNode when index == 0
+					if nextNode, err = t.LoadElement(nextNodeRef); err != nil {
+						return
+					}
+				}
+				currentNode = nextNode
+				if currentNode == nil {
+					return
+				}
+			} else {
+				// Go down
+				index--
+				if index < 0 {
+					break
+				}
+			}
+		}
+	}
+
+	// Where we have a left-most position that needs to be referenced!
+	for i := level; i >= 0; i-- {
+
+		didSomething := false
+
+		if newFirst || normallyInserted {
+
+			if t.StartLevels[i] == nil || bytes.Compare(t.StartLevels[i].Key, key) > 0 {
+				if i == 0 && t.StartLevels[i] != nil {
+					startLevelElement, err := t.LoadElement(t.StartLevels[i])
+					if err != nil {
+						return id, err
+					}
+					if startLevelElement != nil {
+						startLevelElement.Prev = elem.Reference()
+						if err = t.SaveElement(startLevelElement); err != nil {
+							return id, err
+						}
+					}
+				}
+				elem.Next[i] = t.StartLevels[i]
+				t.StartLevels[i] = elem.Reference()
+				t.HasChanges = true
+			}
+
+			// link the EndLevels to this element!
+			if elem.Next[i] == nil {
+				t.EndLevels[i] = elem.Reference()
+				t.HasChanges = true
+			}
+
+			didSomething = true
+		}
+
+		if newLast {
+			// Places the element after the very last element on this level!
+			// This is very important, so we are not linking the very first element (newFirst AND newLast) to itself!
+			if !newFirst {
+				if t.EndLevels[i] != nil {
+					endLevelElement, err := t.LoadElement(t.EndLevels[i])
+					if err != nil {
+						return id, err
+					}
+					if endLevelElement != nil {
+						endLevelElement.Next[i] = elem.Reference()
+						if err = t.SaveElement(endLevelElement); err != nil {
+							return id, err
+						}
+					}
+				}
+				if i == 0 {
+					elem.Prev = t.EndLevels[i]
+				}
+				t.EndLevels[i] = elem.Reference()
+				t.HasChanges = true
+			}
+
+			// Link the startLevels to this element!
+			if t.StartLevels[i] == nil || bytes.Compare(t.StartLevels[i].Key, key) > 0 {
+				t.StartLevels[i] = elem.Reference()
+				t.HasChanges = true
+			}
+
+			didSomething = true
+		}
+
+		if !didSomething {
+			break
+		}
+	}
+
+	if err = t.SaveElement(elem); err != nil {
+		return id, err
+	}
+	return id, nil
+
+}
+
+// GetSmallestNode returns the very first/smallest node in the skiplist.
+// GetSmallestNode runs in O(1)
+func (t *SkipList) GetSmallestNode() (*SkipListElement, error) {
+	return t.LoadElement(t.StartLevels[0])
+}
+
+// GetLargestNode returns the very last/largest node in the skiplist.
+// GetLargestNode runs in O(1)
+func (t *SkipList) GetLargestNode() (*SkipListElement, error) {
+	return t.LoadElement(t.EndLevels[0])
+}
+func (t *SkipList) GetLargestNodeReference() *SkipListElementReference {
+	return t.EndLevels[0]
+}
+
+// Next returns the next element based on the given node.
+// Next will loop around to the first node, if you call it on the last!
+func (t *SkipList) Next(e *SkipListElement) (*SkipListElement, error) {
+	if e.Next[0] == nil {
+		return t.LoadElement(t.StartLevels[0])
+	}
+	return t.LoadElement(e.Next[0])
+}
+
+// Prev returns the previous element based on the given node.
+// Prev will loop around to the last node, if you call it on the first!
+func (t *SkipList) Prev(e *SkipListElement) (*SkipListElement, error) {
+	if e.Prev == nil {
+		return t.LoadElement(t.EndLevels[0])
+	}
+	return t.LoadElement(e.Prev)
+}
+
+// ChangeValue can be used to change the actual value of a node in the skiplist
+// without the need of Deleting and reinserting the node again.
+// Be advised, that ChangeValue only works, if the actual key from ExtractKey() will stay the same!
+// ok is an indicator, whether the value is actually changed.
+func (t *SkipList) ChangeValue(e *SkipListElement, newValue []byte) (err error) {
+	// The key needs to stay correct, so this is very important!
+	e.Value = newValue
+	return t.SaveElement(e)
+}
+
+// String returns a string format of the skiplist. Useful to get a graphical overview and/or debugging.
+func (t *SkipList) println() {
+
+	print("start --> ")
+	for i, l := range t.StartLevels {
+		if l == nil {
+			break
+		}
+		if i > 0 {
+			print(" -> ")
+		}
+		next := "---"
+		if l != nil {
+			next = string(l.Key)
+		}
+		print(fmt.Sprintf("[%v]", next))
+	}
+	println()
+
+	nodeRef := t.StartLevels[0]
+	for nodeRef != nil {
+		print(fmt.Sprintf("%v: ", string(nodeRef.Key)))
+		node, _ := t.LoadElement(nodeRef)
+		if node == nil {
+			break
+		}
+		for i := 0; i <= int(node.Level); i++ {
+
+			l := node.Next[i]
+
+			next := "---"
+			if l != nil {
+				next = string(l.Key)
+			}
+
+			if i == 0 {
+				prev := "---"
+
+				if node.Prev != nil {
+					prev = string(node.Prev.Key)
+				}
+				print(fmt.Sprintf("[%v|%v]", prev, next))
+			} else {
+				print(fmt.Sprintf("[%v]", next))
+			}
+			if i < int(node.Level) {
+				print(" -> ")
+			}
+
+		}
+		nodeRef = node.Next[0]
+		println()
+	}
+
+	print("end --> ")
+	for i, l := range t.EndLevels {
+		if l == nil {
+			break
+		}
+		if i > 0 {
+			print(" -> ")
+		}
+		next := "---"
+		if l != nil {
+			next = string(l.Key)
+		}
+		print(fmt.Sprintf("[%v]", next))
+	}
+	println()
+}

weed/util/skiplist/skiplist.pb.go

@@ -0,0 +1,302 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.6
+// 	protoc        v6.33.4
+// source: skiplist.proto
+
+package skiplist
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type SkipListProto struct {
+	state         protoimpl.MessageState      `protogen:"open.v1"`
+	StartLevels   []*SkipListElementReference `protobuf:"bytes,1,rep,name=start_levels,json=startLevels,proto3" json:"start_levels,omitempty"`
+	EndLevels     []*SkipListElementReference `protobuf:"bytes,2,rep,name=end_levels,json=endLevels,proto3" json:"end_levels,omitempty"`
+	MaxNewLevel   int32                       `protobuf:"varint,3,opt,name=max_new_level,json=maxNewLevel,proto3" json:"max_new_level,omitempty"`
+	MaxLevel      int32                       `protobuf:"varint,4,opt,name=max_level,json=maxLevel,proto3" json:"max_level,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *SkipListProto) Reset() {
+	*x = SkipListProto{}
+	mi := &file_skiplist_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SkipListProto) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SkipListProto) ProtoMessage() {}
+
+func (x *SkipListProto) ProtoReflect() protoreflect.Message {
+	mi := &file_skiplist_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SkipListProto.ProtoReflect.Descriptor instead.
+func (*SkipListProto) Descriptor() ([]byte, []int) {
+	return file_skiplist_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *SkipListProto) GetStartLevels() []*SkipListElementReference {
+	if x != nil {
+		return x.StartLevels
+	}
+	return nil
+}
+
+func (x *SkipListProto) GetEndLevels() []*SkipListElementReference {
+	if x != nil {
+		return x.EndLevels
+	}
+	return nil
+}
+
+func (x *SkipListProto) GetMaxNewLevel() int32 {
+	if x != nil {
+		return x.MaxNewLevel
+	}
+	return 0
+}
+
+func (x *SkipListProto) GetMaxLevel() int32 {
+	if x != nil {
+		return x.MaxLevel
+	}
+	return 0
+}
+
+type SkipListElementReference struct {
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	ElementPointer int64                  `protobuf:"varint,1,opt,name=element_pointer,json=elementPointer,proto3" json:"element_pointer,omitempty"`
+	Key            []byte                 `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
+}
+
+func (x *SkipListElementReference) Reset() {
+	*x = SkipListElementReference{}
+	mi := &file_skiplist_proto_msgTypes[1]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SkipListElementReference) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SkipListElementReference) ProtoMessage() {}
+
+func (x *SkipListElementReference) ProtoReflect() protoreflect.Message {
+	mi := &file_skiplist_proto_msgTypes[1]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SkipListElementReference.ProtoReflect.Descriptor instead.
+func (*SkipListElementReference) Descriptor() ([]byte, []int) {
+	return file_skiplist_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *SkipListElementReference) GetElementPointer() int64 {
+	if x != nil {
+		return x.ElementPointer
+	}
+	return 0
+}
+
+func (x *SkipListElementReference) GetKey() []byte {
+	if x != nil {
+		return x.Key
+	}
+	return nil
+}
+
+type SkipListElement struct {
+	state         protoimpl.MessageState      `protogen:"open.v1"`
+	Id            int64                       `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"`
+	Next          []*SkipListElementReference `protobuf:"bytes,2,rep,name=next,proto3" json:"next,omitempty"`
+	Level         int32                       `protobuf:"varint,3,opt,name=level,proto3" json:"level,omitempty"`
+	Key           []byte                      `protobuf:"bytes,4,opt,name=key,proto3" json:"key,omitempty"`
+	Value         []byte                      `protobuf:"bytes,5,opt,name=value,proto3" json:"value,omitempty"`
+	Prev          *SkipListElementReference   `protobuf:"bytes,6,opt,name=prev,proto3" json:"prev,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *SkipListElement) Reset() {
+	*x = SkipListElement{}
+	mi := &file_skiplist_proto_msgTypes[2]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SkipListElement) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SkipListElement) ProtoMessage() {}
+
+func (x *SkipListElement) ProtoReflect() protoreflect.Message {
+	mi := &file_skiplist_proto_msgTypes[2]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SkipListElement.ProtoReflect.Descriptor instead.
+func (*SkipListElement) Descriptor() ([]byte, []int) {
+	return file_skiplist_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *SkipListElement) GetId() int64 {
+	if x != nil {
+		return x.Id
+	}
+	return 0
+}
+
+func (x *SkipListElement) GetNext() []*SkipListElementReference {
+	if x != nil {
+		return x.Next
+	}
+	return nil
+}
+
+func (x *SkipListElement) GetLevel() int32 {
+	if x != nil {
+		return x.Level
+	}
+	return 0
+}
+
+func (x *SkipListElement) GetKey() []byte {
+	if x != nil {
+		return x.Key
+	}
+	return nil
+}
+
+func (x *SkipListElement) GetValue() []byte {
+	if x != nil {
+		return x.Value
+	}
+	return nil
+}
+
+func (x *SkipListElement) GetPrev() *SkipListElementReference {
+	if x != nil {
+		return x.Prev
+	}
+	return nil
+}
+
+var File_skiplist_proto protoreflect.FileDescriptor
+
+const file_skiplist_proto_rawDesc = "" +
+	"\n" +
+	"\x0eskiplist.proto\x12\bskiplist\"\xda\x01\n" +
+	"\rSkipListProto\x12E\n" +
+	"\fstart_levels\x18\x01 \x03(\v2\".skiplist.SkipListElementReferenceR\vstartLevels\x12A\n" +
+	"\n" +
+	"end_levels\x18\x02 \x03(\v2\".skiplist.SkipListElementReferenceR\tendLevels\x12\"\n" +
+	"\rmax_new_level\x18\x03 \x01(\x05R\vmaxNewLevel\x12\x1b\n" +
+	"\tmax_level\x18\x04 \x01(\x05R\bmaxLevel\"U\n" +
+	"\x18SkipListElementReference\x12'\n" +
+	"\x0felement_pointer\x18\x01 \x01(\x03R\x0eelementPointer\x12\x10\n" +
+	"\x03key\x18\x02 \x01(\fR\x03key\"\xcf\x01\n" +
+	"\x0fSkipListElement\x12\x0e\n" +
+	"\x02id\x18\x01 \x01(\x03R\x02id\x126\n" +
+	"\x04next\x18\x02 \x03(\v2\".skiplist.SkipListElementReferenceR\x04next\x12\x14\n" +
+	"\x05level\x18\x03 \x01(\x05R\x05level\x12\x10\n" +
+	"\x03key\x18\x04 \x01(\fR\x03key\x12\x14\n" +
+	"\x05value\x18\x05 \x01(\fR\x05value\x126\n" +
+	"\x04prev\x18\x06 \x01(\v2\".skiplist.SkipListElementReferenceR\x04prevB3Z1github.com/seaweedfs/seaweedfs/weed/util/skiplistb\x06proto3"
+
+var (
+	file_skiplist_proto_rawDescOnce sync.Once
+	file_skiplist_proto_rawDescData []byte
+)
+
+func file_skiplist_proto_rawDescGZIP() []byte {
+	file_skiplist_proto_rawDescOnce.Do(func() {
+		file_skiplist_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_skiplist_proto_rawDesc), len(file_skiplist_proto_rawDesc)))
+	})
+	return file_skiplist_proto_rawDescData
+}
+
+var file_skiplist_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
+var file_skiplist_proto_goTypes = []any{
+	(*SkipListProto)(nil),            // 0: skiplist.SkipListProto
+	(*SkipListElementReference)(nil), // 1: skiplist.SkipListElementReference
+	(*SkipListElement)(nil),          // 2: skiplist.SkipListElement
+}
+var file_skiplist_proto_depIdxs = []int32{
+	1, // 0: skiplist.SkipListProto.start_levels:type_name -> skiplist.SkipListElementReference
+	1, // 1: skiplist.SkipListProto.end_levels:type_name -> skiplist.SkipListElementReference
+	1, // 2: skiplist.SkipListElement.next:type_name -> skiplist.SkipListElementReference
+	1, // 3: skiplist.SkipListElement.prev:type_name -> skiplist.SkipListElementReference
+	4, // [4:4] is the sub-list for method output_type
+	4, // [4:4] is the sub-list for method input_type
+	4, // [4:4] is the sub-list for extension type_name
+	4, // [4:4] is the sub-list for extension extendee
+	0, // [0:4] is the sub-list for field type_name
+}
+
+func init() { file_skiplist_proto_init() }
+func file_skiplist_proto_init() {
+	if File_skiplist_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_skiplist_proto_rawDesc), len(file_skiplist_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   3,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_skiplist_proto_goTypes,
+		DependencyIndexes: file_skiplist_proto_depIdxs,
+		MessageInfos:      file_skiplist_proto_msgTypes,
+	}.Build()
+	File_skiplist_proto = out.File
+	file_skiplist_proto_goTypes = nil
+	file_skiplist_proto_depIdxs = nil
+}

weed/util/skiplist/skiplist.proto

@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package skiplist;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/util/skiplist";
+
+message SkipListProto {
+  repeated SkipListElementReference start_levels = 1;
+  repeated SkipListElementReference end_levels = 2;
+  int32 max_new_level = 3;
+  int32 max_level = 4;
+}
+
+message SkipListElementReference {
+  int64 element_pointer = 1;
+  bytes key = 2;
+}
+
+message SkipListElement {
+  int64 id = 1;
+  repeated SkipListElementReference next = 2;
+  int32 level = 3;
+  bytes key = 4;
+  bytes value = 5;
+  SkipListElementReference prev = 6;
+}

weed/util/skiplist/skiplist_serde.go

@@ -0,0 +1,51 @@
+package skiplist
+
+import "bytes"
+
+func compareElement(a *SkipListElement, key []byte) int {
+	if len(a.Key) == 0 {
+		return -1
+	}
+	return bytes.Compare(a.Key, key)
+}
+
+func (node *SkipListElement) Reference() *SkipListElementReference {
+	if node == nil {
+		return nil
+	}
+	return &SkipListElementReference{
+		ElementPointer: node.Id,
+		Key:            node.Key,
+	}
+}
+
+func (t *SkipList) SaveElement(element *SkipListElement) error {
+	if element == nil {
+		return nil
+	}
+	return t.ListStore.SaveElement(element.Id, element)
+}
+
+func (t *SkipList) DeleteElement(element *SkipListElement) error {
+	if element == nil {
+		return nil
+	}
+	return t.ListStore.DeleteElement(element.Id)
+}
+
+func (t *SkipList) LoadElement(ref *SkipListElementReference) (*SkipListElement, error) {
+	if ref.IsNil() {
+		return nil, nil
+	}
+	return t.ListStore.LoadElement(ref.ElementPointer)
+}
+
+func (ref *SkipListElementReference) IsNil() bool {
+	if ref == nil {
+		return true
+	}
+	if len(ref.Key) == 0 {
+		return true
+	}
+	return false
+}

weed/util/skiplist/skiplist_test.go

@@ -0,0 +1,332 @@
+package skiplist
+
+import (
+	"bytes"
+	"math/rand/v2"
+	"strconv"
+	"testing"
+)
+
+const (
+	maxN = 10000
+)
+
+var (
+	memStore = newMemStore()
+)
+
+func TestReverseInsert(t *testing.T) {
+	list := NewSeed(100, memStore)
+
+	list.InsertByKey([]byte("zzz"), 0, []byte("zzz"))
+	list.DeleteByKey([]byte("zzz"))
+
+	list.InsertByKey([]byte("aaa"), 0, []byte("aaa"))
+
+	if list.IsEmpty() {
+		t.Fail()
+	}
+
+}
+
+// TestDeleteRekeyedNode reproduces the redis3 filer crash: a node re-keyed in
+// place leaves a stale StartLevels key, so deleting it must still clear both
+// ends together rather than leaving EndLevels[0] nil for the next insert.
+func TestDeleteRekeyedNode(t *testing.T) {
+	list := New(memStore)
+
+	list.InsertByKey([]byte("b"), 1, []byte("b"))
+
+	// Drifted start reference: same element id, stale cached key.
+	list.StartLevels[0] = &SkipListElementReference{ElementPointer: 1, Key: []byte("stale")}
+
+	if _, err := list.DeleteByKey([]byte("b")); err != nil {
+		t.Fatal(err)
+	}
+
+	if (list.StartLevels[0] == nil) != (list.EndLevels[0] == nil) {
+		t.Fatalf("inconsistent ends after delete: start=%v end=%v", list.StartLevels[0], list.EndLevels[0])
+	}
+
+	if _, err := list.InsertByKey([]byte("c"), 2, []byte("c")); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, ok, _ := list.Find([]byte("c")); !ok {
+		t.Fatal("expected to find inserted key")
+	}
+}
+
+// TestInsertByKeyRecoversInconsistentEnds: a persisted skiplist with
+// StartLevels[0] set but EndLevels[0] nil must self-heal, not crash on load.
+func TestInsertByKeyRecoversInconsistentEnds(t *testing.T) {
+	list := New(memStore)
+	list.InsertByKey([]byte("m"), 1, []byte("m"))
+
+	list.EndLevels[0] = nil
+
+	if _, err := list.InsertByKey([]byte("n"), 2, []byte("n")); err != nil {
+		t.Fatal(err)
+	}
+	if list.IsEmpty() {
+		t.Fatal("list should not be empty")
+	}
+}
+
+func TestInsertAndFind(t *testing.T) {
+
+	k0 := []byte("0")
+	var list *SkipList
+
+	var listPointer *SkipList
+	listPointer.InsertByKey(k0, 0, k0)
+	if _, _, ok, _ := listPointer.Find(k0); ok {
+		t.Fail()
+	}
+
+	list = New(memStore)
+	if _, _, ok, _ := list.Find(k0); ok {
+		t.Fail()
+	}
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+
+	// Test at the beginning of the list.
+	for i := 0; i < maxN; i++ {
+		key := []byte(strconv.Itoa(maxN - i))
+		list.InsertByKey(key, 0, key)
+	}
+	for i := 0; i < maxN; i++ {
+		key := []byte(strconv.Itoa(maxN - i))
+		if _, _, ok, _ := list.Find(key); !ok {
+			t.Fail()
+		}
+	}
+
+	list = New(memStore)
+	// Test at the end of the list.
+	for i := 0; i < maxN; i++ {
+		key := []byte(strconv.Itoa(i))
+		list.InsertByKey(key, 0, key)
+	}
+	for i := 0; i < maxN; i++ {
+		key := []byte(strconv.Itoa(i))
+		if _, _, ok, _ := list.Find(key); !ok {
+			t.Fail()
+		}
+	}
+
+	list = New(memStore)
+	// Test at random positions in the list.
+	rList := rand.Perm(maxN)
+	for _, e := range rList {
+		key := []byte(strconv.Itoa(e))
+		// println("insert", e)
+		list.InsertByKey(key, 0, key)
+	}
+	for _, e := range rList {
+		key := []byte(strconv.Itoa(e))
+		// println("find", e)
+		if _, _, ok, _ := list.Find(key); !ok {
+			t.Fail()
+		}
+	}
+	// println("print list")
+	// list.println()
+
+}
+
+func Element(x int) []byte {
+	return []byte(strconv.Itoa(x))
+}
+
+func TestDelete(t *testing.T) {
+
+	k0 := []byte("0")
+
+	var list *SkipList
+
+	// Delete on empty list
+	list.DeleteByKey(k0)
+
+	list = New(memStore)
+
+	list.DeleteByKey(k0)
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+
+	list.InsertByKey(k0, 0, k0)
+	list.DeleteByKey(k0)
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+
+	// Delete elements at the beginning of the list.
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(i), 0, Element(i))
+	}
+	for i := 0; i < maxN; i++ {
+		list.DeleteByKey(Element(i))
+	}
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+
+	list = New(memStore)
+	// Delete elements at the end of the list.
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(i), 0, Element(i))
+	}
+	for i := 0; i < maxN; i++ {
+		list.DeleteByKey(Element(maxN - i - 1))
+	}
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+
+	list = New(memStore)
+	// Delete elements at random positions in the list.
+	rList := rand.Perm(maxN)
+	for _, e := range rList {
+		list.InsertByKey(Element(e), 0, Element(e))
+	}
+	for _, e := range rList {
+		list.DeleteByKey(Element(e))
+	}
+	if !list.IsEmpty() {
+		t.Fail()
+	}
+}
+
+func TestNext(t *testing.T) {
+	list := New(memStore)
+
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(i), 0, Element(i))
+	}
+
+	smallest, _ := list.GetSmallestNode()
+	largest, _ := list.GetLargestNode()
+
+	lastNode := smallest
+	node := lastNode
+	for node != largest {
+		node, _ = list.Next(node)
+		// Must always be incrementing here!
+		if bytes.Compare(node.Key, lastNode.Key) <= 0 {
+			t.Fail()
+		}
+		// Next.Prev must always point to itself!
+		prevNode, _ := list.Prev(node)
+		nextNode, _ := list.Next(prevNode)
+		if nextNode != node {
+			t.Fail()
+		}
+		lastNode = node
+	}
+
+	if nextNode, _ := list.Next(largest); nextNode != smallest {
+		t.Fail()
+	}
+}
+
+func TestPrev(t *testing.T) {
+	list := New(memStore)
+
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(i), 0, Element(i))
+	}
+
+	smallest, _ := list.GetSmallestNode()
+	largest, _ := list.GetLargestNode()
+
+	lastNode := largest
+	node := lastNode
+	for node != smallest {
+		node, _ = list.Prev(node)
+		// Must always be incrementing here!
+		if bytes.Compare(node.Key, lastNode.Key) >= 0 {
+			t.Fail()
+		}
+		// Next.Prev must always point to itself!
+		nextNode, _ := list.Next(node)
+		prevNode, _ := list.Prev(nextNode)
+		if prevNode != node {
+			t.Fail()
+		}
+		lastNode = node
+	}
+
+	if prevNode, _ := list.Prev(smallest); prevNode != largest {
+		t.Fail()
+	}
+}
+
+func TestFindGreaterOrEqual(t *testing.T) {
+
+	maxNumber := maxN * 100
+
+	var list *SkipList
+	var listPointer *SkipList
+
+	// Test on empty list.
+	if _, _, ok, _ := listPointer.FindGreaterOrEqual(Element(0)); ok {
+		t.Errorf("found element 0 in an empty list")
+	}
+
+	list = New(memStore)
+
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(rand.IntN(maxNumber)), 0, Element(i))
+	}
+
+	for i := 0; i < maxN; i++ {
+		key := Element(rand.IntN(maxNumber))
+		if _, v, ok, _ := list.FindGreaterOrEqual(key); ok {
+			// if f is v should be bigger than the element before
+			if v.Prev != nil && bytes.Compare(key, v.Prev.Key) < 0 {
+				t.Errorf("PrevV: %s\n    key: %s\n\n", string(v.Prev.Key), string(key))
+			}
+			// v should be bigger or equal to f
+			// If we compare directly, we get an equal key with a difference on the 10th decimal point, which fails.
+			if bytes.Compare(v.Key, key) < 0 {
+				t.Errorf("v: %s\n    key: %s\n\n", string(v.Key), string(key))
+			}
+		} else {
+			lastNode, _ := list.GetLargestNode()
+			// It is OK, to fail, as long as the key is bigger than the largest key.
+			if bytes.Compare(key, lastNode.Key) <= 0 {
+				t.Errorf("largest key: %s\n    key: %s\n\n", string(lastNode.Key), string(key))
+			}
+		}
+	}
+
+}
+
+func TestChangeValue(t *testing.T) {
+	list := New(memStore)
+
+	for i := 0; i < maxN; i++ {
+		list.InsertByKey(Element(i), 0, []byte("value"))
+	}
+
+	for i := 0; i < maxN; i++ {
+		// The key only looks at the int so the string doesn't matter here!
+		_, f1, ok, _ := list.Find(Element(i))
+		if !ok {
+			t.Fail()
+		}
+		err := list.ChangeValue(f1, []byte("different value"))
+		if err != nil {
+			t.Fail()
+		}
+		_, f2, ok, _ := list.Find(Element(i))
+		if !ok {
+			t.Fail()
+		}
+		if bytes.Compare(f2.GetValue(), []byte("different value")) != 0 {
+			t.Fail()
+		}
+	}
+}