snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/cluster/admin_locks.go

@@ -0,0 +1,6 @@
+package cluster
+
+const (
+	AdminShellLockName          = "shell"
+	AdminServerPresenceLockName = "admin-server"
+)

weed/cluster/cluster.go

@@ -0,0 +1,193 @@
+package cluster
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+const (
+	MasterType       = "master"
+	VolumeServerType = "volumeServer"
+	FilerType        = "filer"
+	BrokerType       = "broker"
+	S3Type           = "s3"
+)
+
+type FilerGroupName string
+type DataCenter string
+type Rack string
+
+type ClusterNode struct {
+	Address    pb.ServerAddress
+	Version    string
+	counter    int
+	CreatedTs  time.Time
+	DataCenter DataCenter
+	Rack       Rack
+}
+
+type ClusterNodeGroups struct {
+	groupMembers map[FilerGroupName]*GroupMembers
+	sync.RWMutex
+}
+type Cluster struct {
+	filerGroups  *ClusterNodeGroups
+	brokerGroups *ClusterNodeGroups
+	s3Groups     *ClusterNodeGroups
+}
+
+func newClusterNodeGroups() *ClusterNodeGroups {
+	return &ClusterNodeGroups{
+		groupMembers: map[FilerGroupName]*GroupMembers{},
+	}
+}
+func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
+	members, found := g.groupMembers[filerGroup]
+	if !found && createIfNotFound {
+		members = newGroupMembers()
+		g.groupMembers[filerGroup] = members
+	}
+	return members
+}
+
+func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	g.Lock()
+	defer g.Unlock()
+	m := g.getGroupMembers(filerGroup, true)
+	if t := m.addMember(dataCenter, rack, address, version); t != nil {
+		return buildClusterNodeUpdateMessage(true, filerGroup, nodeType, address)
+	}
+	return nil
+}
+func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	g.Lock()
+	defer g.Unlock()
+	m := g.getGroupMembers(filerGroup, false)
+	if m == nil {
+		return nil
+	}
+	if m.removeMember(address) {
+		return buildClusterNodeUpdateMessage(false, filerGroup, nodeType, address)
+	}
+	return nil
+}
+func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
+	g.Lock()
+	defer g.Unlock()
+	m := g.getGroupMembers(filerGroup, false)
+	if m == nil {
+		return nil
+	}
+	for _, node := range m.members {
+		nodes = append(nodes, node)
+	}
+	return
+}
+
+func NewCluster() *Cluster {
+	return &Cluster{
+		filerGroups:  newClusterNodeGroups(),
+		brokerGroups: newClusterNodeGroups(),
+		s3Groups:     newClusterNodeGroups(),
+	}
+}
+
+func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
+	filerGroup := FilerGroupName(ns)
+	switch nodeType {
+	case FilerType:
+		return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
+	case BrokerType:
+		return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
+	case S3Type:
+		return cluster.s3Groups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
+	case MasterType:
+		return buildClusterNodeUpdateMessage(true, filerGroup, nodeType, address)
+	}
+	return nil
+}
+
+func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
+	filerGroup := FilerGroupName(ns)
+	switch nodeType {
+	case FilerType:
+		return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
+	case BrokerType:
+		return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
+	case S3Type:
+		return cluster.s3Groups.RemoveClusterNode(filerGroup, nodeType, address)
+	case MasterType:
+		return buildClusterNodeUpdateMessage(false, filerGroup, nodeType, address)
+	}
+	return nil
+}
+
+func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
+	switch nodeType {
+	case FilerType:
+		return cluster.filerGroups.ListClusterNode(filerGroup)
+	case BrokerType:
+		return cluster.brokerGroups.ListClusterNode(filerGroup)
+	case S3Type:
+		return cluster.s3Groups.ListClusterNode(filerGroup)
+	case MasterType:
+	}
+	return
+}
+
+// ListClusterNodeUpdates reports the current members as add updates, so a
+// client that just connected can rebuild the membership it missed while it was
+// away.
+func (cluster *Cluster) ListClusterNodeUpdates(filerGroup FilerGroupName, nodeType string) (updates []*master_pb.KeepConnectedResponse) {
+	for _, node := range cluster.ListClusterNode(filerGroup, nodeType) {
+		updates = append(updates, buildClusterNodeUpdateMessage(true, filerGroup, nodeType, node.Address)...)
+	}
+	return
+}
+
+// IsKnownNode reports whether address is currently registered under nodeType
+// in any filer group. The lookup is intentionally group-agnostic because callers
+// (e.g. Ping admission) only know the target address, not the group it joined.
+func (cluster *Cluster) IsKnownNode(nodeType string, address pb.ServerAddress) bool {
+	var groups *ClusterNodeGroups
+	switch nodeType {
+	case FilerType:
+		groups = cluster.filerGroups
+	case BrokerType:
+		groups = cluster.brokerGroups
+	case S3Type:
+		groups = cluster.s3Groups
+	default:
+		return false
+	}
+	groups.RLock()
+	defer groups.RUnlock()
+	for _, members := range groups.groupMembers {
+		if _, found := members.members[address]; found {
+			return true
+		}
+		// fall back to a port-tolerant comparison so callers that omit the
+		// grpc-port suffix still match a registered peer
+		for stored := range members.members {
+			if stored.Equals(address) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func buildClusterNodeUpdateMessage(isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
+	result = append(result, &master_pb.KeepConnectedResponse{
+		ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
+			FilerGroup: string(filerGroup),
+			NodeType:   nodeType,
+			Address:    string(address),
+			IsAdd:      isAdd,
+		},
+	})
+	return
+}

weed/cluster/cluster_test.go

@@ -0,0 +1,87 @@
+package cluster
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func TestConcurrentAddRemoveNodes(t *testing.T) {
+	c := NewCluster()
+	var wg sync.WaitGroup
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			address := strconv.Itoa(i)
+			c.AddClusterNode("", "filer", "", "", pb.ServerAddress(address), "23.45")
+		}(i)
+	}
+	wg.Wait()
+
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			address := strconv.Itoa(i)
+			node := c.RemoveClusterNode("", "filer", pb.ServerAddress(address))
+
+			if len(node) == 0 {
+				t.Errorf("TestConcurrentAddRemoveNodes: node[%s] not found", address)
+				return
+			} else if node[0].ClusterNodeUpdate.Address != address {
+				t.Errorf("TestConcurrentAddRemoveNodes: expect:%s, actual:%s", address, node[0].ClusterNodeUpdate.Address)
+				return
+			}
+		}(i)
+	}
+	wg.Wait()
+}
+
+func TestListClusterNodeUpdates(t *testing.T) {
+	c := NewCluster()
+	filer := pb.ServerAddress("10.0.0.20:8888")
+	c.AddClusterNode("group", FilerType, "dc1", "rack1", filer, "test")
+	c.AddClusterNode("group", BrokerType, "dc1", "rack1", pb.ServerAddress("10.0.0.20:17777"), "test")
+
+	updates := c.ListClusterNodeUpdates("group", FilerType)
+	if len(updates) != 1 {
+		t.Fatalf("expecting one filer update, got %d", len(updates))
+	}
+	update := updates[0].ClusterNodeUpdate
+	if update.Address != string(filer) || !update.IsAdd || update.FilerGroup != "group" {
+		t.Fatalf("unexpected update %+v", update)
+	}
+
+	c.RemoveClusterNode("group", FilerType, filer)
+	if updates := c.ListClusterNodeUpdates("group", FilerType); len(updates) != 0 {
+		t.Fatalf("expecting no update for a removed filer, got %d", len(updates))
+	}
+}
+
+func TestIsKnownNode(t *testing.T) {
+	c := NewCluster()
+	filer := pb.ServerAddress("10.0.0.20:8888")
+	c.AddClusterNode("", FilerType, "dc1", "rack1", filer, "test")
+
+	if !c.IsKnownNode(FilerType, filer) {
+		t.Fatalf("registered filer %s should be known", filer)
+	}
+	if c.IsKnownNode(VolumeServerType, filer) {
+		t.Fatalf("filer address must not be accepted as a volume server target")
+	}
+	if c.IsKnownNode(FilerType, pb.ServerAddress("127.0.0.1:1")) {
+		t.Fatalf("unregistered low-port target must be rejected")
+	}
+	if c.IsKnownNode(FilerType, pb.ServerAddress("127.0.0.1:65000")) {
+		t.Fatalf("unregistered high-port target must be rejected")
+	}
+	if c.IsKnownNode(FilerType, pb.ServerAddress("example.com:443")) {
+		t.Fatalf("unrelated host must be rejected")
+	}
+	if c.IsKnownNode("garbage", filer) {
+		t.Fatalf("unknown node type must be rejected")
+	}
+}

weed/cluster/group_members.go

@@ -0,0 +1,53 @@
+package cluster
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+type GroupMembers struct {
+	members map[pb.ServerAddress]*ClusterNode
+}
+
+func newGroupMembers() *GroupMembers {
+	return &GroupMembers{
+		members: make(map[pb.ServerAddress]*ClusterNode),
+	}
+}
+
+func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
+	if existingNode, found := m.members[address]; found {
+		existingNode.counter++
+		return nil
+	}
+	t := &ClusterNode{
+		Address:    address,
+		Version:    version,
+		counter:    1,
+		CreatedTs:  time.Now(),
+		DataCenter: dataCenter,
+		Rack:       rack,
+	}
+	m.members[address] = t
+	return t
+}
+func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
+	if existingNode, found := m.members[address]; !found {
+		return false
+	} else {
+		existingNode.counter--
+		if existingNode.counter <= 0 {
+			delete(m.members, address)
+			return true
+		}
+	}
+	return false
+}
+
+func (m *GroupMembers) GetMembers() (addresses []pb.ServerAddress) {
+	for k := range m.members {
+		addresses = append(addresses, k)
+	}
+	return
+}

weed/cluster/lock_client.go

@@ -0,0 +1,436 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+)
+
+type LockClient struct {
+	grpcDialOption  grpc.DialOption
+	maxLockDuration time.Duration
+	sleepDuration   time.Duration
+	seedFiler       pb.ServerAddress
+
+	// ring is an optional client-side view of the filer lock hash ring. When
+	// populated, a new lock starts at the key's primary filer instead of the
+	// seed filer, avoiding the seed->primary forward hop. A stale view stays
+	// correct: the filer forwards to the real primary as a fallback.
+	ringMu      sync.RWMutex
+	ring        *lock_manager.HashRing
+	ringVersion int64
+
+	// priorRing is the ring before the most recent change, kept for priorWindow so a
+	// route-by-key caller can consult a just-moved key's previous owner during a
+	// rebalance. Mirrors the master's lock_manager.LockRing.PriorOwner cooling-off.
+	priorRing     *lock_manager.HashRing
+	ringChangedAt time.Time
+	priorWindow   time.Duration
+}
+
+func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
+	return &LockClient{
+		grpcDialOption:  grpcDialOption,
+		maxLockDuration: 5 * time.Second,
+		sleepDuration:   2473 * time.Millisecond,
+		seedFiler:       seedFiler,
+		priorWindow:     5 * time.Second,
+	}
+}
+
+// SetRing mirrors the master's LockRingUpdate so the client computes the same
+// primary the filers do. A non-zero version at or below the current one is
+// ignored once a ring exists, dropping reordered and redundant broadcasts;
+// version 0 always applies (bootstrap).
+func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
+	lc.ringMu.Lock()
+	defer lc.ringMu.Unlock()
+	if version != 0 && version <= lc.ringVersion && lc.ring != nil {
+		return
+	}
+	lc.ringVersion = version
+	// Build a fresh ring (not an in-place mutation) so the outgoing ring survives as
+	// priorRing with its own servers for the cooling-off window.
+	newRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	newRing.SetServers(servers)
+	if lc.ring != nil {
+		lc.priorRing = lc.ring
+		lc.ringChangedAt = time.Now()
+	}
+	lc.ring = newRing
+}
+
+// hostForKey returns the filer that should own key per the current ring view,
+// falling back to the seed filer when no view has been received yet.
+func (lc *LockClient) hostForKey(key string) pb.ServerAddress {
+	lc.ringMu.RLock()
+	defer lc.ringMu.RUnlock()
+	if lc.ring == nil {
+		return lc.seedFiler
+	}
+	if primary := lc.ring.GetPrimary(key); primary != "" {
+		return primary
+	}
+	return lc.seedFiler
+}
+
+// PrimaryForKey returns the ring owner for key, or "" before any ring arrives.
+// Unlike hostForKey it does not fall back to the seed, so a route-by-key caller
+// stays on the distributed lock until the ring is known.
+func (lc *LockClient) PrimaryForKey(key string) pb.ServerAddress {
+	lc.ringMu.RLock()
+	defer lc.ringMu.RUnlock()
+	if lc.ring == nil {
+		return ""
+	}
+	return lc.ring.GetPrimary(key)
+}
+
+// PriorOwnerForKey returns key's owner from the prior ring while a rebalance is
+// within the cooling-off window and ownership actually moved, else "". Lets a
+// route-by-key reader consult a just-moved key's previous owner before the new
+// owner's NotFound is final (the new owner may not have replicated the key yet).
+func (lc *LockClient) PriorOwnerForKey(key string) pb.ServerAddress {
+	lc.ringMu.RLock()
+	defer lc.ringMu.RUnlock()
+	if lc.ring == nil || lc.priorRing == nil {
+		return ""
+	}
+	if time.Since(lc.ringChangedAt) > lc.priorWindow {
+		return ""
+	}
+	current := lc.ring.GetPrimary(key)
+	prior := lc.priorRing.GetPrimary(key)
+	if prior != "" && prior != current {
+		return prior
+	}
+	return ""
+}
+
+type LiveLock struct {
+	generation          int64 // fencing token from the lock server; MUST stay first so the 64-bit atomic ops stay 8-byte aligned on 32-bit ARM
+	key                 string
+	renewToken          string
+	expireAtNs          int64
+	hostFiler           pb.ServerAddress
+	cancelCh            chan struct{}
+	renewalDone         chan struct{} // closed when the renewal goroutine exits; nil if there is none
+	grpcDialOption      grpc.DialOption
+	isLocked            int32 // 0 = unlocked, 1 = locked; use atomic operations
+	self                string
+	lc                  *LockClient
+	owner               string
+	lockTTL             time.Duration
+	consecutiveFailures int // Track connection failures to trigger fallback
+}
+
+// NewShortLivedLock creates a lock with a 5-second duration
+func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
+	lock = &LiveLock{
+		key:            key,
+		hostFiler:      lc.hostForKey(key),
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(5 * time.Second).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
+		lc:             lc,
+	}
+	lock.retryUntilLocked(5 * time.Second)
+	return
+}
+
+// NewBlockingLongLivedLock blocks until the lock is acquired, then starts a
+// background renewal goroutine that keeps the lock alive. This combines the
+// synchronous acquisition of NewShortLivedLock with the auto-renewal of
+// StartLongLivedLock. Release with Stop().
+func (lc *LockClient) NewBlockingLongLivedLock(key, owner string, lockTTL time.Duration) *LiveLock {
+	if lockTTL == 0 {
+		lockTTL = lock_manager.LiveLockTTL
+	}
+	lock := &LiveLock{
+		key:            key,
+		hostFiler:      lc.hostForKey(key),
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
+		lc:             lc,
+		lockTTL:        lockTTL,
+	}
+	// Block until acquired
+	lock.retryUntilLocked(lockTTL)
+	// Start renewal goroutine using a ticker for interruptible sleep
+	lock.renewalDone = make(chan struct{})
+	go func() {
+		defer close(lock.renewalDone)
+		renewInterval := lockTTL / 2
+		ticker := time.NewTicker(renewInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-lock.cancelCh:
+				return
+			case <-ticker.C:
+				if err := lock.AttemptToLock(lockTTL); err != nil {
+					glog.V(0).Infof("lock renewal failed for %s: %v", key, err)
+					atomic.StoreInt32(&lock.isLocked, 0)
+				}
+			}
+		}
+	}()
+	return lock
+}
+
+// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
+// lockTTL specifies how long the lock should be held. The renewal interval is
+// automatically derived as lockTTL / 2 to ensure timely renewals.
+func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string), lockTTL time.Duration) (lock *LiveLock) {
+	lock = &LiveLock{
+		key:            key,
+		hostFiler:      lc.hostForKey(key),
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
+		lc:             lc,
+		lockTTL:        lockTTL,
+	}
+	if lock.lockTTL == 0 {
+		lock.lockTTL = lock_manager.LiveLockTTL
+	}
+	lock.renewalDone = make(chan struct{})
+	go func() {
+		defer close(lock.renewalDone)
+		renewInterval := lock.lockTTL / 2
+		isLocked := false
+		lockOwner := ""
+		for {
+			// Check for cancellation BEFORE attempting to lock to avoid race condition
+			// where Stop() is called after sleep but before lock attempt
+			select {
+			case <-lock.cancelCh:
+				return
+			default:
+			}
+
+			if isLocked {
+				if err := lock.AttemptToLock(lock.lockTTL); err != nil {
+					glog.V(0).Infof("Lost lock %s: %v", key, err)
+					isLocked = false
+					atomic.StoreInt32(&lock.isLocked, 0)
+				}
+			} else {
+				if err := lock.AttemptToLock(lock.lockTTL); err == nil {
+					isLocked = true
+					// Note: AttemptToLock already sets lock.isLocked atomically on success
+				}
+			}
+			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
+				glog.V(0).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
+				onLockOwnerChange(lock.LockOwner())
+				lockOwner = lock.LockOwner()
+			}
+			// Sleep until the next attempt, but wake immediately on Stop() so
+			// the goroutine exits and closes renewalDone before Stop()'s bounded
+			// wait elapses. An uninterruptible sleep here (up to 5*renewInterval
+			// when unlocked) can outlast that wait and break the shutdown
+			// synchronization.
+			sleepFor := renewInterval
+			if !isLocked {
+				sleepFor = 5 * renewInterval
+			}
+			timer := time.NewTimer(sleepFor)
+			select {
+			case <-lock.cancelCh:
+				timer.Stop()
+				return
+			case <-timer.C:
+			}
+		}
+	}()
+	return
+}
+
+// retryUntilLocked blocks until the lock is acquired, polling at the steady
+// short cadence that AttemptToLock already enforces on contention (~1s). It
+// deliberately avoids util.RetryUntil's exponential backoff (which grows to
+// several seconds): when a holder on another mount releases the lock, the
+// waiter must pick it up promptly, otherwise cross-mount write handoff stalls
+// long enough to time out clients.
+func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
+	for lock.renewToken == "" {
+		if err := lock.AttemptToLock(lockDuration); err != nil {
+			glog.V(1).Infof("create lock %s: %v", lock.key, err)
+		}
+	}
+}
+
+func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
+	glog.V(4).Infof("LOCK: AttemptToLock key=%s owner=%s", lock.key, lock.self)
+	errorMessage, err := lock.doLock(lockDuration)
+	if err != nil {
+		glog.V(1).Infof("LOCK: doLock failed for key=%s: %v", lock.key, err)
+		time.Sleep(time.Second)
+		return err
+	}
+	if errorMessage != "" {
+		if strings.Contains(errorMessage, "lock already owned") {
+			glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		} else {
+			glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
+		}
+		time.Sleep(time.Second)
+		return fmt.Errorf("%v", errorMessage)
+	}
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
+		// Only log when transitioning from unlocked to locked
+		glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
+	}
+	atomic.StoreInt32(&lock.isLocked, 1)
+	return nil
+}
+
+func (lock *LiveLock) StopShortLivedLock() error {
+	if atomic.LoadInt32(&lock.isLocked) == 0 {
+		return nil
+	}
+	defer func() {
+		atomic.StoreInt32(&lock.isLocked, 0)
+	}()
+	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
+			Name:       lock.key,
+			RenewToken: lock.renewToken,
+		})
+		return err
+	})
+}
+
+// Stop stops a long-lived lock by closing the cancel channel and releasing the lock
+func (lock *LiveLock) Stop() error {
+	// Close the cancel channel to stop the long-lived lock goroutine
+	select {
+	case <-lock.cancelCh:
+		// Already closed
+	default:
+		close(lock.cancelCh)
+	}
+
+	// Wait for the renewal goroutine to fully exit before unlocking. A renewal
+	// in flight when we close cancelCh rotates renewToken on the server; if we
+	// then unlock with the token we read here, the unlock fails with a token
+	// mismatch and the lock lingers until its TTL expires — blocking other
+	// mounts waiting on the same file. Waiting for the goroutine to return also
+	// makes the renewToken read below race-free (channel close = happens-before).
+	if lock.renewalDone != nil {
+		select {
+		case <-lock.renewalDone:
+		case <-time.After(lock.lockTTL + 2*time.Second):
+			// The renewal goroutine is wedged, almost certainly in a stuck
+			// renewal RPC. Do not unlock here: the renewToken may be rotated
+			// when that RPC finally returns, so an unlock sent now could race
+			// it, be rejected on a stale token, and leave the lock lingering
+			// anyway. cancelCh is closed, so the goroutine stops renewing once
+			// its in-flight call returns and the lock then expires within its
+			// TTL on its own.
+			glog.Warningf("lock %s: renewal goroutine still running at shutdown; letting lock expire via TTL", lock.key)
+			return nil
+		}
+	}
+
+	// Also release the lock if held
+	// Note: We intentionally don't clear renewToken here because
+	// StopShortLivedLock needs it to properly unlock
+	return lock.StopShortLivedLock()
+}
+
+func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
+	glog.V(4).Infof("LOCK: doLock calling DistributedLock - key=%s filer=%s owner=%s",
+		lock.key, lock.hostFiler, lock.self)
+
+	previousHostFiler := lock.hostFiler
+	previousOwner := lock.owner
+
+	err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
+			Name:          lock.key,
+			SecondsToLock: int64(lockDuration.Seconds()),
+			RenewToken:    lock.renewToken,
+			IsMoved:       false,
+			Owner:         lock.self,
+		})
+		glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
+		if err == nil && resp != nil {
+			lock.renewToken = resp.RenewToken
+			if resp.Generation > 0 {
+				atomic.StoreInt64(&lock.generation, resp.Generation)
+			}
+			lock.consecutiveFailures = 0 // Reset failure counter on success
+			glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
+		} else {
+			//this can be retried. Need to remember the last valid renewToken
+			lock.renewToken = ""
+			glog.V(1).Infof("LOCK: Cleared renewToken for key=%s (err=%v)", lock.key, err)
+		}
+		if resp != nil {
+			errorMessage = resp.Error
+			if resp.LockHostMovedTo != "" && !pb.ServerAddress(resp.LockHostMovedTo).Equals(previousHostFiler) {
+				// Only log if the host actually changed
+				glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
+				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
+				// Don't update seedFiler - keep original for fallback
+			} else if resp.LockHostMovedTo != "" {
+				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
+			}
+			if resp.LockOwner != "" && resp.LockOwner != previousOwner {
+				// Only log if the owner actually changed
+				glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
+				lock.owner = resp.LockOwner
+			} else if resp.LockOwner != "" {
+				lock.owner = resp.LockOwner
+			} else if previousOwner != "" {
+				glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
+				lock.owner = ""
+			}
+		}
+		return err
+	})
+
+	if err != nil && !lock.hostFiler.Equals(lock.lc.seedFiler) {
+		lock.consecutiveFailures++
+		// Fall back to seed filer after 3 consecutive connection failures
+		if lock.consecutiveFailures >= 3 {
+			glog.V(0).Infof("LOCK: Connection failed %d times for key=%s filer=%s, falling back to seed filer=%s",
+				lock.consecutiveFailures, lock.key, lock.hostFiler, lock.lc.seedFiler)
+			lock.hostFiler = lock.lc.seedFiler
+			lock.consecutiveFailures = 0
+			lock.renewToken = ""
+		}
+	}
+
+	return
+}
+
+func (lock *LiveLock) LockOwner() string {
+	return lock.owner
+}
+
+// Generation returns the fencing token for this lock.
+// It increments on each fresh acquisition and stays the same on renewal.
+func (lock *LiveLock) Generation() int64 {
+	return atomic.LoadInt64(&lock.generation)
+}
+
+// IsLocked returns true if this instance currently holds the lock
+func (lock *LiveLock) IsLocked() bool {
+	return atomic.LoadInt32(&lock.isLocked) == 1
+}

weed/cluster/lock_client_test.go

@@ -0,0 +1,167 @@
+package cluster
+
+import (
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+// The gateway must resolve a lock key to the same primary the filers do,
+// otherwise it dials the wrong filer and the lock still gets forwarded. Both
+// sides use the same HashRing over the same server set, so for every key the
+// client's hostForKey must equal the filer ring's GetPrimary.
+func TestLockClientHostMatchesFilerRing(t *testing.T) {
+	servers := []pb.ServerAddress{
+		"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888",
+	}
+
+	filerRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	filerRing.SetServers(servers)
+
+	lc := NewLockClient(nil, "seed:8888")
+	lc.SetRing(servers, 1)
+
+	for _, key := range []string{
+		"s3.object.write:/buckets/b/obj-0",
+		"s3.object.write:/buckets/b/obj-1",
+		"s3.object.write:/buckets/b/obj-2",
+		"s3.object.write:/buckets/gosbench-0/w0obj-kilo-0877",
+		"some/other/key",
+	} {
+		if got, want := lc.hostForKey(key), filerRing.GetPrimary(key); got != want {
+			t.Errorf("key %q: client host %q != filer primary %q", key, got, want)
+		}
+	}
+}
+
+// Without a ring view, the client falls back to the seed filer (which the filer
+// forwards from), preserving the pre-optimization behavior.
+func TestLockClientHostFallsBackToSeed(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+	if got := lc.hostForKey("any-key"); got != "seed:8888" {
+		t.Errorf("expected seed fallback, got %q", got)
+	}
+
+	// An empty ring (no members yet) also falls back to the seed.
+	lc.SetRing(nil, 1)
+	if got := lc.hostForKey("any-key"); got != "seed:8888" {
+		t.Errorf("expected seed fallback on empty ring, got %q", got)
+	}
+}
+
+// A stale (older-version) update must not regress a newer ring view, while
+// version 0 always applies as a bootstrap.
+func TestLockClientSetRingVersionGuard(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+
+	newer := []pb.ServerAddress{"filer-a:8888", "filer-b:8888"}
+	lc.SetRing(newer, 10)
+	primaryAt10 := lc.hostForKey("k")
+
+	// Older version is ignored.
+	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 5)
+	if got := lc.hostForKey("k"); got != primaryAt10 {
+		t.Errorf("stale update applied: host changed to %q", got)
+	}
+
+	// version 0 is always accepted.
+	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 0)
+	if got := lc.hostForKey("k"); got != "filer-z:8888" {
+		t.Errorf("bootstrap update not applied, got %q", got)
+	}
+}
+
+// PrimaryForKey returns "" before any ring is received (so a route-by-key
+// caller falls back to the distributed lock) and the ring owner afterwards,
+// unlike hostForKey which falls back to the seed.
+func TestLockClientPrimaryForKey(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+	if got := lc.PrimaryForKey("k"); got != "" {
+		t.Errorf("expected empty before ring, got %q", got)
+	}
+
+	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 1)
+	got := lc.PrimaryForKey("k")
+	if got == "" {
+		t.Fatal("expected an owner after ring set")
+	}
+	if got != lc.hostForKey("k") {
+		t.Errorf("PrimaryForKey %q disagrees with hostForKey %q", got, lc.hostForKey("k"))
+	}
+}
+
+// A moved key reports its previous owner within the cooling-off window; an unmoved
+// key reports none.
+func TestLockClientPriorOwnerForKey(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+
+	setA := []pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888"}
+	lc.SetRing(setA, 1)
+	// One ring: nothing to fall back to.
+	if got := lc.PriorOwnerForKey("any"); got != "" {
+		t.Fatalf("single ring should have no prior owner, got %q", got)
+	}
+
+	priorRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
+	priorRing.SetServers(setA)
+
+	// Add a server so some keys' ownership moves.
+	setB := []pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888"}
+	lc.SetRing(setB, 2)
+
+	var moved, stable string
+	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
+		key := fmt.Sprintf("key-%d", i)
+		if lc.PrimaryForKey(key) != priorRing.GetPrimary(key) {
+			if moved == "" {
+				moved = key
+			}
+		} else if stable == "" {
+			stable = key
+		}
+	}
+	if moved == "" || stable == "" {
+		t.Skip("could not find both a moved and a stable key")
+	}
+
+	if got, want := lc.PriorOwnerForKey(moved), priorRing.GetPrimary(moved); got != want {
+		t.Fatalf("PriorOwnerForKey(moved)=%q, want %q", got, want)
+	}
+	if got := lc.PriorOwnerForKey(stable); got != "" {
+		t.Fatalf("unmoved key should have no prior owner, got %q", got)
+	}
+}
+
+// The prior owner is only offered within the cooling-off window.
+func TestLockClientPriorOwnerForKeyExpires(t *testing.T) {
+	lc := NewLockClient(nil, "seed:8888")
+	lc.priorWindow = 20 * time.Millisecond
+
+	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888"}, 1)
+	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888"}, 2)
+
+	time.Sleep(40 * time.Millisecond)
+	for i := 0; i < 2000; i++ {
+		if got := lc.PriorOwnerForKey(fmt.Sprintf("key-%d", i)); got != "" {
+			t.Fatalf("prior owner should expire after the cooling window, got %q", got)
+		}
+	}
+}
+
+// LiveLock.generation is a fencing token written and read with 64-bit atomic
+// operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
+// op requires an 8-byte-aligned address, which Go only guarantees for the
+// first field of an allocated struct. generation must therefore stay first:
+// a misaligned field makes StoreInt64 panic with "unaligned 64-bit atomic
+// operation" the moment this test runs on a 32-bit target.
+func TestLiveLockGenerationIsAligned(t *testing.T) {
+	var lock LiveLock
+	atomic.StoreInt64(&lock.generation, 42)
+	if got := lock.Generation(); got != 42 {
+		t.Fatalf("generation = %d, want 42", got)
+	}
+}

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -0,0 +1,192 @@
+package lock_manager
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const RenewInterval = time.Second * 3
+const LiveLockTTL = time.Second * 7
+
+var NoLockServerError = fmt.Errorf("no lock server found")
+
+// ReplicateFunc is called to replicate a lock operation to a backup server.
+// The caller (filer server) provides this to avoid a circular dependency.
+// seq is a per-lock monotonic sequence number for causal ordering — the backup
+// rejects mutations with seq <= its current seq for that key.
+type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
+
+type DistributedLockManager struct {
+	lockManager *LockManager
+	LockRing    *LockRing
+	Host        pb.ServerAddress
+	ReplicateFn ReplicateFunc // set by filer server after creation
+}
+
+func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
+	return &DistributedLockManager{
+		lockManager: NewLockManager(),
+		LockRing:    NewLockRing(time.Second * 5),
+		Host:        host,
+	}
+}
+
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If this is a renewal (non-empty token) and we still hold the lock locally,
+		// serve it here rather than redirecting. This handles the window between
+		// ring update and lock transfer completion — the old primary remains
+		// authoritative for locks it still holds.
+		if token != "" {
+			if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+				var seq int64
+				lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+				if err == nil && renewToken != "" {
+					dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+				}
+				return
+			}
+		}
+		movedTo = primary
+		return
+	}
+	var seq int64
+	lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	if err == nil && renewToken != "" {
+		dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+	}
+	return
+}
+
+func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve it here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
+			owner = lock.Owner
+			return
+		}
+		movedTo = primary
+		servers := dlm.LockRing.GetSnapshot()
+		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
+		return
+	}
+	owner, err = dlm.lockManager.GetLockOwner(key)
+	return
+}
+
+func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve the unlock here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+			var isUnlocked bool
+			var generation int64
+			var seq int64
+			isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+			if isUnlocked {
+				dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+			}
+			return
+		}
+		movedTo = primary
+		return
+	}
+	var isUnlocked bool
+	var generation int64
+	var seq int64
+	isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+	if isUnlocked {
+		dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+	}
+	return
+}
+
+// InsertLock is used to insert a lock to a server unconditionally.
+// It is used when a server is down and the lock is moved to another server.
+// After inserting, it replicates to the backup for this key.
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
+		dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
+	}
+}
+
+// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
+func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+}
+
+// RemoveBackupLock removes a backup lock unconditionally
+func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
+	dlm.lockManager.RemoveLock(key)
+}
+
+// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
+func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
+	dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
+}
+
+func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
+	return dlm.lockManager.SelectLocks(func(key string) bool {
+		server := hashKeyToServer(key, servers)
+		return server != dlm.Host
+	})
+}
+func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
+	return hashKeyToServer(key, servers)
+}
+
+func (dlm *DistributedLockManager) IsLocal(key string) bool {
+	primary := dlm.LockRing.GetPrimary(key)
+	if primary == "" {
+		return true
+	}
+	return primary == dlm.Host
+}
+
+// AllLocks returns all non-expired locks on this node
+func (dlm *DistributedLockManager) AllLocks() []*Lock {
+	return dlm.lockManager.AllLocks()
+}
+
+// PromoteLock promotes a backup lock to primary
+func (dlm *DistributedLockManager) PromoteLock(key string) bool {
+	return dlm.lockManager.PromoteLock(key)
+}
+
+// DemoteLock demotes a primary lock to backup
+func (dlm *DistributedLockManager) DemoteLock(key string) bool {
+	return dlm.lockManager.DemoteLock(key)
+}
+
+// GetLock returns a copy of a lock if it exists
+func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
+	return dlm.lockManager.GetLock(key)
+}
+
+// replicateToBackup asynchronously replicates a lock operation to the backup server
+func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+	_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
+	if backup == "" {
+		return // single-server deployment, no backup
+	}
+	if dlm.ReplicateFn != nil {
+		go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
+	}
+}

weed/cluster/lock_manager/distributed_lock_manager_test.go

@@ -0,0 +1,710 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCluster simulates a cluster of filer nodes with DLMs.
+// It wires up ReplicateFn so that replication calls arrive at the
+// correct peer's DLM, enabling end-to-end backup testing without gRPC.
+type testCluster struct {
+	mu    sync.Mutex
+	nodes map[pb.ServerAddress]*DistributedLockManager
+}
+
+func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
+	c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
+	servers := make([]pb.ServerAddress, len(hosts))
+	copy(servers, hosts)
+
+	for _, host := range hosts {
+		dlm := NewDistributedLockManager(host)
+		dlm.LockRing.SetSnapshot(servers, 0)
+		c.nodes[host] = dlm
+	}
+
+	// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
+	for _, dlm := range c.nodes {
+		d := dlm // capture
+		d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			c.mu.Lock()
+			target, ok := c.nodes[server]
+			c.mu.Unlock()
+			if !ok {
+				return // server is down
+			}
+			if isUnlock {
+				target.RemoveBackupLockIfSeq(key, generation, seq)
+			} else {
+				target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+			}
+		}
+	}
+
+	return c
+}
+
+func (c *testCluster) removeNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	delete(c.nodes, host)
+	c.mu.Unlock()
+
+	// Update all remaining nodes' rings
+	remaining := c.getServers()
+	for _, dlm := range c.getNodes() {
+		dlm.LockRing.SetSnapshot(remaining, 0)
+	}
+}
+
+func (c *testCluster) addNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	dlm := NewDistributedLockManager(host)
+	c.nodes[host] = dlm
+	c.mu.Unlock()
+
+	// Wire up replication
+	dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+		c.mu.Lock()
+		target, ok := c.nodes[server]
+		c.mu.Unlock()
+		if !ok {
+			return
+		}
+		if isUnlock {
+			target.RemoveBackupLockIfSeq(key, generation, seq)
+		} else {
+			target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+		}
+	}
+
+	servers := c.getServers()
+	for _, n := range c.getNodes() {
+		n.LockRing.SetSnapshot(servers, 0)
+	}
+}
+
+func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
+	for k, v := range c.nodes {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (c *testCluster) getServers() []pb.ServerAddress {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var servers []pb.ServerAddress
+	for s := range c.nodes {
+		servers = append(servers, s)
+	}
+	return servers
+}
+
+func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[host]
+}
+
+// acquireLock tries to acquire a lock on the correct primary node.
+// It follows redirects (movedTo) like a real client would.
+func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
+	// Try any node first (simulates client connecting to seed filer)
+	for _, dlm := range c.getNodes() {
+		expiry := time.Now().Add(ttl).UnixNano()
+		var movedTo pb.ServerAddress
+		var lockErr error
+		_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
+		if movedTo != "" && movedTo != dlm.Host {
+			// Follow redirect
+			target := c.get(movedTo)
+			if target == nil {
+				err = fmt.Errorf("primary %s is down", movedTo)
+				return
+			}
+			_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
+			if lockErr != nil {
+				err = lockErr
+				return
+			}
+			primaryHost = movedTo
+			// Wait briefly for async replication to complete
+			time.Sleep(10 * time.Millisecond)
+			return
+		}
+		if lockErr != nil {
+			err = lockErr
+			return
+		}
+		primaryHost = dlm.Host
+		time.Sleep(10 * time.Millisecond)
+		return
+	}
+	err = fmt.Errorf("no nodes available")
+	return
+}
+
+// renewLock renews a lock on the primary node
+func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
+	target := c.get(primaryHost)
+	if target == nil {
+		err = fmt.Errorf("primary %s is down", primaryHost)
+		return
+	}
+	expiry := time.Now().Add(ttl).UnixNano()
+	var movedTo pb.ServerAddress
+	var lockErr error
+	_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	if movedTo != "" && movedTo != primaryHost {
+		target = c.get(movedTo)
+		if target == nil {
+			err = fmt.Errorf("new primary %s is down", movedTo)
+			return
+		}
+		// Pass the existing token so the redirected renewal can match
+		// if the lock was already transferred to the new primary.
+		_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	}
+	err = lockErr
+	time.Sleep(10 * time.Millisecond)
+	return
+}
+
+// --- Test Cases ---
+
+func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
+	// Scenario: Lock is acquired, primary crashes, backup should have the lock
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-primary-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+	require.NotEmpty(t, renewToken)
+
+	// Find the backup for this key
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+	require.NotEmpty(t, backup, "should have a backup server")
+
+	// Verify backup has the lock
+	backupDlm := cluster.get(backup)
+	backupLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "backup should have the lock")
+	assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
+	assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
+
+	// Crash the primary
+	cluster.removeNode(primaryHost)
+
+	// Simulate topology change: promote backup locks
+	for _, dlm := range cluster.getNodes() {
+		locks := dlm.AllLocks()
+		for _, lock := range locks {
+			newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if newPrimary == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+	}
+
+	// The backup should now be the primary
+	newPrimary := backupDlm.LockRing.GetPrimary(key)
+	assert.Equal(t, backup, newPrimary, "backup should be the new primary")
+
+	// The promoted lock should work — verify it's no longer a backup
+	promotedLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "lock should still exist after promotion")
+	assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
+
+	// Client should be able to renew with the same token on the new primary
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
+}
+
+func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-backup-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash the backup
+	cluster.removeNode(backup)
+
+	// Primary should still work — renew the lock
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
+
+	// Verify primary is still the primary for this key
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	assert.Equal(t, primaryHost, newPrimary)
+}
+
+func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-both-crash"
+	_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash both
+	cluster.removeNode(primaryHost)
+	cluster.removeNode(backup)
+
+	// The lock is lost — the surviving node should be able to acquire it fresh
+	newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
+}
+
+func TestDLM_RollingRestart(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire multiple locks
+	type lockState struct {
+		key, owner, token string
+		generation        int64
+		primary           pb.ServerAddress
+	}
+	locks := make([]lockState, 5)
+	for i := range locks {
+		key := fmt.Sprintf("rolling-lock-%d", i)
+		token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
+	}
+
+	// Rolling restart: remove and re-add each node one at a time.
+	// After removing a node, promote backups and re-replicate to new backups
+	// to maintain the invariant that each lock has a backup copy.
+	for _, host := range hosts {
+		cluster.removeNode(host)
+
+		// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
+		for _, dlm := range cluster.getNodes() {
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && lock.IsBackup {
+					dlm.PromoteLock(lock.Key)
+				}
+			}
+			// Re-replicate all primary locks to their new backups
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && !lock.IsBackup {
+					dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		// Re-add the node
+		cluster.addNode(host)
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// After rolling restart, locks should survive via backup promotion
+	survivedCount := 0
+	for _, ls := range locks {
+		for _, dlm := range cluster.getNodes() {
+			lock, found := dlm.GetLock(ls.key)
+			if found && !lock.IsBackup {
+				survivedCount++
+				break
+			}
+		}
+	}
+	t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
+	require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
+}
+
+func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "gen-test-lock"
+
+	// Acquire lock — generation should be > 0
+	token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen1, int64(0))
+
+	// Renew — generation should stay the same
+	token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
+	require.NoError(t, err)
+	assert.Equal(t, gen1, gen2, "generation should not change on renewal")
+
+	// Let lock expire
+	time.Sleep(3 * time.Second)
+
+	// Re-acquire — generation should increment
+	_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
+	_ = token2
+}
+
+func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Break replication by setting a no-op ReplicateFn on all nodes
+	for _, dlm := range cluster.getNodes() {
+		dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			// Simulate replication failure: do nothing
+		}
+	}
+
+	key := "repl-fail-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Primary should have the lock
+	primaryDlm := cluster.get(primaryHost)
+	lock, found := primaryDlm.GetLock(key)
+	require.True(t, found, "primary should have the lock")
+	assert.False(t, lock.IsBackup)
+
+	// Backup should NOT have it (replication failed)
+	_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
+	backupDlm := cluster.get(backup)
+	_, found = backupDlm.GetLock(key)
+	assert.False(t, found, "backup should not have the lock when replication fails")
+
+	// Primary should still be able to renew
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken)
+}
+
+func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "unlock-repl-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Verify backup has the lock
+	_, found := cluster.get(backup).GetLock(key)
+	require.True(t, found, "backup should have the lock")
+
+	// Unlock on primary
+	primaryDlm := cluster.get(primaryHost)
+	movedTo, err := primaryDlm.Unlock(key, renewToken)
+	require.NoError(t, err)
+	assert.Empty(t, movedTo)
+
+	// Wait for async replication
+	time.Sleep(20 * time.Millisecond)
+
+	// Backup should also have removed the lock
+	_, found = cluster.get(backup).GetLock(key)
+	assert.False(t, found, "backup should remove lock after unlock replication")
+}
+
+func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
+	// Start with 2 servers, acquire lock, add a 3rd server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "topo-add-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Add a new server
+	cluster.addNode("filer3:8888")
+	time.Sleep(20 * time.Millisecond)
+
+	// The lock should still be accessible — either the same primary or on a new one
+	// Try to renew on the original primary first
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	if newPrimary == primaryHost {
+		// Still on same primary
+		newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+		require.NoError(t, err)
+		assert.NotEmpty(t, newToken)
+	}
+	// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
+	// which is tested at the server level
+}
+
+func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
+	// Verify that removing a server only affects locks on that server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire 50 locks
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, 50)
+	for i := range locks {
+		key := fmt.Sprintf("min-disrupt-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Count locks per server before removal
+	countBefore := make(map[pb.ServerAddress]int)
+	for _, l := range locks {
+		countBefore[l.primary]++
+	}
+	t.Logf("Lock distribution before: %v", countBefore)
+
+	// Remove filer2
+	cluster.removeNode("filer2:8888")
+
+	// Count how many locks changed primary
+	changed := 0
+	for _, l := range locks {
+		// Check where the lock should be now
+		for _, dlm := range cluster.getNodes() {
+			newPrimary := dlm.LockRing.GetPrimary(l.key)
+			if newPrimary != l.primary {
+				changed++
+			}
+			break
+		}
+	}
+
+	// Only locks from filer2 should have changed
+	assert.Equal(t, countBefore["filer2:8888"], changed,
+		"only locks from removed server should change primary")
+}
+
+func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
+	// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
+	// The new node steals hash ranges from surviving nodes, not just from the
+	// departed node. This test measures the disruption.
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire many locks
+	numLocks := 100
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, numLocks)
+	for i := range locks {
+		key := fmt.Sprintf("churn-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Record primary for each lock before the change
+	beforePrimary := make(map[string]pb.ServerAddress)
+	for _, l := range locks {
+		beforePrimary[l.key] = l.primary
+	}
+
+	// Drop filer3 and immediately add filer4
+	cluster.removeNode("filer3:8888")
+
+	// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
+	for _, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+		// Re-replicate primary locks to new backups
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && !lock.IsBackup {
+				dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Now add filer4 (new node, empty)
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
+	// transfer locks that now belong to filer4
+	for host, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p != host && !lock.IsBackup {
+				// This lock should move to the new primary
+				target := cluster.get(p)
+				if target != nil {
+					target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
+					dlm.DemoteLock(lock.Key)
+				}
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Count disruptions: locks whose primary changed to a node other than filer3's successor
+	disruptedFromSurvivors := 0
+	disruptedFromDeparted := 0
+	movedToFiler4 := 0
+	for _, l := range locks {
+		// What's the new primary?
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		oldPrimary := beforePrimary[l.key]
+		if newPrimary != oldPrimary {
+			if oldPrimary == "filer3:8888" {
+				disruptedFromDeparted++
+			} else {
+				disruptedFromSurvivors++
+			}
+		}
+		if newPrimary == "filer4:8888" {
+			movedToFiler4++
+		}
+	}
+
+	t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
+	t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
+	t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
+
+	// The key concern: filer4 joining disrupts locks on surviving nodes
+	// With consistent hashing, new node steals ~1/N of each surviving node's keys
+	// Verify that the transfer logic above moved those locks to filer4
+	for _, l := range locks {
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		target := cluster.get(newPrimary)
+		require.NotNil(t, target, "primary %s should exist", newPrimary)
+
+		lock, found := target.GetLock(l.key)
+		if !found {
+			// Lock may have only a backup copy if transfer happened but
+			// the lock was on the departed node and wasn't re-replicated.
+			// Check all nodes for any copy.
+			anyFound := false
+			for _, dlm := range cluster.getNodes() {
+				if _, f := dlm.GetLock(l.key); f {
+					anyFound = true
+					break
+				}
+			}
+			if !anyFound {
+				t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
+			}
+			continue
+		}
+		assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
+	}
+}
+
+func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
+	// When a new node joins and steals a key range from a surviving node,
+	// there's a window between ring update and lock transfer. During this
+	// window, a client renewal should still succeed on the old primary
+	// (because it still holds the lock locally).
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Find a key that will move primary when filer4 is added.
+	// Try candidate keys until we find one whose primary changes.
+	var key, renewToken string
+	var primaryHost pb.ServerAddress
+	for i := 0; i < 1000; i++ {
+		candidate := fmt.Sprintf("transfer-window-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
+		require.NoError(t, err)
+
+		// Check if adding filer4 would move this key's primary
+		tmpRing := NewHashRing(DefaultVnodeCount)
+		tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
+		newPrimary := tmpRing.GetPrimary(candidate)
+		if newPrimary != primary {
+			key = candidate
+			renewToken = token
+			primaryHost = primary
+			break
+		}
+	}
+	require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
+
+	// Add filer4 — this changes the primary for our key per the ring
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
+
+	// Renewal on the OLD primary should still succeed because it holds the lock locally
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err, "renewal on old primary should succeed during transfer window")
+	assert.NotEmpty(t, newToken, "should get a new token from old primary")
+	t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
+}
+
+func TestDLM_StaleReplicationRejected(t *testing.T) {
+	// Verify that a stale replication (lower seq) does not overwrite a newer one
+	lm := NewLockManager()
+
+	// Insert backup with seq=3
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
+	lock, found := lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Try to overwrite with stale seq=2 — should be rejected
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Update with higher seq=4 — should succeed
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
+	assert.Equal(t, int64(4), lock.Seq)
+
+	// Stale unlock (seq=2) should not delete the lock
+	removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
+	assert.False(t, removed, "stale unlock should be rejected")
+	_, found = lm.GetLock("key1")
+	assert.True(t, found, "lock should still exist after stale unlock")
+
+	// Valid unlock (seq=5) should delete
+	removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
+	assert.True(t, removed, "valid unlock should be accepted")
+	_, found = lm.GetLock("key1")
+	assert.False(t, found, "lock should be removed after valid unlock")
+}

weed/cluster/lock_manager/hash_ring.go

@@ -0,0 +1,206 @@
+package lock_manager
+
+import (
+	"hash/crc32"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const DefaultVnodeCount = 50
+
+// HashRing implements consistent hashing with virtual nodes.
+// When a server is removed, only the keys that hashed to that server
+// are remapped (to the next server on the ring), leaving all other
+// key-to-server mappings stable.
+//
+// UPGRADE NOTE: This replaces the previous modulo-based hashing
+// (hash % len(servers)). The two schemes compute different primaries
+// for the same key, so all filer nodes in the cluster must be upgraded
+// together (or via a rolling restart that completes within the lock TTL
+// window of ~7 seconds) to avoid routing disagreements.
+type HashRing struct {
+	mu            sync.RWMutex
+	vnodeCount    int
+	sortedHashes  []uint32                      // sorted ring positions
+	vnodeToServer map[uint32]pb.ServerAddress   // ring position → server
+	servers       map[pb.ServerAddress]struct{} // set of all servers
+}
+
+func NewHashRing(vnodeCount int) *HashRing {
+	if vnodeCount <= 0 {
+		vnodeCount = DefaultVnodeCount
+	}
+	return &HashRing{
+		vnodeCount:    vnodeCount,
+		vnodeToServer: make(map[uint32]pb.ServerAddress),
+		servers:       make(map[pb.ServerAddress]struct{}),
+	}
+}
+
+// AddServer adds a server with virtual nodes to the ring.
+func (hr *HashRing) AddServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; exists {
+		return
+	}
+	hr.servers[server] = struct{}{}
+	hr.rebuildRing()
+}
+
+// RemoveServer removes a server and its virtual nodes from the ring.
+func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; !exists {
+		return
+	}
+	delete(hr.servers, server)
+	hr.rebuildRing()
+}
+
+// SetServers replaces the entire server set.
+func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
+	for _, s := range servers {
+		hr.servers[s] = struct{}{}
+	}
+	hr.rebuildRing()
+}
+
+// GetPrimaryAndBackup returns the primary server for a key and its backup
+// (the next distinct server clockwise on the ring).
+// If there is only one server, backup is empty.
+func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return "", ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	primary = hr.vnodeToServer[hr.sortedHashes[idx]]
+
+	// Walk clockwise to find a different server for backup
+	ringLen := len(hr.sortedHashes)
+	for i := 1; i < ringLen; i++ {
+		candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
+		if candidate != primary {
+			backup = candidate
+			return
+		}
+	}
+	// Only one server — no backup
+	return primary, ""
+}
+
+// GetPrimary returns just the primary server for a key.
+func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	return hr.vnodeToServer[hr.sortedHashes[idx]]
+}
+
+// GetServers returns a sorted copy of all servers in the ring.
+func (hr *HashRing) GetServers() []pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for s := range hr.servers {
+		servers = append(servers, s)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+	return servers
+}
+
+// ServerCount returns the number of servers in the ring.
+func (hr *HashRing) ServerCount() int {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+	return len(hr.servers)
+}
+
+// rebuildRing rebuilds the sorted hash ring from the current server set.
+// Caller must hold hr.mu write lock.
+func (hr *HashRing) rebuildRing() {
+	hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
+	hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
+
+	// Sort so a vnode-hash collision resolves to the same server on every node;
+	// map iteration order alone is randomized per process.
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for server := range hr.servers {
+		servers = append(servers, server)
+	}
+	sort.Slice(servers, func(i, j int) bool { return servers[i] < servers[j] })
+
+	for _, server := range servers {
+		for i := 0; i < hr.vnodeCount; i++ {
+			vnodeKey := vnodeKeyFor(server, i)
+			hash := hashKey(vnodeKey)
+			hr.vnodeToServer[hash] = server
+			hr.sortedHashes = append(hr.sortedHashes, hash)
+		}
+	}
+	sort.Slice(hr.sortedHashes, func(i, j int) bool {
+		return hr.sortedHashes[i] < hr.sortedHashes[j]
+	})
+}
+
+// search finds the first ring position >= hash.
+func (hr *HashRing) search(hash uint32) int {
+	idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
+		return hr.sortedHashes[i] >= hash
+	})
+	if idx >= len(hr.sortedHashes) {
+		idx = 0 // wrap around
+	}
+	return idx
+}
+
+func hashKey(key string) uint32 {
+	return crc32.ChecksumIEEE([]byte(key))
+}
+
+func vnodeKeyFor(server pb.ServerAddress, index int) string {
+	// Use a format that distributes well across the ring
+	buf := make([]byte, 0, len(server)+10)
+	buf = append(buf, []byte(server)...)
+	buf = append(buf, '#')
+	buf = appendInt(buf, index)
+	return string(buf)
+}
+
+func appendInt(buf []byte, n int) []byte {
+	if n == 0 {
+		return append(buf, '0')
+	}
+	// Simple int-to-string without importing strconv
+	digits := [20]byte{}
+	pos := len(digits)
+	for n > 0 {
+		pos--
+		digits[pos] = byte('0' + n%10)
+		n /= 10
+	}
+	return append(buf, digits[pos:]...)
+}

weed/cluster/lock_manager/hash_ring_test.go

@@ -0,0 +1,208 @@
+package lock_manager
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHashRing_BasicOperations(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	p, b := hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress(""), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Single server — no backup
+	hr.AddServer("filer1:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress("filer1:8888"), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Two servers — backup is the other server
+	hr.AddServer("filer2:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Three servers
+	hr.AddServer("filer3:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Remove server
+	hr.RemoveServer("filer2:8888")
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_DuplicateAddRemove(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.AddServer("filer1:8888")
+	hr.AddServer("filer1:8888") // duplicate
+	assert.Equal(t, 1, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888")
+	assert.Equal(t, 0, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888") // remove non-existent
+	assert.Equal(t, 0, hr.ServerCount())
+}
+
+func TestHashRing_SetServers(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2", "c:3"})
+	assert.Equal(t, 3, hr.ServerCount())
+
+	servers := hr.GetServers()
+	assert.Equal(t, 3, len(servers))
+
+	// SetServers replaces
+	hr.SetServers([]pb.ServerAddress{"x:1", "y:2"})
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_ConsistencyOnRemoval(t *testing.T) {
+	// The key property of consistent hashing: when a server is removed,
+	// only keys that mapped to the removed server change.
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	numKeys := 1000
+	// Record where each key maps before removal
+	before := make(map[string]pb.ServerAddress, numKeys)
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		before[key] = hr.GetPrimary(key)
+	}
+
+	// Remove filer2
+	hr.RemoveServer("filer2:8888")
+
+	moved := 0
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		after := hr.GetPrimary(key)
+		if before[key] != after {
+			// Only keys from filer2 should move
+			assert.Equal(t, pb.ServerAddress("filer2:8888"), before[key],
+				"key %s moved from %s to %s, but it wasn't on the removed server", key, before[key], after)
+			moved++
+		}
+	}
+	// Roughly 1/3 of keys should move (those that were on filer2)
+	t.Logf("Keys that moved: %d / %d", moved, numKeys)
+	assert.Greater(t, moved, 0, "some keys should have moved")
+	assert.Less(t, moved, numKeys, "not all keys should move")
+}
+
+func TestHashRing_BackupIsSuccessor(t *testing.T) {
+	// After removing primary, the backup should become the new primary
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	// For each key, verify that removing the primary makes the backup the new primary
+	promoted := 0
+	total := 500
+	for i := 0; i < total; i++ {
+		key := fmt.Sprintf("test-lock-%d", i)
+		primary, backup := hr.GetPrimaryAndBackup(key)
+		assert.NotEqual(t, primary, backup)
+
+		// Temporarily remove primary
+		hr.RemoveServer(primary)
+		newPrimary := hr.GetPrimary(key)
+		if newPrimary == backup {
+			promoted++
+		}
+		// Restore
+		hr.AddServer(primary)
+	}
+	// The backup should become new primary for all keys
+	assert.Equal(t, total, promoted,
+		"backup should become new primary for all keys when primary is removed")
+}
+
+func TestHashRing_Distribution(t *testing.T) {
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	counts := make(map[pb.ServerAddress]int)
+	numKeys := 3000
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("dist-key-%d", i)
+		p := hr.GetPrimary(key)
+		counts[p]++
+	}
+
+	expected := float64(numKeys) / float64(len(servers))
+	for server, count := range counts {
+		deviation := math.Abs(float64(count)-expected) / expected
+		t.Logf("Server %s: %d keys (%.1f%% deviation)", server, count, deviation*100)
+		// Allow up to 40% deviation with 50 vnodes and 3 servers
+		assert.Less(t, deviation, 0.40,
+			"server %s has too many or too few keys: %d (expected ~%d)", server, count, int(expected))
+	}
+}
+
+func TestHashRing_GetPrimary(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	assert.Equal(t, pb.ServerAddress(""), hr.GetPrimary("key"))
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2"})
+
+	// Deterministic: same key always maps to same server
+	p1 := hr.GetPrimary("mykey")
+	p2 := hr.GetPrimary("mykey")
+	assert.Equal(t, p1, p2)
+
+	// GetPrimary matches the primary from GetPrimaryAndBackup
+	primary, _ := hr.GetPrimaryAndBackup("mykey")
+	assert.Equal(t, primary, hr.GetPrimary("mykey"))
+}
+
+// The ring must be identical on every node holding the same server set,
+// regardless of the order servers were added or supplied. Build rings several
+// ways and assert they agree on the primary for a wide range of keys.
+func TestHashRing_OrderIndependent(t *testing.T) {
+	servers := []pb.ServerAddress{
+		"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888", "filer-e:8888",
+	}
+
+	bySet := NewHashRing(50)
+	bySet.SetServers(servers)
+
+	byReverse := NewHashRing(50)
+	rev := append([]pb.ServerAddress(nil), servers...)
+	for i, j := 0, len(rev)-1; i < j; i, j = i+1, j-1 {
+		rev[i], rev[j] = rev[j], rev[i]
+	}
+	byReverse.SetServers(rev)
+
+	byAdd := NewHashRing(50)
+	for _, s := range []pb.ServerAddress{"filer-c:8888", "filer-e:8888", "filer-a:8888", "filer-d:8888", "filer-b:8888"} {
+		byAdd.AddServer(s)
+	}
+
+	for i := 0; i < 5000; i++ {
+		key := fmt.Sprintf("s3.object.write:/buckets/b/obj-%d", i)
+		p := bySet.GetPrimary(key)
+		if got := byReverse.GetPrimary(key); got != p {
+			t.Fatalf("reverse-order ring disagrees on %q: %s vs %s", key, got, p)
+		}
+		if got := byAdd.GetPrimary(key); got != p {
+			t.Fatalf("add-order ring disagrees on %q: %s vs %s", key, got, p)
+		}
+	}
+}

weed/cluster/lock_manager/lock_manager.go

@@ -0,0 +1,353 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new lock")
+var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on an expired lock")
+var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
+var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
+var LockNotFound = fmt.Errorf("lock not found")
+
+// LockManager local lock manager, used by distributed lock manager
+type LockManager struct {
+	locks          map[string]*Lock
+	accessLock     sync.RWMutex
+	nextGeneration atomic.Int64
+}
+type Lock struct {
+	Token       string
+	ExpiredAtNs int64
+	Key         string // only used for moving locks
+	Owner       string
+	IsBackup    bool  // true if this node holds the lock as a backup
+	Generation  int64 // monotonic fencing token, increments on fresh acquisition
+	Seq         int64 // per-lock sequence number, increments on every mutation (acquire/renew/unlock)
+}
+
+func NewLockManager() *LockManager {
+	t := &LockManager{
+		locks: make(map[string]*Lock),
+	}
+	go t.CleanUp()
+	return t
+}
+
+func (lm *LockManager) NextGeneration() int64 {
+	return lm.nextGeneration.Add(1)
+}
+
+func compareMutationVersion(generation, seq int64, existingGeneration, existingSeq int64) int {
+	switch {
+	case generation < existingGeneration:
+		return -1
+	case generation > existingGeneration:
+		return 1
+	case seq < existingSeq:
+		return -1
+	case seq > existingSeq:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (lm *LockManager) advanceGenerationFloor(generation int64) {
+	for {
+		current := lm.nextGeneration.Load()
+		if generation < current {
+			return
+		}
+		if lm.nextGeneration.CompareAndSwap(current, generation) {
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, generation int64, seq int64, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	glog.V(4).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
+
+	if oldValue, found := lm.locks[path]; found {
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
+			// lock is expired, set to a new lock
+			if token != "" {
+				glog.V(4).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
+				err = LockErrorNonEmptyTokenOnExpiredLock
+				return
+			} else {
+				// new lock
+				renewToken = uuid.New().String()
+				generation = lm.NextGeneration()
+				seq = 1
+				glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+				return
+			}
+		}
+		// not expired
+		lockOwner = oldValue.Owner
+		if oldValue.Token == token {
+			// token matches, renew the lock
+			renewToken = uuid.New().String()
+			generation = oldValue.Generation // keep same generation on renewal
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+			return
+		} else {
+			if token == "" {
+				// new lock
+				glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
+				err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
+				return
+			}
+			glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
+			err = fmt.Errorf("lock: token mismatch")
+			return
+		}
+	} else {
+		glog.V(4).Infof("key %s no lock owner %v", path, owner)
+		if token == "" {
+			// new lock
+			renewToken = uuid.New().String()
+			generation = lm.NextGeneration()
+			seq = 1
+			glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+			return
+		} else {
+			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
+			err = LockErrorNonEmptyTokenOnNewLock
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, generation int64, seq int64, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if oldValue, found := lm.locks[path]; found {
+		now := time.Now()
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
+			// lock is expired, delete it
+			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
+			delete(lm.locks, path)
+			return
+		}
+		if oldValue.Token == token {
+			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s unlocked with %v", path, token)
+			delete(lm.locks, path)
+			return
+		} else {
+			isUnlocked = false
+			err = UnlockErrorTokenMismatch
+			return
+		}
+	}
+	err = LockNotFound
+	return
+}
+
+func (lm *LockManager) CleanUp() {
+
+	for {
+		time.Sleep(1 * time.Minute)
+		now := time.Now().UnixNano()
+
+		lm.accessLock.Lock()
+		for key, value := range lm.locks {
+			if value == nil {
+				continue
+			}
+			if now > value.ExpiredAtNs {
+				glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
+				delete(lm.locks, key)
+			}
+		}
+		lm.accessLock.Unlock()
+	}
+}
+
+// SelectLocks takes out locks by key
+// if selectFn returns true, the lock will be removed and returned
+func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	now := time.Now().UnixNano()
+
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
+			delete(lm.locks, key)
+			continue
+		}
+		if selectFn(key) {
+			glog.V(4).Infof("key %s selected and deleted", key)
+			delete(lm.locks, key)
+			lock.Key = key
+			locks = append(locks, lock)
+		}
+	}
+	return
+}
+
+// InsertLock inserts a lock from a transfer if it is newer than the current state.
+func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("lock %s: rejecting stale transfer gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return false
+		}
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+	return true
+}
+
+// InsertBackupLock inserts or updates a lock as a backup copy.
+// It rejects stale mutations by comparing (generation, seq). If a current primary
+// already exists on this node, newer replicated state refreshes that primary copy
+// without demoting it back to a backup.
+func (lm *LockManager) InsertBackupLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("backup lock %s: rejecting stale gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return
+		}
+		lm.locks[path] = &Lock{
+			Token:       token,
+			ExpiredAtNs: expiredAtNs,
+			Owner:       owner,
+			IsBackup:    existing.IsBackup,
+			Generation:  generation,
+			Seq:         seq,
+		}
+		lm.advanceGenerationFloor(generation)
+		return
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, IsBackup: true, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+}
+
+// RemoveLock removes a lock by key
+func (lm *LockManager) RemoveLock(path string) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+	delete(lm.locks, path)
+}
+
+// RemoveBackupLockIfSeq removes the local copy only if the incoming mutation is
+// not older than the current (generation, seq). This prevents a late unlock from
+// deleting a newer reacquired lock whose seq has reset.
+func (lm *LockManager) RemoveBackupLockIfSeq(path string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) >= 0 {
+			delete(lm.locks, path)
+			return true
+		}
+		glog.V(4).Infof("backup lock %s: rejecting stale unlock gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+		return false
+	}
+	return true
+}
+
+// GetLock returns a copy of the lock for a key, if it exists and is not expired
+func (lm *LockManager) GetLock(key string) (*Lock, bool) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	lock, found := lm.locks[key]
+	if !found {
+		return nil, false
+	}
+	if time.Now().UnixNano() > lock.ExpiredAtNs {
+		return nil, false
+	}
+	// Return a copy
+	cp := *lock
+	return &cp, true
+}
+
+func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	if lock, found := lm.locks[key]; found {
+		if time.Now().UnixNano() > lock.ExpiredAtNs {
+			err = LockNotFound
+			return
+		}
+		return lock.Owner, nil
+	}
+	err = LockNotFound
+	return
+}
+
+// AllLocks returns a copy of all non-expired locks
+func (lm *LockManager) AllLocks() []*Lock {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	now := time.Now().UnixNano()
+	var result []*Lock
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			continue
+		}
+		cp := *lock
+		cp.Key = key
+		result = append(result, &cp)
+	}
+	return result
+}
+
+// PromoteLock changes a backup lock to a primary lock
+func (lm *LockManager) PromoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && lock.IsBackup {
+		lock.IsBackup = false
+		return true
+	}
+	return false
+}
+
+// DemoteLock changes a primary lock to a backup lock
+func (lm *LockManager) DemoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && !lock.IsBackup {
+		lock.IsBackup = true
+		return true
+	}
+	return false
+}

weed/cluster/lock_manager/lock_manager_test.go

@@ -0,0 +1,86 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockManager_GetLockOwnerIgnoresExpiredLock(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(-time.Second).UnixNano(), "token1", "owner1", 7, 3)
+
+	owner, err := lm.GetLockOwner("key1")
+	assert.Empty(t, owner)
+	assert.ErrorIs(t, err, LockNotFound)
+}
+
+func TestLockManager_InsertLockRejectsStaleTransfer(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 4)
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 3)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(4), lock.Seq)
+}
+
+func TestLockManager_InsertLockAdvancesGenerationCounter(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token1", "owner1", 12, 1)
+
+	_, renewToken, generation, _, err := lm.Lock("key2", time.Now().Add(30*time.Second).UnixNano(), "", "owner2")
+	assert.NoError(t, err)
+	assert.NotEmpty(t, renewToken)
+	assert.Greater(t, generation, int64(12))
+}
+
+func TestLockManager_InsertBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 9)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}
+
+func TestLockManager_InsertBackupLockKeepsPrimaryRole(t *testing.T) {
+	lm := NewLockManager()
+
+	ok := lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 8, 1)
+	assert.True(t, ok)
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 2)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.False(t, lock.IsBackup)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(2), lock.Seq)
+}
+
+func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+
+	removed := lm.RemoveBackupLockIfSeq("key1", 7, 9)
+	assert.False(t, removed)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}

weed/cluster/lock_manager/lock_ring.go

@@ -0,0 +1,211 @@
+package lock_manager
+
+import (
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+type LockRingSnapshot struct {
+	servers []pb.ServerAddress
+	ts      time.Time
+	ring    *HashRing // prebuilt ring for servers, so PriorOwner need not rebuild per call
+}
+
+type LockRing struct {
+	sync.RWMutex
+	snapshots        []*LockRingSnapshot
+	lastCompactTime  time.Time
+	snapshotInterval time.Duration
+	onTakeSnapshot   func(snapshot []pb.ServerAddress)
+	cleanupWg        sync.WaitGroup
+	Ring             *HashRing // consistent hash ring
+	version          int64     // monotonic version from master, rejects stale updates
+}
+
+func NewLockRing(snapshotInterval time.Duration) *LockRing {
+	return &LockRing{
+		snapshotInterval: snapshotInterval,
+		Ring:             NewHashRing(DefaultVnodeCount),
+	}
+}
+
+func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.ServerAddress)) {
+	r.Lock()
+	defer r.Unlock()
+	r.onTakeSnapshot = onTakeSnapshot
+}
+
+// SetSnapshot replaces the ring with a new server list from the master.
+// The version must be >= the current version, otherwise the update is rejected
+// (protects against reordered messages). Version 0 is always accepted (bootstrap).
+func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
+
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+
+	r.Lock()
+	if version > 0 && version < r.version {
+		glog.V(0).Infof("LockRing: rejecting stale update v%d (current v%d)", version, r.version)
+		r.Unlock()
+		return false
+	}
+	r.version = version
+	// Update the ring while holding the lock so version and ring state
+	// are always consistent — prevents a concurrent SetSnapshot from
+	// seeing the new version but applying its servers to the old ring.
+	r.Ring.SetServers(servers)
+	// Append the snapshot under the same lock as the ring update so a concurrent
+	// PriorOwner always sees snapshots[0] matching r.Ring (and snapshots[1] as the
+	// true prior); otherwise it could pair a new ring with a stale prior snapshot.
+	r.addOneSnapshotLocked(servers)
+	r.Unlock()
+
+	r.cleanupWg.Add(1)
+	go func() {
+		defer r.cleanupWg.Done()
+		<-time.After(r.snapshotInterval)
+		r.compactSnapshots()
+	}()
+	return true
+}
+
+// Version returns the current ring version.
+func (r *LockRing) Version() int64 {
+	r.RLock()
+	defer r.RUnlock()
+	return r.version
+}
+
+// addOneSnapshotLocked appends a new snapshot (newest at index 0). The caller
+// must hold r.Lock(), so the ring update and snapshot append are one atomic step.
+func (r *LockRing) addOneSnapshotLocked(servers []pb.ServerAddress) {
+	ts := time.Now()
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	t := &LockRingSnapshot{
+		servers: servers,
+		ts:      ts,
+		ring:    ring,
+	}
+	r.snapshots = append(r.snapshots, t)
+	for i := len(r.snapshots) - 2; i >= 0; i-- {
+		r.snapshots[i+1] = r.snapshots[i]
+	}
+	r.snapshots[0] = t
+
+	if r.onTakeSnapshot != nil {
+		r.onTakeSnapshot(t.servers)
+	}
+}
+
+func (r *LockRing) compactSnapshots() {
+	r.Lock()
+	defer r.Unlock()
+
+	ts := time.Now()
+	recentSnapshotIndex := 1
+	for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
+		if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
+			break
+		}
+	}
+	if recentSnapshotIndex+1 <= len(r.snapshots) {
+		r.snapshots = r.snapshots[:recentSnapshotIndex+1]
+	}
+	r.lastCompactTime = ts
+}
+
+func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
+	r.RLock()
+	defer r.RUnlock()
+
+	if len(r.snapshots) == 0 {
+		return
+	}
+	return r.snapshots[0].servers
+}
+
+// WaitForCleanup waits for all pending cleanup operations to complete
+func (r *LockRing) WaitForCleanup() {
+	r.cleanupWg.Wait()
+}
+
+// GetSnapshotCount safely returns the number of snapshots for testing
+func (r *LockRing) GetSnapshotCount() int {
+	r.RLock()
+	defer r.RUnlock()
+	return len(r.snapshots)
+}
+
+// GetPrimaryAndBackup returns the primary and backup servers for a key
+// using the consistent hash ring.
+func (r *LockRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	return r.Ring.GetPrimaryAndBackup(key)
+}
+
+// GetPrimary returns the primary server for a key using the consistent hash ring.
+func (r *LockRing) GetPrimary(key string) pb.ServerAddress {
+	return r.Ring.GetPrimary(key)
+}
+
+// PriorOwner returns the key's owner from the previous ring snapshot, but only
+// while the ring changed within the last snapshotInterval and that owner differs
+// from the current primary. This is the cooling-off window in which the previous
+// owner may still hold locks the new owner has not yet rebuilt — a caller can
+// consult it before granting so a fresh owner does not double-grant during a
+// rebalance. Returns "" outside the window or when ownership did not move. It
+// uses the snapshot's prebuilt ring, so it does not rebuild a hash ring per call.
+func (r *LockRing) PriorOwner(key string) pb.ServerAddress {
+	r.RLock()
+	defer r.RUnlock()
+	return r.priorOwnerLocked(key)
+}
+
+// WriteOwner returns the filer that should serialize writes to key: the prior
+// owner while a ring change is still within the cooling-off window, otherwise
+// the current primary. Both are read under one lock so the pair cannot come
+// from different rings, which could otherwise name the same filer twice.
+func (r *LockRing) WriteOwner(key string) pb.ServerAddress {
+	r.RLock()
+	defer r.RUnlock()
+	if prior := r.priorOwnerLocked(key); prior != "" {
+		return prior
+	}
+	return r.Ring.GetPrimary(key)
+}
+
+// priorOwnerLocked is PriorOwner's body; the caller holds at least RLock.
+func (r *LockRing) priorOwnerLocked(key string) pb.ServerAddress {
+	if len(r.snapshots) < 2 {
+		return ""
+	}
+	if time.Since(r.snapshots[0].ts) > r.snapshotInterval {
+		return ""
+	}
+	current := r.Ring.GetPrimary(key)
+	var prior pb.ServerAddress
+	if pr := r.snapshots[1].ring; pr != nil {
+		prior = pr.GetPrimary(key)
+	} else {
+		prior = hashKeyToServer(key, r.snapshots[1].servers)
+	}
+	if prior != "" && prior != current {
+		return prior
+	}
+	return ""
+}
+
+// hashKeyToServer uses a temporary consistent hash ring for the given server list.
+func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
+	if len(servers) == 0 {
+		return ""
+	}
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	return ring.GetPrimary(key)
+}

weed/cluster/lock_manager/lock_ring_prior_test.go

@@ -0,0 +1,102 @@
+package lock_manager
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func TestLockRing_PriorOwner(t *testing.T) {
+	r := NewLockRing(5 * time.Second)
+	t.Cleanup(r.WaitForCleanup)
+
+	setA := []pb.ServerAddress{"s1:1", "s2:1", "s3:1"}
+	r.SetSnapshot(setA, 1)
+
+	// Only one snapshot: nothing to fall back to.
+	if got := r.PriorOwner("any"); got != "" {
+		t.Fatalf("single snapshot should have no prior owner, got %q", got)
+	}
+
+	// Add a server so some keys' ownership moves.
+	setB := []pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}
+	r.SetSnapshot(setB, 2)
+
+	var moved, stable string
+	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
+		key := fmt.Sprintf("key-%d", i)
+		if r.GetPrimary(key) != hashKeyToServer(key, setA) {
+			if moved == "" {
+				moved = key
+			}
+		} else if stable == "" {
+			stable = key
+		}
+	}
+	if moved == "" || stable == "" {
+		t.Skip("could not find both a moved and a stable key")
+	}
+
+	if got, want := r.PriorOwner(moved), hashKeyToServer(moved, setA); got != want {
+		t.Fatalf("PriorOwner(moved)=%q, want %q", got, want)
+	}
+	if got := r.PriorOwner(stable); got != "" {
+		t.Fatalf("unmoved key should have no prior owner, got %q", got)
+	}
+}
+
+func TestLockRing_PriorOwnerExpires(t *testing.T) {
+	r := NewLockRing(20 * time.Millisecond)
+	t.Cleanup(r.WaitForCleanup)
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1"}, 1)
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}, 2)
+
+	// Past the cooling interval, the prior owner is no longer offered.
+	time.Sleep(40 * time.Millisecond)
+	for i := 0; i < 2000; i++ {
+		if got := r.PriorOwner(fmt.Sprintf("key-%d", i)); got != "" {
+			t.Fatalf("prior owner should expire after the cooling interval, got %q", got)
+		}
+	}
+}
+
+// WriteOwner resolves prior-else-current under one read lock, so the two views
+// can never come from different rings.
+func TestLockRing_WriteOwner(t *testing.T) {
+	r := NewLockRing(5 * time.Second)
+	t.Cleanup(r.WaitForCleanup)
+
+	setA := []pb.ServerAddress{"s1:1", "s2:1", "s3:1"}
+	r.SetSnapshot(setA, 1)
+
+	// One snapshot: no prior owner, so the primary owns every key.
+	if got, want := r.WriteOwner("any"), r.GetPrimary("any"); got != want {
+		t.Fatalf("WriteOwner=%q, want the primary %q", got, want)
+	}
+
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}, 2)
+
+	var moved, stable string
+	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
+		key := fmt.Sprintf("key-%d", i)
+		if r.PriorOwner(key) != "" {
+			if moved == "" {
+				moved = key
+			}
+		} else if stable == "" {
+			stable = key
+		}
+	}
+	if moved == "" || stable == "" {
+		t.Skip("could not find both a moved and a stable key")
+	}
+
+	if got, want := r.WriteOwner(moved), r.PriorOwner(moved); got != want {
+		t.Fatalf("moved key: WriteOwner=%q, want the prior owner %q", got, want)
+	}
+	if got, want := r.WriteOwner(stable), r.GetPrimary(stable); got != want {
+		t.Fatalf("stable key: WriteOwner=%q, want the primary %q", got, want)
+	}
+}

weed/cluster/lock_manager/lock_ring_test.go

@@ -0,0 +1,98 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockRing_SetSnapshot(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Set 5 servers
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 1)
+
+	servers := r.GetSnapshot()
+	assert.Equal(t, 5, len(servers))
+
+	// Replace with 2 servers
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8081", "localhost:8083"}, 2)
+
+	r.WaitForCleanup()
+	servers = r.GetSnapshot()
+	assert.Equal(t, 2, len(servers))
+	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
+	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
+
+	// Verify compaction
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2)
+}
+
+func TestLockRing_SnapshotCompaction(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"}, 1)
+	assert.Equal(t, 1, r.GetSnapshotCount())
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"}, 2)
+	assert.Equal(t, 2, r.GetSnapshotCount())
+
+	// Wait for compaction
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"}, 3)
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+	servers := r.GetSnapshot()
+	assert.Equal(t, 4, len(servers))
+
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2, "Snapshots should be compacted")
+
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 4)
+	servers = r.GetSnapshot()
+	assert.Equal(t, 5, len(servers))
+}
+
+func TestLockRing_VersionRejectsStale(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Apply version 3
+	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()))
+
+	// Stale version 2 — should be rejected
+	ok = r.SetSnapshot([]pb.ServerAddress{"x:1"}, 2)
+	assert.False(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()), "stale update should not change the ring")
+
+	// Same version 3 — accepted (SetSnapshot accepts version >= current, state-changing)
+	ok = r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, 2, len(r.GetSnapshot()))
+
+	// Newer version 5 — should be accepted
+	ok = r.SetSnapshot([]pb.ServerAddress{"d:1", "e:2", "f:3", "g:4"}, 5)
+	assert.True(t, ok)
+	assert.Equal(t, int64(5), r.Version())
+	assert.Equal(t, 4, len(r.GetSnapshot()))
+
+	// Version 0 always accepted (bootstrap)
+	ok = r.SetSnapshot([]pb.ServerAddress{"z:1"}, 0)
+	assert.True(t, ok)
+	assert.Equal(t, 1, len(r.GetSnapshot()))
+}

weed/cluster/lock_ring_manager.go

@@ -0,0 +1,164 @@
+package cluster
+
+import (
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+const LockRingStabilizationInterval = 1 * time.Second
+
+// LockRingManager tracks filer membership for the distributed lock ring.
+// It batches rapid topology changes (e.g., node drop + join) with a
+// stabilization timer, then broadcasts the complete member list atomically
+// so filers receive a single consistent ring update instead of multiple
+// intermediate states.
+type LockRingManager struct {
+	mu             sync.Mutex
+	members        map[FilerGroupName]map[pb.ServerAddress]struct{}
+	version        map[FilerGroupName]int64
+	lastBroadcast  map[FilerGroupName]*master_pb.LockRingUpdate
+	pendingTimer   map[FilerGroupName]*time.Timer
+	broadcastFn    func(resp *master_pb.KeepConnectedResponse)
+	stabilizeDelay time.Duration
+}
+
+func NewLockRingManager(broadcastFn func(resp *master_pb.KeepConnectedResponse)) *LockRingManager {
+	return &LockRingManager{
+		members:        make(map[FilerGroupName]map[pb.ServerAddress]struct{}),
+		version:        make(map[FilerGroupName]int64),
+		lastBroadcast:  make(map[FilerGroupName]*master_pb.LockRingUpdate),
+		pendingTimer:   make(map[FilerGroupName]*time.Timer),
+		broadcastFn:    broadcastFn,
+		stabilizeDelay: LockRingStabilizationInterval,
+	}
+}
+
+// AddServer records a filer joining and schedules a batched broadcast.
+func (lrm *LockRingManager) AddServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if _, ok := lrm.members[filerGroup]; !ok {
+		lrm.members[filerGroup] = make(map[pb.ServerAddress]struct{})
+	}
+	lrm.members[filerGroup][address] = struct{}{}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// RemoveServer records a filer leaving and schedules a batched broadcast.
+func (lrm *LockRingManager) RemoveServer(filerGroup FilerGroupName, address pb.ServerAddress) {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	if members, ok := lrm.members[filerGroup]; ok {
+		delete(members, address)
+	}
+	lrm.scheduleBroadcast(filerGroup)
+}
+
+// GetServers returns the current member list for a filer group.
+func (lrm *LockRingManager) GetServers(filerGroup FilerGroupName) []string {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	members, ok := lrm.members[filerGroup]
+	if !ok {
+		return nil
+	}
+	servers := make([]string, 0, len(members))
+	for addr := range members {
+		servers = append(servers, string(addr))
+	}
+	return servers
+}
+
+// GetVersion returns the current version for a filer group.
+func (lrm *LockRingManager) GetVersion(filerGroup FilerGroupName) int64 {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+	return lrm.version[filerGroup]
+}
+
+// GetLastUpdate returns a copy of the most recently broadcast lock-ring snapshot
+// for the filer group. It intentionally does not expose pending, unstabilized changes.
+func (lrm *LockRingManager) GetLastUpdate(filerGroup FilerGroupName) *master_pb.LockRingUpdate {
+	lrm.mu.Lock()
+	defer lrm.mu.Unlock()
+
+	update, ok := lrm.lastBroadcast[filerGroup]
+	if !ok || update == nil {
+		return nil
+	}
+	cp := *update
+	cp.Servers = append([]string(nil), update.Servers...)
+	return &cp
+}
+
+// scheduleBroadcast resets the stabilization timer. If another change arrives
+// before the timer fires, the timer resets, batching the changes.
+// Caller must hold lrm.mu.
+func (lrm *LockRingManager) scheduleBroadcast(filerGroup FilerGroupName) {
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if !timer.Stop() {
+			// Timer already fired, callback is running or queued.
+			// It will pick up the latest state from lrm.members, so
+			// just schedule a new one for any further changes.
+		}
+	}
+	lrm.pendingTimer[filerGroup] = time.AfterFunc(lrm.stabilizeDelay, func() {
+		lrm.doBroadcast(filerGroup)
+	})
+}
+
+func (lrm *LockRingManager) doBroadcast(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	// Use wall-clock nanoseconds so the version survives master restarts
+	// without persistence — a restarted master produces a version greater
+	// than any pre-restart value (assuming clocks don't jump backward).
+	version := time.Now().UnixNano()
+	lrm.version[filerGroup] = version
+	servers := make([]string, 0)
+	if members, ok := lrm.members[filerGroup]; ok {
+		for addr := range members {
+			servers = append(servers, string(addr))
+		}
+	}
+	update := &master_pb.LockRingUpdate{
+		FilerGroup: string(filerGroup),
+		Servers:    append([]string(nil), servers...),
+		Version:    version,
+	}
+	lrm.lastBroadcast[filerGroup] = update
+	delete(lrm.pendingTimer, filerGroup)
+	lrm.mu.Unlock()
+
+	glog.V(0).Infof("LockRing: broadcasting ring update for group %q version %d: %v", filerGroup, version, servers)
+
+	if lrm.broadcastFn != nil {
+		lrm.broadcastFn(&master_pb.KeepConnectedResponse{
+			LockRingUpdate: update,
+		})
+	}
+}
+
+// FlushPending fires any pending timer immediately (for testing or shutdown).
+func (lrm *LockRingManager) FlushPending(filerGroup FilerGroupName) {
+	lrm.mu.Lock()
+	if timer, ok := lrm.pendingTimer[filerGroup]; ok {
+		if timer.Stop() {
+			// Timer was pending — we stopped it, so we broadcast now
+			delete(lrm.pendingTimer, filerGroup)
+			lrm.mu.Unlock()
+			lrm.doBroadcast(filerGroup)
+		} else {
+			// Timer already fired, callback is running — let it finish
+			lrm.mu.Unlock()
+		}
+	} else {
+		lrm.mu.Unlock()
+	}
+}

weed/cluster/lock_ring_manager_test.go

@@ -0,0 +1,229 @@
+package cluster
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLockRingManager_BatchesRapidChanges(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Add 3 servers in rapid succession
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+
+	// No broadcast should have happened yet (timer hasn't fired)
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts), "should not broadcast before stabilization delay")
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "should batch into a single broadcast")
+	assert.Equal(t, 3, len(broadcasts[0].Servers), "should include all 3 servers")
+	assert.Greater(t, broadcasts[0].Version, int64(0))
+	mu.Unlock()
+}
+
+func TestLockRingManager_DropAndJoinBatched(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 100 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	// Set up initial state
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.AddServer(group, "filer3:8888")
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	broadcasts = nil // reset
+	mu.Unlock()
+
+	// Simulate drop + join in rapid succession
+	lrm.RemoveServer(group, "filer3:8888")
+	lrm.AddServer(group, "filer4:8888")
+
+	// Should not have broadcast yet
+	mu.Lock()
+	assert.Equal(t, 0, len(broadcasts))
+	mu.Unlock()
+
+	// Wait for stabilization
+	time.Sleep(200 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts), "drop+join should be batched into single broadcast")
+	servers := broadcasts[0].Servers
+	assert.Equal(t, 3, len(servers), "should have filer1, filer2, filer4")
+	// filer3 should be gone, filer4 should be present
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.False(t, serverSet["filer3:8888"], "filer3 should be removed")
+	assert.True(t, serverSet["filer4:8888"], "filer4 should be added")
+	mu.Unlock()
+}
+
+func TestLockRingManager_VersionIncrements(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	lrm.AddServer(group, "filer2:8888")
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	require.Equal(t, 2, len(broadcasts))
+	assert.Greater(t, broadcasts[0].Version, int64(0), "version should be positive")
+	assert.Greater(t, broadcasts[1].Version, broadcasts[0].Version, "versions should be monotonically increasing")
+	mu.Unlock()
+}
+
+func TestLockRingManager_FlushPending(t *testing.T) {
+	var mu sync.Mutex
+	var broadcasts []*master_pb.LockRingUpdate
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcasts = append(broadcasts, resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 10 * time.Second // long delay
+
+	group := FilerGroupName("default")
+
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	// Flush immediately
+	lrm.FlushPending(group)
+
+	mu.Lock()
+	require.Equal(t, 1, len(broadcasts))
+	assert.Equal(t, 2, len(broadcasts[0].Servers))
+	mu.Unlock()
+}
+
+func TestLockRingManager_MultipleGroups(t *testing.T) {
+	var mu sync.Mutex
+	broadcastsByGroup := make(map[string][]*master_pb.LockRingUpdate)
+
+	lrm := NewLockRingManager(func(resp *master_pb.KeepConnectedResponse) {
+		mu.Lock()
+		if resp.LockRingUpdate != nil {
+			broadcastsByGroup[resp.LockRingUpdate.FilerGroup] = append(
+				broadcastsByGroup[resp.LockRingUpdate.FilerGroup], resp.LockRingUpdate)
+		}
+		mu.Unlock()
+	})
+	lrm.stabilizeDelay = 50 * time.Millisecond
+
+	lrm.AddServer("group1", "filer1:8888")
+	lrm.AddServer("group2", "filer2:8888")
+
+	time.Sleep(100 * time.Millisecond)
+
+	mu.Lock()
+	assert.Equal(t, 1, len(broadcastsByGroup["group1"]))
+	assert.Equal(t, 1, len(broadcastsByGroup["group2"]))
+	assert.Equal(t, []string{"filer1:8888"}, broadcastsByGroup["group1"][0].Servers)
+	assert.Equal(t, []string{"filer2:8888"}, broadcastsByGroup["group2"][0].Servers)
+	mu.Unlock()
+}
+
+func TestLockRingManager_GetServers(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+
+	servers := lrm.GetServers(group)
+	assert.Equal(t, 2, len(servers))
+
+	// Contains both
+	serverSet := make(map[string]bool)
+	for _, s := range servers {
+		serverSet[s] = true
+	}
+	assert.True(t, serverSet["filer1:8888"])
+	assert.True(t, serverSet["filer2:8888"])
+
+	// Remove one
+	lrm.RemoveServer(group, "filer1:8888")
+	servers = lrm.GetServers(group)
+	assert.Equal(t, 1, len(servers))
+	assert.Equal(t, "filer2:8888", servers[0])
+}
+
+func TestLockRingManager_NoBroadcastWithoutFn(t *testing.T) {
+	// No panic when broadcastFn is nil
+	lrm := NewLockRingManager(nil)
+	lrm.stabilizeDelay = 10 * time.Millisecond
+
+	lrm.AddServer("default", pb.ServerAddress("filer1:8888"))
+	time.Sleep(50 * time.Millisecond) // should not panic
+}
+
+func TestLockRingManager_GetLastUpdateReturnsBroadcastState(t *testing.T) {
+	lrm := NewLockRingManager(nil)
+
+	group := FilerGroupName("default")
+	lrm.AddServer(group, "filer1:8888")
+	lrm.AddServer(group, "filer2:8888")
+	lrm.FlushPending(group)
+
+	update := lrm.GetLastUpdate(group)
+	require.NotNil(t, update)
+	assert.Equal(t, "default", update.FilerGroup)
+	assert.ElementsMatch(t, []string{"filer1:8888", "filer2:8888"}, update.Servers)
+	assert.Greater(t, update.Version, int64(0))
+}

weed/cluster/maintenance/maintenance_config.go

@@ -0,0 +1,16 @@
+package maintenance
+
+const DefaultMaintenanceSleepMinutes = 17
+
+const DefaultMasterMaintenanceScripts = `
+  lock
+  ec.encode -fullPercent=95 -quietFor=1h
+  ec.rebuild -apply
+  ec.balance -apply
+  fs.log.purge -daysAgo=7
+  volume.deleteEmpty -quietFor=24h -apply
+  volume.balance -apply
+  volume.fix.replication -apply
+  s3.clean.uploads -timeAgo=24h
+  unlock
+`

weed/cluster/master_client.go

@@ -0,0 +1,38 @@
+package cluster
+
+import (
+	"context"
+
+	"github.com/dustin/go-humanize/english"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+)
+
+func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
+
+	if grpcErr := pb.WithMasterClient(context.Background(), false, master, grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: clientType,
+			FilerGroup: filerGroup,
+		})
+		if err != nil {
+			return err
+		}
+
+		glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), english.PluralWord(len(resp.ClusterNodes), clientType, ""))
+		for _, node := range resp.ClusterNodes {
+			existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
+				NodeType:    FilerType,
+				Address:     node.Address,
+				IsAdd:       true,
+				CreatedAtNs: node.CreatedAtNs,
+			})
+		}
+		return nil
+	}); grpcErr != nil {
+		glog.V(0).Infof("connect to %s: %v", master, grpcErr)
+	}
+	return
+}