snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -0,0 +1,192 @@
+package lock_manager
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const RenewInterval = time.Second * 3
+const LiveLockTTL = time.Second * 7
+
+var NoLockServerError = fmt.Errorf("no lock server found")
+
+// ReplicateFunc is called to replicate a lock operation to a backup server.
+// The caller (filer server) provides this to avoid a circular dependency.
+// seq is a per-lock monotonic sequence number for causal ordering — the backup
+// rejects mutations with seq <= its current seq for that key.
+type ReplicateFunc func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool)
+
+type DistributedLockManager struct {
+	lockManager *LockManager
+	LockRing    *LockRing
+	Host        pb.ServerAddress
+	ReplicateFn ReplicateFunc // set by filer server after creation
+}
+
+func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
+	return &DistributedLockManager{
+		lockManager: NewLockManager(),
+		LockRing:    NewLockRing(time.Second * 5),
+		Host:        host,
+	}
+}
+
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, generation int64, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If this is a renewal (non-empty token) and we still hold the lock locally,
+		// serve it here rather than redirecting. This handles the window between
+		// ring update and lock transfer completion — the old primary remains
+		// authoritative for locks it still holds.
+		if token != "" {
+			if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+				var seq int64
+				lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+				if err == nil && renewToken != "" {
+					dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+				}
+				return
+			}
+		}
+		movedTo = primary
+		return
+	}
+	var seq int64
+	lockOwner, renewToken, generation, seq, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	if err == nil && renewToken != "" {
+		dlm.replicateToBackup(key, expiredAtNs, renewToken, owner, generation, seq, false)
+	}
+	return
+}
+
+func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve it here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup {
+			owner = lock.Owner
+			return
+		}
+		movedTo = primary
+		servers := dlm.LockRing.GetSnapshot()
+		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
+		return
+	}
+	owner, err = dlm.lockManager.GetLockOwner(key)
+	return
+}
+
+func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
+	primary, _ := dlm.LockRing.GetPrimaryAndBackup(key)
+	if primary == "" {
+		err = NoLockServerError
+		return
+	}
+	if primary != dlm.Host {
+		// If we still hold this lock locally, serve the unlock here
+		if lock, found := dlm.lockManager.GetLock(key); found && !lock.IsBackup && lock.Token == token {
+			var isUnlocked bool
+			var generation int64
+			var seq int64
+			isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+			if isUnlocked {
+				dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+			}
+			return
+		}
+		movedTo = primary
+		return
+	}
+	var isUnlocked bool
+	var generation int64
+	var seq int64
+	isUnlocked, generation, seq, err = dlm.lockManager.Unlock(key, token)
+	if isUnlocked {
+		dlm.replicateToBackup(key, 0, "", "", generation, seq, true)
+	}
+	return
+}
+
+// InsertLock is used to insert a lock to a server unconditionally.
+// It is used when a server is down and the lock is moved to another server.
+// After inserting, it replicates to the backup for this key.
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	if dlm.lockManager.InsertLock(key, expiredAtNs, token, owner, generation, seq) {
+		dlm.replicateToBackup(key, expiredAtNs, token, owner, generation, seq, false)
+	}
+}
+
+// InsertBackupLock inserts a lock as a backup copy, rejecting stale seq
+func (dlm *DistributedLockManager) InsertBackupLock(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	dlm.lockManager.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+}
+
+// RemoveBackupLock removes a backup lock unconditionally
+func (dlm *DistributedLockManager) RemoveBackupLock(key string) {
+	dlm.lockManager.RemoveLock(key)
+}
+
+// RemoveBackupLockIfSeq removes a local copy only if the incoming mutation is not older.
+func (dlm *DistributedLockManager) RemoveBackupLockIfSeq(key string, generation int64, seq int64) {
+	dlm.lockManager.RemoveBackupLockIfSeq(key, generation, seq)
+}
+
+func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
+	return dlm.lockManager.SelectLocks(func(key string) bool {
+		server := hashKeyToServer(key, servers)
+		return server != dlm.Host
+	})
+}
+func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
+	return hashKeyToServer(key, servers)
+}
+
+func (dlm *DistributedLockManager) IsLocal(key string) bool {
+	primary := dlm.LockRing.GetPrimary(key)
+	if primary == "" {
+		return true
+	}
+	return primary == dlm.Host
+}
+
+// AllLocks returns all non-expired locks on this node
+func (dlm *DistributedLockManager) AllLocks() []*Lock {
+	return dlm.lockManager.AllLocks()
+}
+
+// PromoteLock promotes a backup lock to primary
+func (dlm *DistributedLockManager) PromoteLock(key string) bool {
+	return dlm.lockManager.PromoteLock(key)
+}
+
+// DemoteLock demotes a primary lock to backup
+func (dlm *DistributedLockManager) DemoteLock(key string) bool {
+	return dlm.lockManager.DemoteLock(key)
+}
+
+// GetLock returns a copy of a lock if it exists
+func (dlm *DistributedLockManager) GetLock(key string) (*Lock, bool) {
+	return dlm.lockManager.GetLock(key)
+}
+
+// replicateToBackup asynchronously replicates a lock operation to the backup server
+func (dlm *DistributedLockManager) replicateToBackup(key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+	_, backup := dlm.LockRing.GetPrimaryAndBackup(key)
+	if backup == "" {
+		return // single-server deployment, no backup
+	}
+	if dlm.ReplicateFn != nil {
+		go dlm.ReplicateFn(backup, key, expiredAtNs, token, owner, generation, seq, isUnlock)
+	}
+}

weed/cluster/lock_manager/distributed_lock_manager_test.go

@@ -0,0 +1,710 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCluster simulates a cluster of filer nodes with DLMs.
+// It wires up ReplicateFn so that replication calls arrive at the
+// correct peer's DLM, enabling end-to-end backup testing without gRPC.
+type testCluster struct {
+	mu    sync.Mutex
+	nodes map[pb.ServerAddress]*DistributedLockManager
+}
+
+func newTestCluster(hosts ...pb.ServerAddress) *testCluster {
+	c := &testCluster{nodes: make(map[pb.ServerAddress]*DistributedLockManager)}
+	servers := make([]pb.ServerAddress, len(hosts))
+	copy(servers, hosts)
+
+	for _, host := range hosts {
+		dlm := NewDistributedLockManager(host)
+		dlm.LockRing.SetSnapshot(servers, 0)
+		c.nodes[host] = dlm
+	}
+
+	// Wire up replication: each node's ReplicateFn calls the backup's DLM directly
+	for _, dlm := range c.nodes {
+		d := dlm // capture
+		d.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			c.mu.Lock()
+			target, ok := c.nodes[server]
+			c.mu.Unlock()
+			if !ok {
+				return // server is down
+			}
+			if isUnlock {
+				target.RemoveBackupLockIfSeq(key, generation, seq)
+			} else {
+				target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+			}
+		}
+	}
+
+	return c
+}
+
+func (c *testCluster) removeNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	delete(c.nodes, host)
+	c.mu.Unlock()
+
+	// Update all remaining nodes' rings
+	remaining := c.getServers()
+	for _, dlm := range c.getNodes() {
+		dlm.LockRing.SetSnapshot(remaining, 0)
+	}
+}
+
+func (c *testCluster) addNode(host pb.ServerAddress) {
+	c.mu.Lock()
+	dlm := NewDistributedLockManager(host)
+	c.nodes[host] = dlm
+	c.mu.Unlock()
+
+	// Wire up replication
+	dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+		c.mu.Lock()
+		target, ok := c.nodes[server]
+		c.mu.Unlock()
+		if !ok {
+			return
+		}
+		if isUnlock {
+			target.RemoveBackupLockIfSeq(key, generation, seq)
+		} else {
+			target.InsertBackupLock(key, expiredAtNs, token, owner, generation, seq)
+		}
+	}
+
+	servers := c.getServers()
+	for _, n := range c.getNodes() {
+		n.LockRing.SetSnapshot(servers, 0)
+	}
+}
+
+func (c *testCluster) getNodes() map[pb.ServerAddress]*DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	cp := make(map[pb.ServerAddress]*DistributedLockManager, len(c.nodes))
+	for k, v := range c.nodes {
+		cp[k] = v
+	}
+	return cp
+}
+
+func (c *testCluster) getServers() []pb.ServerAddress {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	var servers []pb.ServerAddress
+	for s := range c.nodes {
+		servers = append(servers, s)
+	}
+	return servers
+}
+
+func (c *testCluster) get(host pb.ServerAddress) *DistributedLockManager {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[host]
+}
+
+// acquireLock tries to acquire a lock on the correct primary node.
+// It follows redirects (movedTo) like a real client would.
+func (c *testCluster) acquireLock(key, owner string, ttl time.Duration) (renewToken string, generation int64, primaryHost pb.ServerAddress, err error) {
+	// Try any node first (simulates client connecting to seed filer)
+	for _, dlm := range c.getNodes() {
+		expiry := time.Now().Add(ttl).UnixNano()
+		var movedTo pb.ServerAddress
+		var lockErr error
+		_, renewToken, generation, movedTo, lockErr = dlm.LockWithTimeout(key, expiry, "", owner)
+		if movedTo != "" && movedTo != dlm.Host {
+			// Follow redirect
+			target := c.get(movedTo)
+			if target == nil {
+				err = fmt.Errorf("primary %s is down", movedTo)
+				return
+			}
+			_, renewToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, "", owner)
+			if lockErr != nil {
+				err = lockErr
+				return
+			}
+			primaryHost = movedTo
+			// Wait briefly for async replication to complete
+			time.Sleep(10 * time.Millisecond)
+			return
+		}
+		if lockErr != nil {
+			err = lockErr
+			return
+		}
+		primaryHost = dlm.Host
+		time.Sleep(10 * time.Millisecond)
+		return
+	}
+	err = fmt.Errorf("no nodes available")
+	return
+}
+
+// renewLock renews a lock on the primary node
+func (c *testCluster) renewLock(key, owner, token string, ttl time.Duration, primaryHost pb.ServerAddress) (newToken string, generation int64, err error) {
+	target := c.get(primaryHost)
+	if target == nil {
+		err = fmt.Errorf("primary %s is down", primaryHost)
+		return
+	}
+	expiry := time.Now().Add(ttl).UnixNano()
+	var movedTo pb.ServerAddress
+	var lockErr error
+	_, newToken, generation, movedTo, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	if movedTo != "" && movedTo != primaryHost {
+		target = c.get(movedTo)
+		if target == nil {
+			err = fmt.Errorf("new primary %s is down", movedTo)
+			return
+		}
+		// Pass the existing token so the redirected renewal can match
+		// if the lock was already transferred to the new primary.
+		_, newToken, generation, _, lockErr = target.LockWithTimeout(key, expiry, token, owner)
+	}
+	err = lockErr
+	time.Sleep(10 * time.Millisecond)
+	return
+}
+
+// --- Test Cases ---
+
+func TestDLM_PrimaryCrash_BackupPromotes(t *testing.T) {
+	// Scenario: Lock is acquired, primary crashes, backup should have the lock
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-primary-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+	require.NotEmpty(t, renewToken)
+
+	// Find the backup for this key
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+	require.NotEmpty(t, backup, "should have a backup server")
+
+	// Verify backup has the lock
+	backupDlm := cluster.get(backup)
+	backupLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "backup should have the lock")
+	assert.True(t, backupLock.IsBackup, "lock on backup should be marked as backup")
+	assert.Equal(t, renewToken, backupLock.Token, "backup should have the same token")
+
+	// Crash the primary
+	cluster.removeNode(primaryHost)
+
+	// Simulate topology change: promote backup locks
+	for _, dlm := range cluster.getNodes() {
+		locks := dlm.AllLocks()
+		for _, lock := range locks {
+			newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if newPrimary == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+	}
+
+	// The backup should now be the primary
+	newPrimary := backupDlm.LockRing.GetPrimary(key)
+	assert.Equal(t, backup, newPrimary, "backup should be the new primary")
+
+	// The promoted lock should work — verify it's no longer a backup
+	promotedLock, found := backupDlm.GetLock(key)
+	require.True(t, found, "lock should still exist after promotion")
+	assert.False(t, promotedLock.IsBackup, "lock should be promoted to primary")
+
+	// Client should be able to renew with the same token on the new primary
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, backup)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "renewal on new primary should succeed")
+}
+
+func TestDLM_BackupCrash_PrimaryContinues(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-backup-crash"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash the backup
+	cluster.removeNode(backup)
+
+	// Primary should still work — renew the lock
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "primary should continue working after backup crash")
+
+	// Verify primary is still the primary for this key
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	assert.Equal(t, primaryHost, newPrimary)
+}
+
+func TestDLM_BothPrimaryAndBackupCrash(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "test-lock-both-crash"
+	_, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Crash both
+	cluster.removeNode(primaryHost)
+	cluster.removeNode(backup)
+
+	// The lock is lost — the surviving node should be able to acquire it fresh
+	newToken, _, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken, "new owner should acquire lock after both crash")
+}
+
+func TestDLM_RollingRestart(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire multiple locks
+	type lockState struct {
+		key, owner, token string
+		generation        int64
+		primary           pb.ServerAddress
+	}
+	locks := make([]lockState, 5)
+	for i := range locks {
+		key := fmt.Sprintf("rolling-lock-%d", i)
+		token, gen, primary, err := cluster.acquireLock(key, fmt.Sprintf("owner-%d", i), 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockState{key: key, owner: fmt.Sprintf("owner-%d", i), token: token, generation: gen, primary: primary}
+	}
+
+	// Rolling restart: remove and re-add each node one at a time.
+	// After removing a node, promote backups and re-replicate to new backups
+	// to maintain the invariant that each lock has a backup copy.
+	for _, host := range hosts {
+		cluster.removeNode(host)
+
+		// Simulate full OnDlmChangeSnapshot: promote backups and re-replicate
+		for _, dlm := range cluster.getNodes() {
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && lock.IsBackup {
+					dlm.PromoteLock(lock.Key)
+				}
+			}
+			// Re-replicate all primary locks to their new backups
+			for _, lock := range dlm.AllLocks() {
+				newPrimary, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+				if newPrimary == dlm.Host && !lock.IsBackup {
+					dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Millisecond)
+
+		// Re-add the node
+		cluster.addNode(host)
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	// After rolling restart, locks should survive via backup promotion
+	survivedCount := 0
+	for _, ls := range locks {
+		for _, dlm := range cluster.getNodes() {
+			lock, found := dlm.GetLock(ls.key)
+			if found && !lock.IsBackup {
+				survivedCount++
+				break
+			}
+		}
+	}
+	t.Logf("Locks survived rolling restart: %d / %d", survivedCount, len(locks))
+	require.Greater(t, survivedCount, 0, "at least some locks should survive a rolling restart via backup promotion")
+}
+
+func TestDLM_GenerationIncrementsOnNewAcquisition(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "gen-test-lock"
+
+	// Acquire lock — generation should be > 0
+	token1, gen1, primary, err := cluster.acquireLock(key, "owner1", 2*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen1, int64(0))
+
+	// Renew — generation should stay the same
+	token2, gen2, err := cluster.renewLock(key, "owner1", token1, 2*time.Second, primary)
+	require.NoError(t, err)
+	assert.Equal(t, gen1, gen2, "generation should not change on renewal")
+
+	// Let lock expire
+	time.Sleep(3 * time.Second)
+
+	// Re-acquire — generation should increment
+	_, gen3, _, err := cluster.acquireLock(key, "owner2", 30*time.Second)
+	require.NoError(t, err)
+	assert.Greater(t, gen3, gen1, "generation should increment on new acquisition")
+	_ = token2
+}
+
+func TestDLM_ReplicationFailure_PrimaryStillWorks(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Break replication by setting a no-op ReplicateFn on all nodes
+	for _, dlm := range cluster.getNodes() {
+		dlm.ReplicateFn = func(server pb.ServerAddress, key string, expiredAtNs int64, token string, owner string, generation int64, seq int64, isUnlock bool) {
+			// Simulate replication failure: do nothing
+		}
+	}
+
+	key := "repl-fail-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Primary should have the lock
+	primaryDlm := cluster.get(primaryHost)
+	lock, found := primaryDlm.GetLock(key)
+	require.True(t, found, "primary should have the lock")
+	assert.False(t, lock.IsBackup)
+
+	// Backup should NOT have it (replication failed)
+	_, backup := primaryDlm.LockRing.GetPrimaryAndBackup(key)
+	backupDlm := cluster.get(backup)
+	_, found = backupDlm.GetLock(key)
+	assert.False(t, found, "backup should not have the lock when replication fails")
+
+	// Primary should still be able to renew
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err)
+	assert.NotEmpty(t, newToken)
+}
+
+func TestDLM_UnlockReplicatesToBackup(t *testing.T) {
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "unlock-repl-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	_, backup := cluster.get(primaryHost).LockRing.GetPrimaryAndBackup(key)
+
+	// Verify backup has the lock
+	_, found := cluster.get(backup).GetLock(key)
+	require.True(t, found, "backup should have the lock")
+
+	// Unlock on primary
+	primaryDlm := cluster.get(primaryHost)
+	movedTo, err := primaryDlm.Unlock(key, renewToken)
+	require.NoError(t, err)
+	assert.Empty(t, movedTo)
+
+	// Wait for async replication
+	time.Sleep(20 * time.Millisecond)
+
+	// Backup should also have removed the lock
+	_, found = cluster.get(backup).GetLock(key)
+	assert.False(t, found, "backup should remove lock after unlock replication")
+}
+
+func TestDLM_TopologyChange_LockSurvivesServerAddition(t *testing.T) {
+	// Start with 2 servers, acquire lock, add a 3rd server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888"}
+	cluster := newTestCluster(hosts...)
+
+	key := "topo-add-lock"
+	renewToken, _, primaryHost, err := cluster.acquireLock(key, "owner1", 30*time.Second)
+	require.NoError(t, err)
+
+	// Add a new server
+	cluster.addNode("filer3:8888")
+	time.Sleep(20 * time.Millisecond)
+
+	// The lock should still be accessible — either the same primary or on a new one
+	// Try to renew on the original primary first
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	if newPrimary == primaryHost {
+		// Still on same primary
+		newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+		require.NoError(t, err)
+		assert.NotEmpty(t, newToken)
+	}
+	// If primary changed, the lock may need transfer — that's handled by OnDlmChangeSnapshot
+	// which is tested at the server level
+}
+
+func TestDLM_ConsistentHashing_MinimalDisruption(t *testing.T) {
+	// Verify that removing a server only affects locks on that server
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire 50 locks
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, 50)
+	for i := range locks {
+		key := fmt.Sprintf("min-disrupt-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Count locks per server before removal
+	countBefore := make(map[pb.ServerAddress]int)
+	for _, l := range locks {
+		countBefore[l.primary]++
+	}
+	t.Logf("Lock distribution before: %v", countBefore)
+
+	// Remove filer2
+	cluster.removeNode("filer2:8888")
+
+	// Count how many locks changed primary
+	changed := 0
+	for _, l := range locks {
+		// Check where the lock should be now
+		for _, dlm := range cluster.getNodes() {
+			newPrimary := dlm.LockRing.GetPrimary(l.key)
+			if newPrimary != l.primary {
+				changed++
+			}
+			break
+		}
+	}
+
+	// Only locks from filer2 should have changed
+	assert.Equal(t, countBefore["filer2:8888"], changed,
+		"only locks from removed server should change primary")
+}
+
+func TestDLM_NodeDropAndJoin_OwnershipDisruption(t *testing.T) {
+	// Scenario: 3 nodes, acquire locks, one drops and a NEW node joins quickly.
+	// The new node steals hash ranges from surviving nodes, not just from the
+	// departed node. This test measures the disruption.
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Acquire many locks
+	numLocks := 100
+	type lockInfo struct {
+		key, token string
+		primary    pb.ServerAddress
+	}
+	locks := make([]lockInfo, numLocks)
+	for i := range locks {
+		key := fmt.Sprintf("churn-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(key, "owner", 30*time.Second)
+		require.NoError(t, err)
+		locks[i] = lockInfo{key: key, token: token, primary: primary}
+	}
+
+	// Record primary for each lock before the change
+	beforePrimary := make(map[string]pb.ServerAddress)
+	for _, l := range locks {
+		beforePrimary[l.key] = l.primary
+	}
+
+	// Drop filer3 and immediately add filer4
+	cluster.removeNode("filer3:8888")
+
+	// Promote backups on remaining nodes (simulates OnDlmChangeSnapshot)
+	for _, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && lock.IsBackup {
+				dlm.PromoteLock(lock.Key)
+			}
+		}
+		// Re-replicate primary locks to new backups
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p == dlm.Host && !lock.IsBackup {
+				dlm.replicateToBackup(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq, false)
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Now add filer4 (new node, empty)
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	// Simulate OnDlmChangeSnapshot on all nodes after filer4 joins:
+	// transfer locks that now belong to filer4
+	for host, dlm := range cluster.getNodes() {
+		for _, lock := range dlm.AllLocks() {
+			p, _ := dlm.LockRing.GetPrimaryAndBackup(lock.Key)
+			if p != host && !lock.IsBackup {
+				// This lock should move to the new primary
+				target := cluster.get(p)
+				if target != nil {
+					target.InsertLock(lock.Key, lock.ExpiredAtNs, lock.Token, lock.Owner, lock.Generation, lock.Seq)
+					dlm.DemoteLock(lock.Key)
+				}
+			}
+		}
+	}
+	time.Sleep(10 * time.Millisecond)
+
+	// Count disruptions: locks whose primary changed to a node other than filer3's successor
+	disruptedFromSurvivors := 0
+	disruptedFromDeparted := 0
+	movedToFiler4 := 0
+	for _, l := range locks {
+		// What's the new primary?
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		oldPrimary := beforePrimary[l.key]
+		if newPrimary != oldPrimary {
+			if oldPrimary == "filer3:8888" {
+				disruptedFromDeparted++
+			} else {
+				disruptedFromSurvivors++
+			}
+		}
+		if newPrimary == "filer4:8888" {
+			movedToFiler4++
+		}
+	}
+
+	t.Logf("Locks disrupted from departed filer3: %d / %d", disruptedFromDeparted, numLocks)
+	t.Logf("Locks disrupted from surviving filer1/filer2: %d / %d", disruptedFromSurvivors, numLocks)
+	t.Logf("Locks now on new filer4: %d / %d", movedToFiler4, numLocks)
+
+	// The key concern: filer4 joining disrupts locks on surviving nodes
+	// With consistent hashing, new node steals ~1/N of each surviving node's keys
+	// Verify that the transfer logic above moved those locks to filer4
+	for _, l := range locks {
+		var newPrimary pb.ServerAddress
+		for _, dlm := range cluster.getNodes() {
+			newPrimary = dlm.LockRing.GetPrimary(l.key)
+			break
+		}
+		target := cluster.get(newPrimary)
+		require.NotNil(t, target, "primary %s should exist", newPrimary)
+
+		lock, found := target.GetLock(l.key)
+		if !found {
+			// Lock may have only a backup copy if transfer happened but
+			// the lock was on the departed node and wasn't re-replicated.
+			// Check all nodes for any copy.
+			anyFound := false
+			for _, dlm := range cluster.getNodes() {
+				if _, f := dlm.GetLock(l.key); f {
+					anyFound = true
+					break
+				}
+			}
+			if !anyFound {
+				t.Errorf("lock %s completely lost (primary should be %s)", l.key, newPrimary)
+			}
+			continue
+		}
+		assert.False(t, lock.IsBackup, "lock %s on primary %s should not be a backup", l.key, newPrimary)
+	}
+}
+
+func TestDLM_RenewalDuringTransferWindow(t *testing.T) {
+	// When a new node joins and steals a key range from a surviving node,
+	// there's a window between ring update and lock transfer. During this
+	// window, a client renewal should still succeed on the old primary
+	// (because it still holds the lock locally).
+	hosts := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	cluster := newTestCluster(hosts...)
+
+	// Find a key that will move primary when filer4 is added.
+	// Try candidate keys until we find one whose primary changes.
+	var key, renewToken string
+	var primaryHost pb.ServerAddress
+	for i := 0; i < 1000; i++ {
+		candidate := fmt.Sprintf("transfer-window-lock-%d", i)
+		token, _, primary, err := cluster.acquireLock(candidate, "owner1", 30*time.Second)
+		require.NoError(t, err)
+
+		// Check if adding filer4 would move this key's primary
+		tmpRing := NewHashRing(DefaultVnodeCount)
+		tmpRing.SetServers([]pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888", "filer4:8888"})
+		newPrimary := tmpRing.GetPrimary(candidate)
+		if newPrimary != primary {
+			key = candidate
+			renewToken = token
+			primaryHost = primary
+			break
+		}
+	}
+	require.NotEmpty(t, key, "should find a key that moves primary when filer4 joins")
+
+	// Add filer4 — this changes the primary for our key per the ring
+	cluster.addNode("filer4:8888")
+	time.Sleep(10 * time.Millisecond)
+
+	newPrimary := cluster.get(primaryHost).LockRing.GetPrimary(key)
+	require.NotEqual(t, primaryHost, newPrimary, "key should have moved to a different primary")
+
+	// Renewal on the OLD primary should still succeed because it holds the lock locally
+	newToken, _, err := cluster.renewLock(key, "owner1", renewToken, 30*time.Second, primaryHost)
+	require.NoError(t, err, "renewal on old primary should succeed during transfer window")
+	assert.NotEmpty(t, newToken, "should get a new token from old primary")
+	t.Logf("Key %s: primary changed from %s to %s, but renewal on old primary succeeded", key, primaryHost, newPrimary)
+}
+
+func TestDLM_StaleReplicationRejected(t *testing.T) {
+	// Verify that a stale replication (lower seq) does not overwrite a newer one
+	lm := NewLockManager()
+
+	// Insert backup with seq=3
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 1, 3)
+	lock, found := lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Try to overwrite with stale seq=2 — should be rejected
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 1, 2)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-new", lock.Token, "stale replication should be rejected")
+	assert.Equal(t, int64(3), lock.Seq)
+
+	// Update with higher seq=4 — should succeed
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-newer", "owner1", 1, 4)
+	lock, found = lm.GetLock("key1")
+	require.True(t, found)
+	assert.Equal(t, "token-newer", lock.Token, "newer replication should be accepted")
+	assert.Equal(t, int64(4), lock.Seq)
+
+	// Stale unlock (seq=2) should not delete the lock
+	removed := lm.RemoveBackupLockIfSeq("key1", 1, 2)
+	assert.False(t, removed, "stale unlock should be rejected")
+	_, found = lm.GetLock("key1")
+	assert.True(t, found, "lock should still exist after stale unlock")
+
+	// Valid unlock (seq=5) should delete
+	removed = lm.RemoveBackupLockIfSeq("key1", 1, 5)
+	assert.True(t, removed, "valid unlock should be accepted")
+	_, found = lm.GetLock("key1")
+	assert.False(t, found, "lock should be removed after valid unlock")
+}

weed/cluster/lock_manager/hash_ring.go

@@ -0,0 +1,206 @@
+package lock_manager
+
+import (
+	"hash/crc32"
+	"sort"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+const DefaultVnodeCount = 50
+
+// HashRing implements consistent hashing with virtual nodes.
+// When a server is removed, only the keys that hashed to that server
+// are remapped (to the next server on the ring), leaving all other
+// key-to-server mappings stable.
+//
+// UPGRADE NOTE: This replaces the previous modulo-based hashing
+// (hash % len(servers)). The two schemes compute different primaries
+// for the same key, so all filer nodes in the cluster must be upgraded
+// together (or via a rolling restart that completes within the lock TTL
+// window of ~7 seconds) to avoid routing disagreements.
+type HashRing struct {
+	mu            sync.RWMutex
+	vnodeCount    int
+	sortedHashes  []uint32                      // sorted ring positions
+	vnodeToServer map[uint32]pb.ServerAddress   // ring position → server
+	servers       map[pb.ServerAddress]struct{} // set of all servers
+}
+
+func NewHashRing(vnodeCount int) *HashRing {
+	if vnodeCount <= 0 {
+		vnodeCount = DefaultVnodeCount
+	}
+	return &HashRing{
+		vnodeCount:    vnodeCount,
+		vnodeToServer: make(map[uint32]pb.ServerAddress),
+		servers:       make(map[pb.ServerAddress]struct{}),
+	}
+}
+
+// AddServer adds a server with virtual nodes to the ring.
+func (hr *HashRing) AddServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; exists {
+		return
+	}
+	hr.servers[server] = struct{}{}
+	hr.rebuildRing()
+}
+
+// RemoveServer removes a server and its virtual nodes from the ring.
+func (hr *HashRing) RemoveServer(server pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	if _, exists := hr.servers[server]; !exists {
+		return
+	}
+	delete(hr.servers, server)
+	hr.rebuildRing()
+}
+
+// SetServers replaces the entire server set.
+func (hr *HashRing) SetServers(servers []pb.ServerAddress) {
+	hr.mu.Lock()
+	defer hr.mu.Unlock()
+
+	hr.servers = make(map[pb.ServerAddress]struct{}, len(servers))
+	for _, s := range servers {
+		hr.servers[s] = struct{}{}
+	}
+	hr.rebuildRing()
+}
+
+// GetPrimaryAndBackup returns the primary server for a key and its backup
+// (the next distinct server clockwise on the ring).
+// If there is only one server, backup is empty.
+func (hr *HashRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return "", ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	primary = hr.vnodeToServer[hr.sortedHashes[idx]]
+
+	// Walk clockwise to find a different server for backup
+	ringLen := len(hr.sortedHashes)
+	for i := 1; i < ringLen; i++ {
+		candidate := hr.vnodeToServer[hr.sortedHashes[(idx+i)%ringLen]]
+		if candidate != primary {
+			backup = candidate
+			return
+		}
+	}
+	// Only one server — no backup
+	return primary, ""
+}
+
+// GetPrimary returns just the primary server for a key.
+func (hr *HashRing) GetPrimary(key string) pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	if len(hr.sortedHashes) == 0 {
+		return ""
+	}
+
+	hash := hashKey(key)
+	idx := hr.search(hash)
+	return hr.vnodeToServer[hr.sortedHashes[idx]]
+}
+
+// GetServers returns a sorted copy of all servers in the ring.
+func (hr *HashRing) GetServers() []pb.ServerAddress {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for s := range hr.servers {
+		servers = append(servers, s)
+	}
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+	return servers
+}
+
+// ServerCount returns the number of servers in the ring.
+func (hr *HashRing) ServerCount() int {
+	hr.mu.RLock()
+	defer hr.mu.RUnlock()
+	return len(hr.servers)
+}
+
+// rebuildRing rebuilds the sorted hash ring from the current server set.
+// Caller must hold hr.mu write lock.
+func (hr *HashRing) rebuildRing() {
+	hr.vnodeToServer = make(map[uint32]pb.ServerAddress, len(hr.servers)*hr.vnodeCount)
+	hr.sortedHashes = make([]uint32, 0, len(hr.servers)*hr.vnodeCount)
+
+	// Sort so a vnode-hash collision resolves to the same server on every node;
+	// map iteration order alone is randomized per process.
+	servers := make([]pb.ServerAddress, 0, len(hr.servers))
+	for server := range hr.servers {
+		servers = append(servers, server)
+	}
+	sort.Slice(servers, func(i, j int) bool { return servers[i] < servers[j] })
+
+	for _, server := range servers {
+		for i := 0; i < hr.vnodeCount; i++ {
+			vnodeKey := vnodeKeyFor(server, i)
+			hash := hashKey(vnodeKey)
+			hr.vnodeToServer[hash] = server
+			hr.sortedHashes = append(hr.sortedHashes, hash)
+		}
+	}
+	sort.Slice(hr.sortedHashes, func(i, j int) bool {
+		return hr.sortedHashes[i] < hr.sortedHashes[j]
+	})
+}
+
+// search finds the first ring position >= hash.
+func (hr *HashRing) search(hash uint32) int {
+	idx := sort.Search(len(hr.sortedHashes), func(i int) bool {
+		return hr.sortedHashes[i] >= hash
+	})
+	if idx >= len(hr.sortedHashes) {
+		idx = 0 // wrap around
+	}
+	return idx
+}
+
+func hashKey(key string) uint32 {
+	return crc32.ChecksumIEEE([]byte(key))
+}
+
+func vnodeKeyFor(server pb.ServerAddress, index int) string {
+	// Use a format that distributes well across the ring
+	buf := make([]byte, 0, len(server)+10)
+	buf = append(buf, []byte(server)...)
+	buf = append(buf, '#')
+	buf = appendInt(buf, index)
+	return string(buf)
+}
+
+func appendInt(buf []byte, n int) []byte {
+	if n == 0 {
+		return append(buf, '0')
+	}
+	// Simple int-to-string without importing strconv
+	digits := [20]byte{}
+	pos := len(digits)
+	for n > 0 {
+		pos--
+		digits[pos] = byte('0' + n%10)
+		n /= 10
+	}
+	return append(buf, digits[pos:]...)
+}

weed/cluster/lock_manager/hash_ring_test.go

@@ -0,0 +1,208 @@
+package lock_manager
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHashRing_BasicOperations(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	p, b := hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress(""), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Single server — no backup
+	hr.AddServer("filer1:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.Equal(t, pb.ServerAddress("filer1:8888"), p)
+	assert.Equal(t, pb.ServerAddress(""), b)
+
+	// Two servers — backup is the other server
+	hr.AddServer("filer2:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Three servers
+	hr.AddServer("filer3:8888")
+	p, b = hr.GetPrimaryAndBackup("key1")
+	assert.NotEqual(t, p, b)
+	assert.NotEmpty(t, b)
+
+	// Remove server
+	hr.RemoveServer("filer2:8888")
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_DuplicateAddRemove(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.AddServer("filer1:8888")
+	hr.AddServer("filer1:8888") // duplicate
+	assert.Equal(t, 1, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888")
+	assert.Equal(t, 0, hr.ServerCount())
+
+	hr.RemoveServer("filer1:8888") // remove non-existent
+	assert.Equal(t, 0, hr.ServerCount())
+}
+
+func TestHashRing_SetServers(t *testing.T) {
+	hr := NewHashRing(50)
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2", "c:3"})
+	assert.Equal(t, 3, hr.ServerCount())
+
+	servers := hr.GetServers()
+	assert.Equal(t, 3, len(servers))
+
+	// SetServers replaces
+	hr.SetServers([]pb.ServerAddress{"x:1", "y:2"})
+	assert.Equal(t, 2, hr.ServerCount())
+}
+
+func TestHashRing_ConsistencyOnRemoval(t *testing.T) {
+	// The key property of consistent hashing: when a server is removed,
+	// only keys that mapped to the removed server change.
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	numKeys := 1000
+	// Record where each key maps before removal
+	before := make(map[string]pb.ServerAddress, numKeys)
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		before[key] = hr.GetPrimary(key)
+	}
+
+	// Remove filer2
+	hr.RemoveServer("filer2:8888")
+
+	moved := 0
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("lock-key-%d", i)
+		after := hr.GetPrimary(key)
+		if before[key] != after {
+			// Only keys from filer2 should move
+			assert.Equal(t, pb.ServerAddress("filer2:8888"), before[key],
+				"key %s moved from %s to %s, but it wasn't on the removed server", key, before[key], after)
+			moved++
+		}
+	}
+	// Roughly 1/3 of keys should move (those that were on filer2)
+	t.Logf("Keys that moved: %d / %d", moved, numKeys)
+	assert.Greater(t, moved, 0, "some keys should have moved")
+	assert.Less(t, moved, numKeys, "not all keys should move")
+}
+
+func TestHashRing_BackupIsSuccessor(t *testing.T) {
+	// After removing primary, the backup should become the new primary
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	// For each key, verify that removing the primary makes the backup the new primary
+	promoted := 0
+	total := 500
+	for i := 0; i < total; i++ {
+		key := fmt.Sprintf("test-lock-%d", i)
+		primary, backup := hr.GetPrimaryAndBackup(key)
+		assert.NotEqual(t, primary, backup)
+
+		// Temporarily remove primary
+		hr.RemoveServer(primary)
+		newPrimary := hr.GetPrimary(key)
+		if newPrimary == backup {
+			promoted++
+		}
+		// Restore
+		hr.AddServer(primary)
+	}
+	// The backup should become new primary for all keys
+	assert.Equal(t, total, promoted,
+		"backup should become new primary for all keys when primary is removed")
+}
+
+func TestHashRing_Distribution(t *testing.T) {
+	hr := NewHashRing(50)
+	servers := []pb.ServerAddress{"filer1:8888", "filer2:8888", "filer3:8888"}
+	hr.SetServers(servers)
+
+	counts := make(map[pb.ServerAddress]int)
+	numKeys := 3000
+	for i := 0; i < numKeys; i++ {
+		key := fmt.Sprintf("dist-key-%d", i)
+		p := hr.GetPrimary(key)
+		counts[p]++
+	}
+
+	expected := float64(numKeys) / float64(len(servers))
+	for server, count := range counts {
+		deviation := math.Abs(float64(count)-expected) / expected
+		t.Logf("Server %s: %d keys (%.1f%% deviation)", server, count, deviation*100)
+		// Allow up to 40% deviation with 50 vnodes and 3 servers
+		assert.Less(t, deviation, 0.40,
+			"server %s has too many or too few keys: %d (expected ~%d)", server, count, int(expected))
+	}
+}
+
+func TestHashRing_GetPrimary(t *testing.T) {
+	hr := NewHashRing(50)
+
+	// Empty ring
+	assert.Equal(t, pb.ServerAddress(""), hr.GetPrimary("key"))
+
+	hr.SetServers([]pb.ServerAddress{"a:1", "b:2"})
+
+	// Deterministic: same key always maps to same server
+	p1 := hr.GetPrimary("mykey")
+	p2 := hr.GetPrimary("mykey")
+	assert.Equal(t, p1, p2)
+
+	// GetPrimary matches the primary from GetPrimaryAndBackup
+	primary, _ := hr.GetPrimaryAndBackup("mykey")
+	assert.Equal(t, primary, hr.GetPrimary("mykey"))
+}
+
+// The ring must be identical on every node holding the same server set,
+// regardless of the order servers were added or supplied. Build rings several
+// ways and assert they agree on the primary for a wide range of keys.
+func TestHashRing_OrderIndependent(t *testing.T) {
+	servers := []pb.ServerAddress{
+		"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888", "filer-e:8888",
+	}
+
+	bySet := NewHashRing(50)
+	bySet.SetServers(servers)
+
+	byReverse := NewHashRing(50)
+	rev := append([]pb.ServerAddress(nil), servers...)
+	for i, j := 0, len(rev)-1; i < j; i, j = i+1, j-1 {
+		rev[i], rev[j] = rev[j], rev[i]
+	}
+	byReverse.SetServers(rev)
+
+	byAdd := NewHashRing(50)
+	for _, s := range []pb.ServerAddress{"filer-c:8888", "filer-e:8888", "filer-a:8888", "filer-d:8888", "filer-b:8888"} {
+		byAdd.AddServer(s)
+	}
+
+	for i := 0; i < 5000; i++ {
+		key := fmt.Sprintf("s3.object.write:/buckets/b/obj-%d", i)
+		p := bySet.GetPrimary(key)
+		if got := byReverse.GetPrimary(key); got != p {
+			t.Fatalf("reverse-order ring disagrees on %q: %s vs %s", key, got, p)
+		}
+		if got := byAdd.GetPrimary(key); got != p {
+			t.Fatalf("add-order ring disagrees on %q: %s vs %s", key, got, p)
+		}
+	}
+}

weed/cluster/lock_manager/lock_manager.go

@@ -0,0 +1,353 @@
+package lock_manager
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new lock")
+var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on an expired lock")
+var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
+var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
+var LockNotFound = fmt.Errorf("lock not found")
+
+// LockManager local lock manager, used by distributed lock manager
+type LockManager struct {
+	locks          map[string]*Lock
+	accessLock     sync.RWMutex
+	nextGeneration atomic.Int64
+}
+type Lock struct {
+	Token       string
+	ExpiredAtNs int64
+	Key         string // only used for moving locks
+	Owner       string
+	IsBackup    bool  // true if this node holds the lock as a backup
+	Generation  int64 // monotonic fencing token, increments on fresh acquisition
+	Seq         int64 // per-lock sequence number, increments on every mutation (acquire/renew/unlock)
+}
+
+func NewLockManager() *LockManager {
+	t := &LockManager{
+		locks: make(map[string]*Lock),
+	}
+	go t.CleanUp()
+	return t
+}
+
+func (lm *LockManager) NextGeneration() int64 {
+	return lm.nextGeneration.Add(1)
+}
+
+func compareMutationVersion(generation, seq int64, existingGeneration, existingSeq int64) int {
+	switch {
+	case generation < existingGeneration:
+		return -1
+	case generation > existingGeneration:
+		return 1
+	case seq < existingSeq:
+		return -1
+	case seq > existingSeq:
+		return 1
+	default:
+		return 0
+	}
+}
+
+func (lm *LockManager) advanceGenerationFloor(generation int64) {
+	for {
+		current := lm.nextGeneration.Load()
+		if generation < current {
+			return
+		}
+		if lm.nextGeneration.CompareAndSwap(current, generation) {
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, generation int64, seq int64, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	glog.V(4).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
+
+	if oldValue, found := lm.locks[path]; found {
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
+			// lock is expired, set to a new lock
+			if token != "" {
+				glog.V(4).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
+				err = LockErrorNonEmptyTokenOnExpiredLock
+				return
+			} else {
+				// new lock
+				renewToken = uuid.New().String()
+				generation = lm.NextGeneration()
+				seq = 1
+				glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+				return
+			}
+		}
+		// not expired
+		lockOwner = oldValue.Owner
+		if oldValue.Token == token {
+			// token matches, renew the lock
+			renewToken = uuid.New().String()
+			generation = oldValue.Generation // keep same generation on renewal
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+			return
+		} else {
+			if token == "" {
+				// new lock
+				glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
+				err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
+				return
+			}
+			glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
+			err = fmt.Errorf("lock: token mismatch")
+			return
+		}
+	} else {
+		glog.V(4).Infof("key %s no lock owner %v", path, owner)
+		if token == "" {
+			// new lock
+			renewToken = uuid.New().String()
+			generation = lm.NextGeneration()
+			seq = 1
+			glog.V(4).Infof("key %s new token %v owner %v generation %d", path, renewToken, owner, generation)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+			return
+		} else {
+			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
+			err = LockErrorNonEmptyTokenOnNewLock
+			return
+		}
+	}
+}
+
+func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, generation int64, seq int64, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if oldValue, found := lm.locks[path]; found {
+		now := time.Now()
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
+			// lock is expired, delete it
+			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
+			delete(lm.locks, path)
+			return
+		}
+		if oldValue.Token == token {
+			isUnlocked = true
+			generation = oldValue.Generation
+			seq = oldValue.Seq + 1
+			glog.V(4).Infof("key %s unlocked with %v", path, token)
+			delete(lm.locks, path)
+			return
+		} else {
+			isUnlocked = false
+			err = UnlockErrorTokenMismatch
+			return
+		}
+	}
+	err = LockNotFound
+	return
+}
+
+func (lm *LockManager) CleanUp() {
+
+	for {
+		time.Sleep(1 * time.Minute)
+		now := time.Now().UnixNano()
+
+		lm.accessLock.Lock()
+		for key, value := range lm.locks {
+			if value == nil {
+				continue
+			}
+			if now > value.ExpiredAtNs {
+				glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
+				delete(lm.locks, key)
+			}
+		}
+		lm.accessLock.Unlock()
+	}
+}
+
+// SelectLocks takes out locks by key
+// if selectFn returns true, the lock will be removed and returned
+func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	now := time.Now().UnixNano()
+
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
+			delete(lm.locks, key)
+			continue
+		}
+		if selectFn(key) {
+			glog.V(4).Infof("key %s selected and deleted", key)
+			delete(lm.locks, key)
+			lock.Key = key
+			locks = append(locks, lock)
+		}
+	}
+	return
+}
+
+// InsertLock inserts a lock from a transfer if it is newer than the current state.
+func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("lock %s: rejecting stale transfer gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return false
+		}
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+	return true
+}
+
+// InsertBackupLock inserts or updates a lock as a backup copy.
+// It rejects stale mutations by comparing (generation, seq). If a current primary
+// already exists on this node, newer replicated state refreshes that primary copy
+// without demoting it back to a backup.
+func (lm *LockManager) InsertBackupLock(path string, expiredAtNs int64, token string, owner string, generation int64, seq int64) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) <= 0 {
+			glog.V(4).Infof("backup lock %s: rejecting stale gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+			return
+		}
+		lm.locks[path] = &Lock{
+			Token:       token,
+			ExpiredAtNs: expiredAtNs,
+			Owner:       owner,
+			IsBackup:    existing.IsBackup,
+			Generation:  generation,
+			Seq:         seq,
+		}
+		lm.advanceGenerationFloor(generation)
+		return
+	}
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner, IsBackup: true, Generation: generation, Seq: seq}
+	lm.advanceGenerationFloor(generation)
+}
+
+// RemoveLock removes a lock by key
+func (lm *LockManager) RemoveLock(path string) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+	delete(lm.locks, path)
+}
+
+// RemoveBackupLockIfSeq removes the local copy only if the incoming mutation is
+// not older than the current (generation, seq). This prevents a late unlock from
+// deleting a newer reacquired lock whose seq has reset.
+func (lm *LockManager) RemoveBackupLockIfSeq(path string, generation int64, seq int64) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if existing, found := lm.locks[path]; found {
+		if compareMutationVersion(generation, seq, existing.Generation, existing.Seq) >= 0 {
+			delete(lm.locks, path)
+			return true
+		}
+		glog.V(4).Infof("backup lock %s: rejecting stale unlock gen=%d seq=%d (current gen=%d seq=%d)", path, generation, seq, existing.Generation, existing.Seq)
+		return false
+	}
+	return true
+}
+
+// GetLock returns a copy of the lock for a key, if it exists and is not expired
+func (lm *LockManager) GetLock(key string) (*Lock, bool) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	lock, found := lm.locks[key]
+	if !found {
+		return nil, false
+	}
+	if time.Now().UnixNano() > lock.ExpiredAtNs {
+		return nil, false
+	}
+	// Return a copy
+	cp := *lock
+	return &cp, true
+}
+
+func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	if lock, found := lm.locks[key]; found {
+		if time.Now().UnixNano() > lock.ExpiredAtNs {
+			err = LockNotFound
+			return
+		}
+		return lock.Owner, nil
+	}
+	err = LockNotFound
+	return
+}
+
+// AllLocks returns a copy of all non-expired locks
+func (lm *LockManager) AllLocks() []*Lock {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
+	now := time.Now().UnixNano()
+	var result []*Lock
+	for key, lock := range lm.locks {
+		if now > lock.ExpiredAtNs {
+			continue
+		}
+		cp := *lock
+		cp.Key = key
+		result = append(result, &cp)
+	}
+	return result
+}
+
+// PromoteLock changes a backup lock to a primary lock
+func (lm *LockManager) PromoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && lock.IsBackup {
+		lock.IsBackup = false
+		return true
+	}
+	return false
+}
+
+// DemoteLock changes a primary lock to a backup lock
+func (lm *LockManager) DemoteLock(key string) bool {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if lock, found := lm.locks[key]; found && !lock.IsBackup {
+		lock.IsBackup = true
+		return true
+	}
+	return false
+}

weed/cluster/lock_manager/lock_manager_test.go

@@ -0,0 +1,86 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockManager_GetLockOwnerIgnoresExpiredLock(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(-time.Second).UnixNano(), "token1", "owner1", 7, 3)
+
+	owner, err := lm.GetLockOwner("key1")
+	assert.Empty(t, owner)
+	assert.ErrorIs(t, err, LockNotFound)
+}
+
+func TestLockManager_InsertLockRejectsStaleTransfer(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 4)
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 3)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(4), lock.Seq)
+}
+
+func TestLockManager_InsertLockAdvancesGenerationCounter(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token1", "owner1", 12, 1)
+
+	_, renewToken, generation, _, err := lm.Lock("key2", time.Now().Add(30*time.Second).UnixNano(), "", "owner2")
+	assert.NoError(t, err)
+	assert.NotEmpty(t, renewToken)
+	assert.Greater(t, generation, int64(12))
+}
+
+func TestLockManager_InsertBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 7, 9)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}
+
+func TestLockManager_InsertBackupLockKeepsPrimaryRole(t *testing.T) {
+	lm := NewLockManager()
+
+	ok := lm.InsertLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-old", "owner1", 8, 1)
+	assert.True(t, ok)
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 2)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.False(t, lock.IsBackup)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(2), lock.Seq)
+}
+
+func TestLockManager_RemoveBackupLockRejectsOlderGeneration(t *testing.T) {
+	lm := NewLockManager()
+
+	lm.InsertBackupLock("key1", time.Now().Add(30*time.Second).UnixNano(), "token-new", "owner1", 8, 1)
+
+	removed := lm.RemoveBackupLockIfSeq("key1", 7, 9)
+	assert.False(t, removed)
+
+	lock, found := lm.GetLock("key1")
+	assert.True(t, found)
+	assert.Equal(t, "token-new", lock.Token)
+	assert.Equal(t, int64(8), lock.Generation)
+	assert.Equal(t, int64(1), lock.Seq)
+}

weed/cluster/lock_manager/lock_ring.go

@@ -0,0 +1,211 @@
+package lock_manager
+
+import (
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+type LockRingSnapshot struct {
+	servers []pb.ServerAddress
+	ts      time.Time
+	ring    *HashRing // prebuilt ring for servers, so PriorOwner need not rebuild per call
+}
+
+type LockRing struct {
+	sync.RWMutex
+	snapshots        []*LockRingSnapshot
+	lastCompactTime  time.Time
+	snapshotInterval time.Duration
+	onTakeSnapshot   func(snapshot []pb.ServerAddress)
+	cleanupWg        sync.WaitGroup
+	Ring             *HashRing // consistent hash ring
+	version          int64     // monotonic version from master, rejects stale updates
+}
+
+func NewLockRing(snapshotInterval time.Duration) *LockRing {
+	return &LockRing{
+		snapshotInterval: snapshotInterval,
+		Ring:             NewHashRing(DefaultVnodeCount),
+	}
+}
+
+func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.ServerAddress)) {
+	r.Lock()
+	defer r.Unlock()
+	r.onTakeSnapshot = onTakeSnapshot
+}
+
+// SetSnapshot replaces the ring with a new server list from the master.
+// The version must be >= the current version, otherwise the update is rejected
+// (protects against reordered messages). Version 0 is always accepted (bootstrap).
+func (r *LockRing) SetSnapshot(servers []pb.ServerAddress, version int64) bool {
+
+	sort.Slice(servers, func(i, j int) bool {
+		return servers[i] < servers[j]
+	})
+
+	r.Lock()
+	if version > 0 && version < r.version {
+		glog.V(0).Infof("LockRing: rejecting stale update v%d (current v%d)", version, r.version)
+		r.Unlock()
+		return false
+	}
+	r.version = version
+	// Update the ring while holding the lock so version and ring state
+	// are always consistent — prevents a concurrent SetSnapshot from
+	// seeing the new version but applying its servers to the old ring.
+	r.Ring.SetServers(servers)
+	// Append the snapshot under the same lock as the ring update so a concurrent
+	// PriorOwner always sees snapshots[0] matching r.Ring (and snapshots[1] as the
+	// true prior); otherwise it could pair a new ring with a stale prior snapshot.
+	r.addOneSnapshotLocked(servers)
+	r.Unlock()
+
+	r.cleanupWg.Add(1)
+	go func() {
+		defer r.cleanupWg.Done()
+		<-time.After(r.snapshotInterval)
+		r.compactSnapshots()
+	}()
+	return true
+}
+
+// Version returns the current ring version.
+func (r *LockRing) Version() int64 {
+	r.RLock()
+	defer r.RUnlock()
+	return r.version
+}
+
+// addOneSnapshotLocked appends a new snapshot (newest at index 0). The caller
+// must hold r.Lock(), so the ring update and snapshot append are one atomic step.
+func (r *LockRing) addOneSnapshotLocked(servers []pb.ServerAddress) {
+	ts := time.Now()
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	t := &LockRingSnapshot{
+		servers: servers,
+		ts:      ts,
+		ring:    ring,
+	}
+	r.snapshots = append(r.snapshots, t)
+	for i := len(r.snapshots) - 2; i >= 0; i-- {
+		r.snapshots[i+1] = r.snapshots[i]
+	}
+	r.snapshots[0] = t
+
+	if r.onTakeSnapshot != nil {
+		r.onTakeSnapshot(t.servers)
+	}
+}
+
+func (r *LockRing) compactSnapshots() {
+	r.Lock()
+	defer r.Unlock()
+
+	ts := time.Now()
+	recentSnapshotIndex := 1
+	for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
+		if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
+			break
+		}
+	}
+	if recentSnapshotIndex+1 <= len(r.snapshots) {
+		r.snapshots = r.snapshots[:recentSnapshotIndex+1]
+	}
+	r.lastCompactTime = ts
+}
+
+func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
+	r.RLock()
+	defer r.RUnlock()
+
+	if len(r.snapshots) == 0 {
+		return
+	}
+	return r.snapshots[0].servers
+}
+
+// WaitForCleanup waits for all pending cleanup operations to complete
+func (r *LockRing) WaitForCleanup() {
+	r.cleanupWg.Wait()
+}
+
+// GetSnapshotCount safely returns the number of snapshots for testing
+func (r *LockRing) GetSnapshotCount() int {
+	r.RLock()
+	defer r.RUnlock()
+	return len(r.snapshots)
+}
+
+// GetPrimaryAndBackup returns the primary and backup servers for a key
+// using the consistent hash ring.
+func (r *LockRing) GetPrimaryAndBackup(key string) (primary, backup pb.ServerAddress) {
+	return r.Ring.GetPrimaryAndBackup(key)
+}
+
+// GetPrimary returns the primary server for a key using the consistent hash ring.
+func (r *LockRing) GetPrimary(key string) pb.ServerAddress {
+	return r.Ring.GetPrimary(key)
+}
+
+// PriorOwner returns the key's owner from the previous ring snapshot, but only
+// while the ring changed within the last snapshotInterval and that owner differs
+// from the current primary. This is the cooling-off window in which the previous
+// owner may still hold locks the new owner has not yet rebuilt — a caller can
+// consult it before granting so a fresh owner does not double-grant during a
+// rebalance. Returns "" outside the window or when ownership did not move. It
+// uses the snapshot's prebuilt ring, so it does not rebuild a hash ring per call.
+func (r *LockRing) PriorOwner(key string) pb.ServerAddress {
+	r.RLock()
+	defer r.RUnlock()
+	return r.priorOwnerLocked(key)
+}
+
+// WriteOwner returns the filer that should serialize writes to key: the prior
+// owner while a ring change is still within the cooling-off window, otherwise
+// the current primary. Both are read under one lock so the pair cannot come
+// from different rings, which could otherwise name the same filer twice.
+func (r *LockRing) WriteOwner(key string) pb.ServerAddress {
+	r.RLock()
+	defer r.RUnlock()
+	if prior := r.priorOwnerLocked(key); prior != "" {
+		return prior
+	}
+	return r.Ring.GetPrimary(key)
+}
+
+// priorOwnerLocked is PriorOwner's body; the caller holds at least RLock.
+func (r *LockRing) priorOwnerLocked(key string) pb.ServerAddress {
+	if len(r.snapshots) < 2 {
+		return ""
+	}
+	if time.Since(r.snapshots[0].ts) > r.snapshotInterval {
+		return ""
+	}
+	current := r.Ring.GetPrimary(key)
+	var prior pb.ServerAddress
+	if pr := r.snapshots[1].ring; pr != nil {
+		prior = pr.GetPrimary(key)
+	} else {
+		prior = hashKeyToServer(key, r.snapshots[1].servers)
+	}
+	if prior != "" && prior != current {
+		return prior
+	}
+	return ""
+}
+
+// hashKeyToServer uses a temporary consistent hash ring for the given server list.
+func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
+	if len(servers) == 0 {
+		return ""
+	}
+	ring := NewHashRing(DefaultVnodeCount)
+	ring.SetServers(servers)
+	return ring.GetPrimary(key)
+}

weed/cluster/lock_manager/lock_ring_prior_test.go

@@ -0,0 +1,102 @@
+package lock_manager
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func TestLockRing_PriorOwner(t *testing.T) {
+	r := NewLockRing(5 * time.Second)
+	t.Cleanup(r.WaitForCleanup)
+
+	setA := []pb.ServerAddress{"s1:1", "s2:1", "s3:1"}
+	r.SetSnapshot(setA, 1)
+
+	// Only one snapshot: nothing to fall back to.
+	if got := r.PriorOwner("any"); got != "" {
+		t.Fatalf("single snapshot should have no prior owner, got %q", got)
+	}
+
+	// Add a server so some keys' ownership moves.
+	setB := []pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}
+	r.SetSnapshot(setB, 2)
+
+	var moved, stable string
+	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
+		key := fmt.Sprintf("key-%d", i)
+		if r.GetPrimary(key) != hashKeyToServer(key, setA) {
+			if moved == "" {
+				moved = key
+			}
+		} else if stable == "" {
+			stable = key
+		}
+	}
+	if moved == "" || stable == "" {
+		t.Skip("could not find both a moved and a stable key")
+	}
+
+	if got, want := r.PriorOwner(moved), hashKeyToServer(moved, setA); got != want {
+		t.Fatalf("PriorOwner(moved)=%q, want %q", got, want)
+	}
+	if got := r.PriorOwner(stable); got != "" {
+		t.Fatalf("unmoved key should have no prior owner, got %q", got)
+	}
+}
+
+func TestLockRing_PriorOwnerExpires(t *testing.T) {
+	r := NewLockRing(20 * time.Millisecond)
+	t.Cleanup(r.WaitForCleanup)
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1"}, 1)
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}, 2)
+
+	// Past the cooling interval, the prior owner is no longer offered.
+	time.Sleep(40 * time.Millisecond)
+	for i := 0; i < 2000; i++ {
+		if got := r.PriorOwner(fmt.Sprintf("key-%d", i)); got != "" {
+			t.Fatalf("prior owner should expire after the cooling interval, got %q", got)
+		}
+	}
+}
+
+// WriteOwner resolves prior-else-current under one read lock, so the two views
+// can never come from different rings.
+func TestLockRing_WriteOwner(t *testing.T) {
+	r := NewLockRing(5 * time.Second)
+	t.Cleanup(r.WaitForCleanup)
+
+	setA := []pb.ServerAddress{"s1:1", "s2:1", "s3:1"}
+	r.SetSnapshot(setA, 1)
+
+	// One snapshot: no prior owner, so the primary owns every key.
+	if got, want := r.WriteOwner("any"), r.GetPrimary("any"); got != want {
+		t.Fatalf("WriteOwner=%q, want the primary %q", got, want)
+	}
+
+	r.SetSnapshot([]pb.ServerAddress{"s1:1", "s2:1", "s3:1", "s4:1"}, 2)
+
+	var moved, stable string
+	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
+		key := fmt.Sprintf("key-%d", i)
+		if r.PriorOwner(key) != "" {
+			if moved == "" {
+				moved = key
+			}
+		} else if stable == "" {
+			stable = key
+		}
+	}
+	if moved == "" || stable == "" {
+		t.Skip("could not find both a moved and a stable key")
+	}
+
+	if got, want := r.WriteOwner(moved), r.PriorOwner(moved); got != want {
+		t.Fatalf("moved key: WriteOwner=%q, want the prior owner %q", got, want)
+	}
+	if got, want := r.WriteOwner(stable), r.GetPrimary(stable); got != want {
+		t.Fatalf("stable key: WriteOwner=%q, want the primary %q", got, want)
+	}
+}

weed/cluster/lock_manager/lock_ring_test.go

@@ -0,0 +1,98 @@
+package lock_manager
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLockRing_SetSnapshot(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Set 5 servers
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 1)
+
+	servers := r.GetSnapshot()
+	assert.Equal(t, 5, len(servers))
+
+	// Replace with 2 servers
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8081", "localhost:8083"}, 2)
+
+	r.WaitForCleanup()
+	servers = r.GetSnapshot()
+	assert.Equal(t, 2, len(servers))
+	assert.Contains(t, servers, pb.ServerAddress("localhost:8081"))
+	assert.Contains(t, servers, pb.ServerAddress("localhost:8083"))
+
+	// Verify compaction
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2)
+}
+
+func TestLockRing_SnapshotCompaction(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"}, 1)
+	assert.Equal(t, 1, r.GetSnapshotCount())
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"}, 2)
+	assert.Equal(t, 2, r.GetSnapshotCount())
+
+	// Wait for compaction
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+
+	r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"}, 3)
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 3)
+	servers := r.GetSnapshot()
+	assert.Equal(t, 4, len(servers))
+
+	time.Sleep(110 * time.Millisecond)
+	r.WaitForCleanup()
+	assert.LessOrEqual(t, r.GetSnapshotCount(), 2, "Snapshots should be compacted")
+
+	r.SetSnapshot([]pb.ServerAddress{
+		"localhost:8080", "localhost:8081", "localhost:8082",
+		"localhost:8083", "localhost:8084",
+	}, 4)
+	servers = r.GetSnapshot()
+	assert.Equal(t, 5, len(servers))
+}
+
+func TestLockRing_VersionRejectsStale(t *testing.T) {
+	r := NewLockRing(100 * time.Millisecond)
+
+	// Apply version 3
+	ok := r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2", "c:3"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()))
+
+	// Stale version 2 — should be rejected
+	ok = r.SetSnapshot([]pb.ServerAddress{"x:1"}, 2)
+	assert.False(t, ok)
+	assert.Equal(t, int64(3), r.Version())
+	assert.Equal(t, 3, len(r.GetSnapshot()), "stale update should not change the ring")
+
+	// Same version 3 — accepted (SetSnapshot accepts version >= current, state-changing)
+	ok = r.SetSnapshot([]pb.ServerAddress{"a:1", "b:2"}, 3)
+	assert.True(t, ok)
+	assert.Equal(t, 2, len(r.GetSnapshot()))
+
+	// Newer version 5 — should be accepted
+	ok = r.SetSnapshot([]pb.ServerAddress{"d:1", "e:2", "f:3", "g:4"}, 5)
+	assert.True(t, ok)
+	assert.Equal(t, int64(5), r.Version())
+	assert.Equal(t, 4, len(r.GetSnapshot()))
+
+	// Version 0 always accepted (bootstrap)
+	ok = r.SetSnapshot([]pb.ServerAddress{"z:1"}, 0)
+	assert.True(t, ok)
+	assert.Equal(t, 1, len(r.GetSnapshot()))
+}