snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/wdclient/exclusive_locks/exclusive_locker.go

@@ -0,0 +1,171 @@
+package exclusive_locks
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+const (
+	RenewInterval     = 4 * time.Second
+	SafeRenewInterval = 3 * time.Second
+	InitLockInterval  = 1 * time.Second
+	// bounds each lease and renew RPC attempt so an unresponsive master
+	// cannot hold l.mu (and block ReleaseLock) indefinitely
+	rpcTimeout = 3 * time.Second
+)
+
+type ExclusiveLocker struct {
+	token        int64
+	lockTsNs     int64
+	isLocked     atomic.Bool
+	masterClient *wdclient.MasterClient
+	lockName     string
+	message      string
+	clientName   string
+	// serializes renew and release RPCs: a renewal in flight during a release
+	// would re-create the lock on the master and leave it held until expiry
+	mu sync.Mutex
+	// Each lock has and only has one goroutine
+	renewGoroutineRunning atomic.Bool
+}
+
+func NewExclusiveLocker(masterClient *wdclient.MasterClient, lockName string) *ExclusiveLocker {
+	return &ExclusiveLocker{
+		masterClient: masterClient,
+		lockName:     lockName,
+	}
+}
+
+func (l *ExclusiveLocker) IsLocked() bool {
+	return l.isLocked.Load()
+}
+
+func (l *ExclusiveLocker) GetToken() (token int64, lockTsNs int64) {
+	for time.Unix(0, atomic.LoadInt64(&l.lockTsNs)).Add(SafeRenewInterval).Before(time.Now()) {
+		// wait until now is within the safe lock period, no immediate renewal to change the token
+		time.Sleep(100 * time.Millisecond)
+	}
+	return atomic.LoadInt64(&l.token), atomic.LoadInt64(&l.lockTsNs)
+}
+
+func (l *ExclusiveLocker) RequestLock(clientName string) {
+	if l.isLocked.Load() {
+		return
+	}
+
+	// retry to get the lease
+	for {
+		if err := l.masterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+			attemptCtx, cancel := context.WithTimeout(context.Background(), rpcTimeout)
+			defer cancel()
+			resp, err := client.LeaseAdminToken(attemptCtx, &master_pb.LeaseAdminTokenRequest{
+				PreviousToken:    atomic.LoadInt64(&l.token),
+				PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
+				LockName:         l.lockName,
+				ClientName:       clientName,
+			})
+			if err == nil {
+				atomic.StoreInt64(&l.token, resp.Token)
+				atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
+			}
+			return err
+		}); err != nil {
+			glog.V(2).Infof("Failed to acquire lock %s: %v", l.lockName, err)
+			time.Sleep(InitLockInterval)
+		} else {
+			break
+		}
+	}
+
+	l.mu.Lock()
+	l.clientName = clientName
+	l.isLocked.Store(true)
+	l.mu.Unlock()
+	glog.V(1).Infof("Acquired lock %s", l.lockName)
+
+	// Each lock has and only has one goroutine
+	if l.renewGoroutineRunning.CompareAndSwap(false, true) {
+		// start a goroutine to renew the lease
+		go func() {
+			ctx2, cancel2 := context.WithCancel(context.Background())
+			defer cancel2()
+
+			for {
+				if err := l.renewLease(ctx2); err != nil {
+					glog.Warningf("Failed to renew lock %s: %v", l.lockName, err)
+					// clear the running flag before isLocked, so a RequestLock that
+					// reacquires (once isLocked is false) starts a replacement renewer
+					l.renewGoroutineRunning.Store(false)
+					l.isLocked.Store(false)
+					return
+				}
+				time.Sleep(RenewInterval)
+			}
+		}()
+	}
+
+}
+
+func (l *ExclusiveLocker) renewLease(ctx context.Context) error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if !l.isLocked.Load() {
+		return nil
+	}
+	return l.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		attemptCtx, cancel := context.WithTimeout(ctx, rpcTimeout)
+		defer cancel()
+		resp, err := client.LeaseAdminToken(attemptCtx, &master_pb.LeaseAdminTokenRequest{
+			PreviousToken:    atomic.LoadInt64(&l.token),
+			PreviousLockTime: atomic.LoadInt64(&l.lockTsNs),
+			LockName:         l.lockName,
+			ClientName:       l.clientName,
+			Message:          l.message,
+		})
+		if err == nil {
+			atomic.StoreInt64(&l.token, resp.Token)
+			atomic.StoreInt64(&l.lockTsNs, resp.LockTsNs)
+			glog.V(2).Infof("Renewed lock %s: ts %d token %d", l.lockName, l.lockTsNs, l.token)
+		}
+		return err
+	})
+}
+
+func (l *ExclusiveLocker) ReleaseLock() {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	l.isLocked.Store(false)
+	l.clientName = ""
+
+	prevToken := atomic.LoadInt64(&l.token)
+	prevLockTsNs := atomic.LoadInt64(&l.lockTsNs)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// single unbounded attempt: a release cut short by a deadline leaves the
+	// lock held until it expires, turning a slow unlock into a ghost lock
+	l.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		client.ReleaseAdminToken(ctx, &master_pb.ReleaseAdminTokenRequest{
+			PreviousToken:    prevToken,
+			PreviousLockTime: prevLockTsNs,
+			LockName:         l.lockName,
+		})
+		return nil
+	})
+	// compare on clear: a RequestLock racing a slow release must not have its
+	// fresh token zeroed
+	atomic.CompareAndSwapInt64(&l.token, prevToken, 0)
+	atomic.CompareAndSwapInt64(&l.lockTsNs, prevLockTsNs, 0)
+}
+
+func (l *ExclusiveLocker) SetMessage(message string) {
+	l.message = message
+}

weed/wdclient/filer_client.go

@@ -0,0 +1,743 @@
+package wdclient
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// UrlPreference controls which URL to use for volume access
+type UrlPreference string
+
+const (
+	PreferUrl       UrlPreference = "url"       // Use private URL (default)
+	PreferPublicUrl UrlPreference = "publicUrl" // Use public URL
+)
+
+// filerHealth tracks the health status of a filer
+type filerHealth struct {
+	lastFailureTimeNs int64 // atomic: last failure time in Unix nanoseconds
+	failureCount      int32 // atomic: consecutive failures
+}
+
+// FilerClient provides volume location services by querying a filer
+// It uses the shared vidMap cache for efficient lookups
+// Supports multiple filer addresses with automatic failover for high availability
+// Tracks filer health to avoid repeatedly trying known-unhealthy filers
+// Can discover additional filers from master server when configured with filer group
+type FilerClient struct {
+	*vidMapClient
+	filerAddresses     []pb.ServerAddress
+	filerAddressesMu   sync.RWMutex   // Protects filerAddresses and filerHealth
+	filerIndex         int32          // atomic: current filer index for round-robin
+	filerHealth        []*filerHealth // health status per filer (same order as filerAddresses)
+	grpcDialOption     grpc.DialOption
+	urlPreference      UrlPreference
+	grpcTimeout        time.Duration
+	cacheSize          int           // Number of historical vidMap snapshots to keep
+	clientId           int32         // Unique client identifier for gRPC metadata
+	failureThreshold   int32         // Circuit breaker: consecutive failures before circuit opens
+	resetTimeout       time.Duration // Circuit breaker: time before re-checking unhealthy filer
+	maxRetries         int           // Retry: maximum retry attempts for transient failures
+	initialRetryWait   time.Duration // Retry: initial wait time before first retry
+	retryBackoffFactor float64       // Retry: backoff multiplier for wait time
+
+	// Filer discovery fields
+	masterClient       *MasterClient // Optional: for discovering filers in the same group
+	filerGroup         string        // Optional: filer group for discovery
+	discoveryInterval  time.Duration // How often to refresh filer list from master
+	stopDiscovery      chan struct{} // Signal to stop discovery goroutine
+	closeDiscoveryOnce sync.Once     // Ensures discovery channel is closed at most once
+}
+
+// filerVolumeProvider implements VolumeLocationProvider by querying filer
+// Supports multiple filer addresses with automatic failover
+type filerVolumeProvider struct {
+	filerClient *FilerClient
+}
+
+// FilerClientOption holds optional configuration for FilerClient
+type FilerClientOption struct {
+	GrpcTimeout        time.Duration
+	UrlPreference      UrlPreference
+	CacheSize          int           // Number of historical vidMap snapshots (0 = use default)
+	FailureThreshold   int32         // Circuit breaker: consecutive failures before skipping filer (0 = use default of 3)
+	ResetTimeout       time.Duration // Circuit breaker: time before re-checking unhealthy filer (0 = use default of 30s)
+	MaxRetries         int           // Retry: maximum retry attempts for transient failures (0 = use default of 3)
+	InitialRetryWait   time.Duration // Retry: initial wait time before first retry (0 = use default of 1s)
+	RetryBackoffFactor float64       // Retry: backoff multiplier for wait time (0 = use default of 1.5)
+
+	// Filer discovery options
+	MasterClient      *MasterClient // Optional: enables filer discovery from master
+	FilerGroup        string        // Optional: filer group name for discovery (required if MasterClient is set)
+	DiscoveryInterval time.Duration // Optional: how often to refresh filer list (0 = use default of 5 minutes)
+}
+
+// NewFilerClient creates a new client that queries filer(s) for volume locations
+// Supports multiple filer addresses for high availability with automatic failover
+// Uses sensible defaults: 5-second gRPC timeout, PreferUrl, DefaultVidMapCacheSize
+func NewFilerClient(filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, dataCenter string, opts ...*FilerClientOption) *FilerClient {
+	if len(filerAddresses) == 0 {
+		glog.Fatal("NewFilerClient requires at least one filer address")
+	}
+
+	// Apply defaults
+	grpcTimeout := 5 * time.Second
+	urlPref := PreferUrl
+	cacheSize := DefaultVidMapCacheSize
+	failureThreshold := int32(3)     // Default: 3 consecutive failures before circuit opens
+	resetTimeout := 30 * time.Second // Default: 30 seconds before re-checking unhealthy filer
+	maxRetries := 3                  // Default: 3 retry attempts for transient failures
+	initialRetryWait := time.Second  // Default: 1 second initial retry wait
+	retryBackoffFactor := 1.5        // Default: 1.5x backoff multiplier
+	var masterClient *MasterClient
+	var filerGroup string
+	discoveryInterval := 5 * time.Minute // Default: refresh every 5 minutes
+
+	// Override with provided options
+	if len(opts) > 0 && opts[0] != nil {
+		opt := opts[0]
+		if opt.GrpcTimeout > 0 {
+			grpcTimeout = opt.GrpcTimeout
+		}
+		if opt.UrlPreference != "" {
+			urlPref = opt.UrlPreference
+		}
+		if opt.CacheSize > 0 {
+			cacheSize = opt.CacheSize
+		}
+		if opt.FailureThreshold > 0 {
+			failureThreshold = opt.FailureThreshold
+		}
+		if opt.ResetTimeout > 0 {
+			resetTimeout = opt.ResetTimeout
+		}
+		if opt.MaxRetries > 0 {
+			maxRetries = opt.MaxRetries
+		}
+		if opt.InitialRetryWait > 0 {
+			initialRetryWait = opt.InitialRetryWait
+		}
+		if opt.RetryBackoffFactor > 0 {
+			retryBackoffFactor = opt.RetryBackoffFactor
+		}
+		if opt.MasterClient != nil {
+			masterClient = opt.MasterClient
+			filerGroup = opt.FilerGroup
+			if opt.DiscoveryInterval > 0 {
+				discoveryInterval = opt.DiscoveryInterval
+			}
+		}
+	}
+
+	// Initialize health tracking for each filer
+	health := make([]*filerHealth, len(filerAddresses))
+	for i := range health {
+		health[i] = &filerHealth{}
+	}
+
+	fc := &FilerClient{
+		filerAddresses:     filerAddresses,
+		filerIndex:         0,
+		filerHealth:        health,
+		grpcDialOption:     grpcDialOption,
+		urlPreference:      urlPref,
+		grpcTimeout:        grpcTimeout,
+		cacheSize:          cacheSize,
+		clientId:           rand.Int31(), // Random client ID for gRPC metadata tracking
+		failureThreshold:   failureThreshold,
+		resetTimeout:       resetTimeout,
+		maxRetries:         maxRetries,
+		initialRetryWait:   initialRetryWait,
+		retryBackoffFactor: retryBackoffFactor,
+		masterClient:       masterClient,
+		filerGroup:         filerGroup,
+		discoveryInterval:  discoveryInterval,
+	}
+
+	// Start filer discovery if master client is configured
+	// Empty filerGroup is valid (represents default group)
+	if masterClient != nil {
+		fc.stopDiscovery = make(chan struct{})
+		go fc.discoverFilers()
+		glog.V(0).Infof("FilerClient: started filer discovery for group '%s' (refresh interval: %v)", filerGroup, discoveryInterval)
+	}
+
+	// Create provider that references this FilerClient for failover support
+	provider := &filerVolumeProvider{
+		filerClient: fc,
+	}
+
+	fc.vidMapClient = newVidMapClient(provider, dataCenter, cacheSize)
+
+	return fc
+}
+
+// GetCurrentFiler returns the currently active filer address
+// This is the filer that was last successfully used or the one indicated by round-robin
+// Returns empty string if no filers are configured
+func (fc *FilerClient) GetCurrentFiler() pb.ServerAddress {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	if len(fc.filerAddresses) == 0 {
+		return ""
+	}
+
+	// Get current index (atomically updated on successful operations)
+	index := atomic.LoadInt32(&fc.filerIndex)
+	if index >= int32(len(fc.filerAddresses)) {
+		index = 0
+	}
+
+	return fc.filerAddresses[index]
+}
+
+// GetAllFilers returns a snapshot of all filer addresses
+// Returns a copy to avoid concurrent modification issues
+func (fc *FilerClient) GetAllFilers() []pb.ServerAddress {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	// Return a copy to avoid concurrent modification
+	filers := make([]pb.ServerAddress, len(fc.filerAddresses))
+	copy(filers, fc.filerAddresses)
+	return filers
+}
+
+// SetCurrentFiler updates the current filer index to the specified address
+// This is useful after successful failover to prefer the healthy filer for future requests
+func (fc *FilerClient) SetCurrentFiler(addr pb.ServerAddress) {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	// Find the index of the specified filer address
+	for i, filer := range fc.filerAddresses {
+		if filer == addr {
+			atomic.StoreInt32(&fc.filerIndex, int32(i))
+			return
+		}
+	}
+	// If address not found, leave index unchanged
+}
+
+// ShouldSkipUnhealthyFiler checks if a filer address should be skipped based on health tracking
+// Returns true if the filer has exceeded failure threshold and reset timeout hasn't elapsed
+func (fc *FilerClient) ShouldSkipUnhealthyFiler(addr pb.ServerAddress) bool {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	// Find the health for this filer address
+	for i, filer := range fc.filerAddresses {
+		if filer == addr {
+			if i < len(fc.filerHealth) {
+				return fc.shouldSkipUnhealthyFilerWithHealth(fc.filerHealth[i])
+			}
+			return false
+		}
+	}
+	// If address not found, don't skip it
+	return false
+}
+
+// RecordFilerSuccess resets failure tracking for a successful filer
+func (fc *FilerClient) RecordFilerSuccess(addr pb.ServerAddress) {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	// Find the health for this filer address
+	for i, filer := range fc.filerAddresses {
+		if filer == addr {
+			if i < len(fc.filerHealth) {
+				fc.recordFilerSuccessWithHealth(fc.filerHealth[i])
+			}
+			return
+		}
+	}
+}
+
+// RecordFilerFailure increments failure count for an unhealthy filer
+func (fc *FilerClient) RecordFilerFailure(addr pb.ServerAddress) {
+	fc.filerAddressesMu.RLock()
+	defer fc.filerAddressesMu.RUnlock()
+
+	// Find the health for this filer address
+	for i, filer := range fc.filerAddresses {
+		if filer == addr {
+			if i < len(fc.filerHealth) {
+				fc.recordFilerFailureWithHealth(fc.filerHealth[i])
+			}
+			return
+		}
+	}
+}
+
+// Close stops the filer discovery goroutine if running
+// Safe to call multiple times (idempotent)
+func (fc *FilerClient) Close() {
+	if fc.stopDiscovery != nil {
+		fc.closeDiscoveryOnce.Do(func() {
+			close(fc.stopDiscovery)
+		})
+	}
+}
+
+// discoverFilers periodically queries the master to discover filers in the same group
+// and updates the filer list. This runs in a background goroutine.
+func (fc *FilerClient) discoverFilers() {
+	defer func() {
+		if r := recover(); r != nil {
+			glog.Errorf("FilerClient: panic in filer discovery goroutine for group '%s': %v", fc.filerGroup, r)
+		}
+	}()
+
+	// Do an initial discovery
+	fc.refreshFilerList()
+
+	ticker := time.NewTicker(fc.discoveryInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			fc.refreshFilerList()
+		case <-fc.stopDiscovery:
+			glog.V(0).Infof("FilerClient: stopping filer discovery for group '%s'", fc.filerGroup)
+			return
+		}
+	}
+}
+
+// refreshFilerList queries the master for the current list of filers and updates the local list
+func (fc *FilerClient) refreshFilerList() {
+	if fc.masterClient == nil {
+		return
+	}
+
+	// Get current master address
+	currentMaster := fc.masterClient.GetMaster(context.Background())
+	if currentMaster == "" {
+		glog.V(1).Infof("FilerClient: no master available for filer discovery")
+		return
+	}
+
+	// Query master for filers in our group
+	updates := cluster.ListExistingPeerUpdates(currentMaster, fc.grpcDialOption, fc.filerGroup, cluster.FilerType)
+
+	if len(updates) == 0 {
+		glog.V(2).Infof("FilerClient: no filers found in group '%s'", fc.filerGroup)
+		return
+	}
+
+	// Build new filer address set
+	discoveredFilers := make(map[pb.ServerAddress]struct{}, len(updates))
+	for _, update := range updates {
+		if update.Address != "" {
+			discoveredFilers[pb.ServerAddress(update.Address)] = struct{}{}
+		}
+	}
+
+	// Ignore snapshots whose addresses are all empty; reconciling against an
+	// empty set would wipe the in-memory list.
+	if len(discoveredFilers) == 0 {
+		glog.V(1).Infof("FilerClient: discovery snapshot for group '%s' had no usable addresses, keeping existing list", fc.filerGroup)
+		return
+	}
+
+	fc.applyDiscoveredFilers(discoveredFilers)
+}
+
+// applyDiscoveredFilers treats the master snapshot as authoritative: survivors
+// keep their health counters, new addresses get fresh health, addresses missing
+// from the snapshot are pruned so replaced pods (e.g. rolled K8s filer pods
+// with new IPs) don't linger and get retried after the circuit-breaker reset.
+func (fc *FilerClient) applyDiscoveredFilers(discoveredFilers map[pb.ServerAddress]struct{}) {
+	fc.filerAddressesMu.Lock()
+	defer fc.filerAddressesMu.Unlock()
+
+	existingFilers := make(map[pb.ServerAddress]struct{}, len(fc.filerAddresses))
+	for _, f := range fc.filerAddresses {
+		existingFilers[f] = struct{}{}
+	}
+
+	var newFilers []pb.ServerAddress
+	for addr := range discoveredFilers {
+		if _, found := existingFilers[addr]; !found {
+			newFilers = append(newFilers, addr)
+		}
+	}
+
+	var removedFilers []pb.ServerAddress
+	for _, f := range fc.filerAddresses {
+		if _, found := discoveredFilers[f]; !found {
+			removedFilers = append(removedFilers, f)
+		}
+	}
+
+	if len(newFilers) == 0 && len(removedFilers) == 0 {
+		return
+	}
+
+	// Remember the active filer so the round-robin pointer can follow it across the rebuild.
+	currentIndex := atomic.LoadInt32(&fc.filerIndex)
+	var currentFiler pb.ServerAddress
+	if currentIndex >= 0 && currentIndex < int32(len(fc.filerAddresses)) {
+		currentFiler = fc.filerAddresses[currentIndex]
+	}
+
+	newAddresses := make([]pb.ServerAddress, 0, len(fc.filerAddresses)-len(removedFilers)+len(newFilers))
+	newHealth := make([]*filerHealth, 0, cap(newAddresses))
+	for i, f := range fc.filerAddresses {
+		if _, found := discoveredFilers[f]; found {
+			newAddresses = append(newAddresses, f)
+			newHealth = append(newHealth, fc.filerHealth[i])
+		}
+	}
+	for _, f := range newFilers {
+		newAddresses = append(newAddresses, f)
+		newHealth = append(newHealth, &filerHealth{})
+	}
+
+	fc.filerAddresses = newAddresses
+	fc.filerHealth = newHealth
+
+	var newIndex int32
+	if currentFiler != "" {
+		for i, f := range newAddresses {
+			if f == currentFiler {
+				newIndex = int32(i)
+				break
+			}
+		}
+	}
+	atomic.StoreInt32(&fc.filerIndex, newIndex)
+
+	if len(removedFilers) > 0 {
+		glog.V(0).Infof("FilerClient: removed %d filer(s) no longer in group '%s': %v", len(removedFilers), fc.filerGroup, removedFilers)
+	}
+	if len(newFilers) > 0 {
+		glog.V(0).Infof("FilerClient: discovered %d new filer(s) in group '%s': %v", len(newFilers), fc.filerGroup, newFilers)
+	}
+}
+
+// GetLookupFileIdFunction returns a lookup function with URL preference handling
+func (fc *FilerClient) GetLookupFileIdFunction() LookupFileIdFunctionType {
+	if fc.urlPreference == PreferUrl {
+		// Use the default implementation from vidMapClient
+		return fc.vidMapClient.GetLookupFileIdFunction()
+	}
+
+	// Custom implementation that prefers PublicUrl
+	return func(ctx context.Context, fileId string) (fullUrls []string, err error) {
+		// Parse file ID to extract volume ID
+		parts := strings.Split(fileId, ",")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("invalid fileId format: %s", fileId)
+		}
+		volumeIdStr := parts[0]
+
+		// First try the cache using LookupVolumeIdsWithFallback
+		vidLocations, err := fc.LookupVolumeIdsWithFallback(ctx, []string{volumeIdStr})
+
+		// Check for partial results first (important for multi-volume batched lookups)
+		locations, found := vidLocations[volumeIdStr]
+		if !found || len(locations) == 0 {
+			// Volume not found - return specific error with context from lookup if available
+			if err != nil {
+				return nil, fmt.Errorf("volume %s not found for fileId %s: %w", volumeIdStr, fileId, err)
+			}
+			return nil, fmt.Errorf("volume %s not found for fileId %s", volumeIdStr, fileId)
+		}
+
+		// Volume found successfully - ignore any errors about other volumes
+		// (not relevant for single-volume lookup, but defensive for future batching)
+
+		// Build URLs with publicUrl preference, and also prefer same DC
+		var sameDcUrls, otherDcUrls []string
+		localUrls := make(map[string]bool)
+		dataCenter := fc.GetDataCenter()
+		for _, loc := range locations {
+			url := loc.PublicUrl
+			if url == "" {
+				url = loc.Url
+			}
+			httpUrl := "http://" + url + "/" + fileId
+			glog.V(4).Infof("lookup %s => %s, data in remote storage tier: %v", fileId, url, loc.DataInRemote)
+			if !loc.DataInRemote {
+				localUrls[httpUrl] = true
+			}
+			if dataCenter != "" && dataCenter == loc.DataCenter {
+				sameDcUrls = append(sameDcUrls, httpUrl)
+			} else {
+				otherDcUrls = append(otherDcUrls, httpUrl)
+			}
+		}
+		// Shuffle to distribute load across volume servers
+		rand.Shuffle(len(sameDcUrls), func(i, j int) { sameDcUrls[i], sameDcUrls[j] = sameDcUrls[j], sameDcUrls[i] })
+		rand.Shuffle(len(otherDcUrls), func(i, j int) { otherDcUrls[i], otherDcUrls[j] = otherDcUrls[j], otherDcUrls[i] })
+		// Local replicas go first inside each data center, but never ahead of
+		// the data-center preference itself. Mirrors
+		// vidMap.LookupVolumeServerUrl so all client lookup paths agree.
+		if len(localUrls) > 0 {
+			sameDcUrls = util.ReorderToFront(localUrls, sameDcUrls)
+			otherDcUrls = util.ReorderToFront(localUrls, otherDcUrls)
+		}
+		fullUrls = append(sameDcUrls, otherDcUrls...)
+		return fullUrls, nil
+	}
+}
+
+// isRetryableGrpcError checks if a gRPC error is transient and should be retried
+//
+// Note on codes.Aborted: While Aborted can indicate application-level conflicts
+// (e.g., transaction failures), in the context of volume location lookups (which
+// are simple read-only operations with no transactions), Aborted is more likely
+// to indicate transient server issues during restart/recovery. We include it here
+// for volume lookups but log it for visibility in case misclassification occurs.
+func isRetryableGrpcError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	// Check gRPC status code
+	st, ok := status.FromError(err)
+	if ok {
+		switch st.Code() {
+		case codes.Unavailable: // Server unavailable (temporary)
+			return true
+		case codes.DeadlineExceeded: // Request timeout
+			return true
+		case codes.ResourceExhausted: // Rate limited or overloaded
+			return true
+		case codes.Aborted:
+			// Aborted during read-only volume lookups is likely transient
+			// (e.g., filer restarting), but log for visibility
+			glog.V(1).Infof("Treating Aborted as retryable for volume lookup: %v", err)
+			return true
+		}
+	}
+
+	// Fallback for non-gRPC errors (e.g. network errors). "connection" and
+	// "timeout" are deliberately broader than the shared classifier: a volume
+	// lookup is a cheap read-only call, so leaning towards a retry is fine.
+	errStr := strings.ToLower(err.Error())
+	return util.IsTransientError(err) ||
+		strings.Contains(errStr, "connection") ||
+		strings.Contains(errStr, "timeout")
+}
+
+// jitter returns a duration in the range [d/2, d) using equal jitter.
+// This prevents thundering herds when many clients retry simultaneously
+// after a transient failure (e.g., network partition healing).
+func jitter(d time.Duration) time.Duration {
+	if d <= 0 {
+		return 0
+	}
+	half := d / 2
+	if half <= 0 {
+		return d
+	}
+	return half + time.Duration(rand.Int63n(int64(half)))
+}
+
+// shouldSkipUnhealthyFiler checks if we should skip a filer based on recent failures
+// Circuit breaker pattern: skip filers with multiple recent consecutive failures
+// shouldSkipUnhealthyFilerWithHealth checks if a filer should be skipped based on health
+// Uses atomic operations only - safe to call without locks
+func (fc *FilerClient) shouldSkipUnhealthyFilerWithHealth(health *filerHealth) bool {
+	failureCount := atomic.LoadInt32(&health.failureCount)
+
+	// Check if failure count exceeds threshold
+	if failureCount < fc.failureThreshold {
+		return false
+	}
+
+	// Re-check unhealthy filers after reset timeout
+	lastFailureNs := atomic.LoadInt64(&health.lastFailureTimeNs)
+	if lastFailureNs == 0 {
+		return false // Never failed, shouldn't skip
+	}
+	lastFailureTime := time.Unix(0, lastFailureNs)
+	if time.Since(lastFailureTime) > fc.resetTimeout {
+		return false // Time to re-check
+	}
+
+	return true // Skip this unhealthy filer
+}
+
+// recordFilerSuccessWithHealth resets failure tracking for a successful filer
+func (fc *FilerClient) recordFilerSuccessWithHealth(health *filerHealth) {
+	atomic.StoreInt32(&health.failureCount, 0)
+}
+
+// recordFilerFailureWithHealth increments failure count for an unhealthy filer
+func (fc *FilerClient) recordFilerFailureWithHealth(health *filerHealth) {
+	atomic.AddInt32(&health.failureCount, 1)
+	atomic.StoreInt64(&health.lastFailureTimeNs, time.Now().UnixNano())
+}
+
+// LookupVolumeIds queries the filer for volume locations with automatic failover
+// Tries all configured filer addresses until one succeeds (high availability)
+// Retries transient gRPC errors (Unavailable, DeadlineExceeded, etc.) with exponential backoff
+// Note: Unlike master's VolumeIdLocation, filer's Locations message doesn't currently have
+// an Error field. This implementation handles the current structure while being prepared
+// for future error reporting enhancements.
+func (p *filerVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
+	fc := p.filerClient
+	result := make(map[string][]Location)
+
+	// Retry transient failures with configurable backoff
+	var lastErr error
+	waitTime := fc.initialRetryWait
+	maxRetries := fc.maxRetries
+
+	for retry := 0; retry < maxRetries; retry++ {
+		// Try all filer addresses with round-robin starting from current index
+		// Skip known-unhealthy filers (circuit breaker pattern)
+		i := atomic.LoadInt32(&fc.filerIndex)
+
+		// Get filer count with read lock
+		fc.filerAddressesMu.RLock()
+		n := int32(len(fc.filerAddresses))
+		fc.filerAddressesMu.RUnlock()
+
+		for x := int32(0); x < n; x++ {
+			// Get current filer address and health with read lock
+			fc.filerAddressesMu.RLock()
+			if len(fc.filerAddresses) == 0 {
+				fc.filerAddressesMu.RUnlock()
+				lastErr = fmt.Errorf("no filers available")
+				break
+			}
+			if i >= int32(len(fc.filerAddresses)) {
+				// Filer list changed, reset index
+				i = 0
+			}
+
+			// Get health pointer while holding lock
+			health := fc.filerHealth[i]
+			filerAddress := fc.filerAddresses[i]
+			fc.filerAddressesMu.RUnlock()
+
+			// Circuit breaker: skip unhealthy filers (no lock needed - uses atomics)
+			if fc.shouldSkipUnhealthyFilerWithHealth(health) {
+				glog.V(2).Infof("FilerClient: skipping unhealthy filer %s (consecutive failures: %d)",
+					filerAddress, atomic.LoadInt32(&health.failureCount))
+				i++
+				if i >= n {
+					i = 0
+				}
+				continue
+			}
+
+			// Use anonymous function to ensure defer cancel() is called per iteration, not accumulated
+			err := func() error {
+				// Create a fresh timeout context for each filer attempt
+				// This ensures each retry gets the full grpcTimeout, not a diminishing deadline
+				timeoutCtx, cancel := context.WithTimeout(ctx, fc.grpcTimeout)
+				defer cancel() // Always clean up context, even on panic or early return
+
+				return pb.WithGrpcFilerClient(false, fc.clientId, filerAddress, fc.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+					resp, err := client.LookupVolume(timeoutCtx, &filer_pb.LookupVolumeRequest{
+						VolumeIds: volumeIds,
+					})
+					if err != nil {
+						return fmt.Errorf("filer.LookupVolume failed: %w", err)
+					}
+
+					// Process each volume in the response
+					for vid, locs := range resp.LocationsMap {
+						// Convert locations from protobuf to internal format
+						var locations []Location
+						for _, loc := range locs.Locations {
+							locations = append(locations, Location{
+								Url:        loc.Url,
+								PublicUrl:  loc.PublicUrl,
+								DataCenter: loc.DataCenter,
+								GrpcPort:   int(loc.GrpcPort),
+							})
+						}
+
+						// Only add to result if we have locations
+						// Empty locations with no gRPC error means "not found" (volume doesn't exist)
+						if len(locations) > 0 {
+							result[vid] = locations
+							glog.V(4).Infof("FilerClient: volume %s found with %d location(s)", vid, len(locations))
+						} else {
+							glog.V(2).Infof("FilerClient: volume %s not found (no locations in response)", vid)
+						}
+					}
+
+					// Check for volumes that weren't in the response at all
+					// This could indicate a problem with the filer
+					for _, vid := range volumeIds {
+						if _, found := resp.LocationsMap[vid]; !found {
+							glog.V(1).Infof("FilerClient: volume %s missing from filer response", vid)
+						}
+					}
+
+					return nil
+				})
+			}()
+
+			if err != nil {
+				glog.V(1).Infof("FilerClient: filer %s lookup failed (attempt %d/%d, retry %d/%d): %v", filerAddress, x+1, n, retry+1, maxRetries, err)
+				fc.recordFilerFailureWithHealth(health)
+				lastErr = err
+				i++
+				if i >= n {
+					i = 0
+				}
+				continue
+			}
+
+			// Success - update the preferred filer index and reset health tracking
+			atomic.StoreInt32(&fc.filerIndex, i)
+			fc.recordFilerSuccessWithHealth(health)
+			glog.V(3).Infof("FilerClient: looked up %d volumes on %s, found %d", len(volumeIds), filerAddress, len(result))
+			return result, nil
+		}
+
+		// All filers failed on this attempt
+		// Check if the error is retryable (transient gRPC error)
+		if !isRetryableGrpcError(lastErr) {
+			// Non-retryable error (e.g., NotFound, PermissionDenied) - fail immediately
+			return nil, fmt.Errorf("all %d filer(s) failed with non-retryable error: %w", n, lastErr)
+		}
+
+		// Transient error - retry if we have attempts left
+		if retry < maxRetries-1 {
+			jitteredWait := jitter(waitTime)
+			glog.V(1).Infof("FilerClient: all %d filer(s) failed with retryable error (attempt %d/%d), retrying in %v: %v",
+				n, retry+1, maxRetries, jitteredWait, lastErr)
+			timer := time.NewTimer(jitteredWait)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return nil, ctx.Err()
+			case <-timer.C:
+			}
+			waitTime = time.Duration(float64(waitTime) * fc.retryBackoffFactor)
+		}
+	}
+
+	// All retries exhausted
+	fc.filerAddressesMu.RLock()
+	totalFilers := len(fc.filerAddresses)
+	fc.filerAddressesMu.RUnlock()
+	return nil, fmt.Errorf("all %d filer(s) failed after %d attempts, last error: %w", totalFilers, maxRetries, lastErr)
+}

weed/wdclient/filer_client_test.go

@@ -0,0 +1,102 @@
+package wdclient
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func newTestFilerClient(addrs ...pb.ServerAddress) *FilerClient {
+	health := make([]*filerHealth, len(addrs))
+	for i := range health {
+		health[i] = &filerHealth{}
+	}
+	return &FilerClient{
+		filerAddresses: addrs,
+		filerHealth:    health,
+	}
+}
+
+func filerAddressList(fc *FilerClient) []pb.ServerAddress {
+	out := make([]pb.ServerAddress, len(fc.filerAddresses))
+	copy(out, fc.filerAddresses)
+	return out
+}
+
+func TestApplyDiscoveredFilersPrunesStaleAddress(t *testing.T) {
+	a := pb.ServerAddress("10.0.0.1:18888")
+	b := pb.ServerAddress("10.0.0.2:18888") // gets replaced by c
+	c := pb.ServerAddress("10.0.0.3:18888")
+
+	fc := newTestFilerClient(a, b)
+	// Give b a non-trivial failure count so we can confirm it leaves with its health.
+	atomic.StoreInt32(&fc.filerHealth[1].failureCount, 7)
+	// Give a a known failure count so we can confirm survivor health is preserved.
+	atomic.StoreInt32(&fc.filerHealth[0].failureCount, 2)
+	atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b
+
+	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
+		a: {},
+		c: {},
+	})
+
+	got := filerAddressList(fc)
+	if len(got) != 2 || got[0] != a || got[1] != c {
+		t.Fatalf("expected [%s %s], got %v", a, c, got)
+	}
+	if len(fc.filerHealth) != 2 {
+		t.Fatalf("expected 2 health entries, got %d", len(fc.filerHealth))
+	}
+	if got := atomic.LoadInt32(&fc.filerHealth[0].failureCount); got != 2 {
+		t.Errorf("survivor health was reset: want 2, got %d", got)
+	}
+	if got := atomic.LoadInt32(&fc.filerHealth[1].failureCount); got != 0 {
+		t.Errorf("newly added filer should start with fresh health, got failureCount=%d", got)
+	}
+	// Active filer (b) disappeared; index must reset rather than dangle.
+	if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
+		t.Errorf("expected filerIndex reset to 0 after active filer removed, got %d", idx)
+	}
+}
+
+func TestApplyDiscoveredFilersKeepsIndexOnSurvivor(t *testing.T) {
+	a := pb.ServerAddress("10.0.0.1:18888") // gets replaced
+	b := pb.ServerAddress("10.0.0.2:18888") // active, survives
+	c := pb.ServerAddress("10.0.0.3:18888") // new
+
+	fc := newTestFilerClient(a, b)
+	atomic.StoreInt32(&fc.filerIndex, 1) // active filer is b
+
+	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
+		b: {},
+		c: {},
+	})
+
+	got := filerAddressList(fc)
+	if len(got) != 2 || got[0] != b || got[1] != c {
+		t.Fatalf("expected [%s %s], got %v", b, c, got)
+	}
+	// b moved to index 0 after a was pruned; index should follow.
+	if idx := atomic.LoadInt32(&fc.filerIndex); idx != 0 {
+		t.Errorf("expected filerIndex to follow surviving active filer to position 0, got %d", idx)
+	}
+}
+
+func TestApplyDiscoveredFilersNoChangeIsNoop(t *testing.T) {
+	a := pb.ServerAddress("10.0.0.1:18888")
+	b := pb.ServerAddress("10.0.0.2:18888")
+
+	fc := newTestFilerClient(a, b)
+	originalHealthA := fc.filerHealth[0]
+	originalHealthB := fc.filerHealth[1]
+
+	fc.applyDiscoveredFilers(map[pb.ServerAddress]struct{}{
+		a: {},
+		b: {},
+	})
+
+	if fc.filerHealth[0] != originalHealthA || fc.filerHealth[1] != originalHealthB {
+		t.Errorf("no-op refresh should not reallocate health entries")
+	}
+}

weed/wdclient/jitter_test.go

@@ -0,0 +1,65 @@
+package wdclient
+
+import (
+	"testing"
+	"time"
+)
+
+func TestJitterBounds(t *testing.T) {
+	cases := []time.Duration{
+		1 * time.Millisecond,
+		100 * time.Millisecond,
+		1 * time.Second,
+		5 * time.Second,
+	}
+
+	for _, d := range cases {
+		for i := 0; i < 100; i++ {
+			j := jitter(d)
+			if j < d/2 {
+				t.Errorf("jitter(%v) = %v, below lower bound %v", d, j, d/2)
+			}
+			if j >= d {
+				t.Errorf("jitter(%v) = %v, at or above upper bound %v", d, j, d)
+			}
+		}
+	}
+}
+
+func TestJitterZeroAndNegative(t *testing.T) {
+	if j := jitter(0); j != 0 {
+		t.Errorf("jitter(0) = %v, want 0", j)
+	}
+	if j := jitter(-1 * time.Second); j != 0 {
+		t.Errorf("jitter(-1s) = %v, want 0", j)
+	}
+}
+
+func TestJitterTinyDuration(t *testing.T) {
+	// When d < 2, half == 0 and rand.Int63n(0) panics.
+	// We should return d unmodified in that case.
+	j := jitter(1 * time.Nanosecond)
+	if j != 1*time.Nanosecond {
+		t.Errorf("jitter(1ns) = %v, want 1ns", j)
+	}
+}
+
+func TestJitterDistribution(t *testing.T) {
+	const iterations = 10000
+	const base = 100 * time.Millisecond
+	var sum time.Duration
+
+	for i := 0; i < iterations; i++ {
+		j := jitter(base)
+		sum += j
+	}
+
+	avg := sum / iterations
+	// Equal jitter average should be around 75% of base (midpoint of [50%, 100%))
+	expected := base * 3 / 4
+	tolerance := base / 10 // ±10%
+
+	if avg < expected-tolerance || avg > expected+tolerance {
+		t.Errorf("average jitter %v deviated from expected %v", avg, expected)
+	}
+}

weed/wdclient/masterclient.go

@@ -0,0 +1,621 @@
+package wdclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+)
+
+// masterVolumeProvider implements VolumeLocationProvider by querying master
+// This is rarely called since master pushes updates proactively via KeepConnected stream
+type masterVolumeProvider struct {
+	masterClient *MasterClient
+}
+
+func isCanceledErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	if statusErr, ok := status.FromError(err); ok {
+		switch statusErr.Code() {
+		case codes.Canceled, codes.DeadlineExceeded:
+			return true
+		}
+	}
+	return false
+}
+
+// LookupVolumeIds queries the master for volume locations (fallback when cache misses).
+// Returns partial results with aggregated errors for volumes that failed.
+// Retries on codes.Unavailable (e.g. master warming up after restart) with backoff.
+func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
+	var result map[string][]Location
+	var lookupErrors []error
+
+	glog.V(2).Infof("Looking up %d volumes from master: %v", len(volumeIds), volumeIds)
+
+	retryErr := util.RetryWithBackoff(ctx, "lookup", 30*time.Second,
+		func(err error) bool {
+			st, ok := status.FromError(err)
+			return ok && st.Code() == codes.Unavailable
+		},
+		func() error {
+			result = make(map[string][]Location)
+			lookupErrors = nil
+
+			// Per-attempt timeout bounds both master resolution and the RPC
+			// so a single attempt cannot consume the entire retry budget.
+			timeoutCtx, cancel := context.WithTimeout(ctx, p.masterClient.grpcTimeout)
+			defer cancel()
+
+			master := p.masterClient.GetMaster(timeoutCtx)
+			if master == "" {
+				if ctx.Err() != nil {
+					return ctx.Err()
+				}
+				return status.Errorf(codes.Unavailable, "no master available")
+			}
+
+			return pb.WithMasterClient(timeoutCtx, false, master, p.masterClient.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+				resp, err := client.LookupVolume(timeoutCtx, &master_pb.LookupVolumeRequest{
+					VolumeOrFileIds: volumeIds,
+				})
+				if err != nil {
+					return err
+				}
+
+				for _, vidLoc := range resp.VolumeIdLocations {
+					// Preserve per-volume errors from master response
+					// These could indicate misconfiguration, volume deletion, etc.
+					if vidLoc.Error != "" {
+						lookupErrors = append(lookupErrors, fmt.Errorf("volume %s: %s", vidLoc.VolumeOrFileId, vidLoc.Error))
+						glog.V(1).Infof("volume %s lookup error from master: %s", vidLoc.VolumeOrFileId, vidLoc.Error)
+						continue
+					}
+
+					// Parse volume ID from response
+					parts := strings.Split(vidLoc.VolumeOrFileId, ",")
+					vidOnly := parts[0]
+					vid, err := strconv.ParseUint(vidOnly, 10, 32)
+					if err != nil {
+						lookupErrors = append(lookupErrors, fmt.Errorf("volume %s: invalid volume ID format: %w", vidLoc.VolumeOrFileId, err))
+						glog.Warningf("Failed to parse volume id '%s' from master response '%s': %v", vidOnly, vidLoc.VolumeOrFileId, err)
+						continue
+					}
+
+					var locations []Location
+					for _, masterLoc := range vidLoc.Locations {
+						loc := Location{
+							Url:               masterLoc.Url,
+							PublicUrl:         masterLoc.PublicUrl,
+							GrpcPort:          int(masterLoc.GrpcPort),
+							DataCenter:        masterLoc.DataCenter,
+							DataInRemote:      masterLoc.DataInRemote,
+							ReadOnly:          masterLoc.ReadOnly,
+							ReadOnlyCanDelete: masterLoc.ReadOnlyCanDelete,
+						}
+						// Update cache with the location
+						p.masterClient.addLocation(uint32(vid), loc)
+						locations = append(locations, loc)
+					}
+
+					if len(locations) > 0 {
+						result[vidOnly] = locations
+					}
+				}
+				return nil
+			})
+		})
+	if retryErr != nil {
+		return nil, retryErr
+	}
+
+	// Return partial results with detailed errors
+	// Callers should check both result map and error
+	if len(lookupErrors) > 0 {
+		glog.V(2).Infof("MasterClient: looked up %d volumes, found %d, %d errors", len(volumeIds), len(result), len(lookupErrors))
+		return result, fmt.Errorf("master volume lookup errors: %w", errors.Join(lookupErrors...))
+	}
+
+	glog.V(3).Infof("MasterClient: looked up %d volumes, found %d", len(volumeIds), len(result))
+	return result, nil
+}
+
+// MasterClient connects to master servers and maintains volume location cache
+// It receives real-time updates via KeepConnected streaming and uses vidMapClient for caching
+type MasterClient struct {
+	*vidMapClient // Embedded cache with shared logic
+
+	FilerGroup           string
+	clientType           string
+	clientHost           pb.ServerAddress
+	rack                 string
+	currentMaster        pb.ServerAddress
+	currentMasterLock    sync.RWMutex
+	masters              pb.ServerDiscovery
+	grpcDialOption       grpc.DialOption
+	grpcTimeout          time.Duration // Timeout for gRPC calls to master
+	OnPeerUpdate         func(update *master_pb.ClusterNodeUpdate, startFrom time.Time)
+	OnPeerUpdateLock     sync.RWMutex
+	OnLockRingUpdate     func(update *master_pb.LockRingUpdate)
+	OnLockRingUpdateLock sync.RWMutex
+}
+
+func NewMasterClient(grpcDialOption grpc.DialOption, filerGroup string, clientType string, clientHost pb.ServerAddress, clientDataCenter string, rack string, masters pb.ServerDiscovery) *MasterClient {
+	mc := &MasterClient{
+		FilerGroup:     filerGroup,
+		clientType:     clientType,
+		clientHost:     clientHost,
+		rack:           rack,
+		masters:        masters,
+		grpcDialOption: grpcDialOption,
+		grpcTimeout:    5 * time.Second, // Default: 5 seconds for gRPC calls to master
+	}
+
+	// Create provider that references this MasterClient
+	provider := &masterVolumeProvider{masterClient: mc}
+
+	// Initialize embedded vidMapClient with the provider and default cache size
+	mc.vidMapClient = newVidMapClient(provider, clientDataCenter, DefaultVidMapCacheSize)
+
+	return mc
+}
+
+func (mc *MasterClient) SetOnPeerUpdateFn(onPeerUpdate func(update *master_pb.ClusterNodeUpdate, startFrom time.Time)) {
+	mc.OnPeerUpdateLock.Lock()
+	mc.OnPeerUpdate = onPeerUpdate
+	mc.OnPeerUpdateLock.Unlock()
+}
+
+func (mc *MasterClient) SetOnLockRingUpdateFn(fn func(update *master_pb.LockRingUpdate)) {
+	mc.OnLockRingUpdateLock.Lock()
+	mc.OnLockRingUpdate = fn
+	mc.OnLockRingUpdateLock.Unlock()
+}
+
+func (mc *MasterClient) tryAllMasters(ctx context.Context) {
+	var nextHintedLeader pb.ServerAddress
+	failedMasters := make(map[pb.ServerAddress]struct{})
+	mc.masters.RefreshBySrvIfAvailable()
+	for _, master := range mc.masters.GetInstances() {
+		if _, failed := failedMasters[master]; failed {
+			continue
+		}
+		nextHintedLeader = mc.tryConnectToMaster(ctx, master)
+		for nextHintedLeader != "" {
+			if _, failed := failedMasters[nextHintedLeader]; failed {
+				break // don't follow redirect to a known-unreachable master
+			}
+			select {
+			case <-ctx.Done():
+				glog.V(0).Infof("Connection attempt to all masters stopped: %v", ctx.Err())
+				return
+			default:
+				target := nextHintedLeader
+				nextHintedLeader = mc.tryConnectToMaster(ctx, target)
+				if nextHintedLeader == "" {
+					// connection to target failed; remember it so we skip
+					// stale redirects pointing back to it this cycle
+					failedMasters[target] = struct{}{}
+				}
+			}
+		}
+		mc.setCurrentMaster("")
+	}
+}
+
+func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.ServerAddress) (nextHintedLeader pb.ServerAddress) {
+	glog.V(1).Infof("%s.%s masterClient Connecting to master %v", mc.FilerGroup, mc.clientType, master)
+	stats.MasterClientConnectCounter.WithLabelValues("total").Inc()
+	connectStartTime := time.Now()
+	gprcErr := pb.WithMasterClient(context.Background(), true, master, mc.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		stream, err := client.KeepConnected(ctx)
+		if err != nil {
+			glog.V(1).Infof("%s.%s masterClient failed to keep connected to %s: %v", mc.FilerGroup, mc.clientType, master, err)
+			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToKeepConnected).Inc()
+			return err
+		}
+		glog.V(1).Infof("%s.%s masterClient gRPC stream established to %s in %v", mc.FilerGroup, mc.clientType, master, time.Since(connectStartTime))
+
+		if err = stream.Send(&master_pb.KeepConnectedRequest{
+			FilerGroup:    mc.FilerGroup,
+			DataCenter:    mc.GetDataCenter(),
+			Rack:          mc.rack,
+			ClientType:    mc.clientType,
+			ClientAddress: string(mc.clientHost),
+			Version:       version.Version(),
+		}); err != nil {
+			glog.V(0).Infof("%s.%s masterClient failed to send to %s: %v", mc.FilerGroup, mc.clientType, master, err)
+			stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToSend).Inc()
+			return err
+		}
+		glog.V(1).Infof("%s.%s masterClient Connected to %v", mc.FilerGroup, mc.clientType, master)
+
+		resp, err := stream.Recv()
+		if err != nil {
+			canceled := isCanceledErr(err) || ctx.Err() != nil
+			if canceled {
+				glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+			} else {
+				glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+				stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+			}
+			return err
+		}
+
+		// check if it is the leader to determine whether to reset the vidMap
+		if resp.VolumeLocation != nil {
+			if resp.VolumeLocation.Leader != "" && !master.Equals(pb.ServerAddress(resp.VolumeLocation.Leader)) {
+				glog.V(1).Infof("master %v redirected to leader %v", master, resp.VolumeLocation.Leader)
+				nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
+				stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
+				return nil
+			}
+			mc.resetVidMap()
+			mc.updateVidMap(resp)
+		} else {
+			// First message from master is not VolumeLocation (e.g., ClusterNodeUpdate)
+			// Still need to reset cache to ensure we don't use stale data from previous master
+			mc.resetVidMap()
+		}
+		mc.setCurrentMaster(master)
+
+		for {
+			resp, err := stream.Recv()
+			if err != nil {
+				canceled := isCanceledErr(err) || ctx.Err() != nil
+				if canceled {
+					glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+				} else {
+					glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
+					stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
+				}
+				return err
+			}
+
+			if resp.VolumeLocation != nil {
+				// Check for leader change during the stream
+				// If master announces a new leader, reconnect to it
+				if currentMaster := mc.GetMaster(ctx); resp.VolumeLocation.Leader != "" && !currentMaster.Equals(pb.ServerAddress(resp.VolumeLocation.Leader)) {
+					glog.V(1).Infof("currentMaster %v redirected to leader %v", currentMaster, resp.VolumeLocation.Leader)
+					nextHintedLeader = pb.ServerAddress(resp.VolumeLocation.Leader)
+					stats.MasterClientConnectCounter.WithLabelValues(stats.RedirectedToLeader).Inc()
+					return nil
+				}
+				mc.updateVidMap(resp)
+			}
+			if resp.ClusterNodeUpdate != nil {
+				update := resp.ClusterNodeUpdate
+				mc.OnPeerUpdateLock.RLock()
+				if mc.OnPeerUpdate != nil {
+					if update.FilerGroup == mc.FilerGroup {
+						if update.IsAdd {
+							glog.V(0).Infof("+ %s@%s noticed %s.%s %s\n", mc.clientType, mc.clientHost, update.FilerGroup, update.NodeType, update.Address)
+						} else {
+							glog.V(0).Infof("- %s@%s noticed %s.%s %s\n", mc.clientType, mc.clientHost, update.FilerGroup, update.NodeType, update.Address)
+						}
+						stats.MasterClientConnectCounter.WithLabelValues(stats.OnPeerUpdate).Inc()
+						mc.OnPeerUpdate(update, time.Now())
+					}
+				}
+				mc.OnPeerUpdateLock.RUnlock()
+			}
+			if resp.LockRingUpdate != nil {
+				update := resp.LockRingUpdate
+				mc.OnLockRingUpdateLock.RLock()
+				if mc.OnLockRingUpdate != nil {
+					if update.FilerGroup == mc.FilerGroup {
+						glog.V(0).Infof("LockRing: %s@%s received ring update v%d: %v", mc.clientType, mc.clientHost, update.Version, update.Servers)
+						mc.OnLockRingUpdate(update)
+					}
+				}
+				mc.OnLockRingUpdateLock.RUnlock()
+			}
+			if err := ctx.Err(); err != nil {
+				if isCanceledErr(err) {
+					glog.V(1).Infof("Connection attempt to master stopped: %v", err)
+				} else {
+					glog.V(0).Infof("Connection attempt to master stopped: %v", err)
+				}
+				return err
+			}
+		}
+	})
+	if gprcErr != nil {
+		if isCanceledErr(gprcErr) || ctx.Err() != nil {
+			glog.V(1).Infof("%s.%s masterClient connection closed to %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
+			return nextHintedLeader
+		}
+		stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
+		glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
+	}
+	return nextHintedLeader
+}
+
+// addedVids indexes added ids that are also being removed. A volume moved
+// between a server's disks is reported both ways in one message, and the server
+// still has it -- acting on the removal would drop a good location, whichever
+// order the two lists happen to be applied in. Empty unless both lists are
+// non-empty, so the full id list a client gets on connect costs nothing.
+func addedVids(added, removed []uint32) map[uint32]struct{} {
+	if len(added) == 0 || len(removed) == 0 {
+		return nil
+	}
+	index := make(map[uint32]struct{}, len(added))
+	for _, vid := range added {
+		index[vid] = struct{}{}
+	}
+	return index
+}
+
+// updateVidMap applies a KeepConnectedResponse volume-location message to the
+// local vidMap. NewVids adds the entries; RemoteVids names the subset of them
+// backed by remote storage, which is added with DataInRemote so read paths can
+// prefer the cheap local replica. DeletedVids drops the named entry unless the
+// same message also added it back (volume moved between this server's disks).
+// EC vid changes go through the parallel addEcLocation / deleteEcLocation pair.
+func (mc *MasterClient) updateVidMap(resp *master_pb.KeepConnectedResponse) {
+	if resp.VolumeLocation.IsEmptyUrl() {
+		glog.V(0).Infof("updateVidMap ignore short heartbeat: %+v", resp)
+		return
+	}
+	// process new volume location
+	loc := Location{
+		Url:        resp.VolumeLocation.Url,
+		PublicUrl:  resp.VolumeLocation.PublicUrl,
+		DataCenter: resp.VolumeLocation.DataCenter,
+		GrpcPort:   int(resp.VolumeLocation.GrpcPort),
+	}
+	stillOnServer := addedVids(resp.VolumeLocation.NewVids, resp.VolumeLocation.DeletedVids)
+	// RemoteVids repeats ids NewVids already carries, so the tier is settled
+	// before anything is written rather than adding each one twice.
+	var remoteVids map[uint32]struct{}
+	if len(resp.VolumeLocation.RemoteVids) > 0 {
+		remoteVids = make(map[uint32]struct{}, len(resp.VolumeLocation.RemoteVids))
+		for _, vid := range resp.VolumeLocation.RemoteVids {
+			remoteVids[vid] = struct{}{}
+		}
+	}
+	var readOnlyVids map[uint32]struct{}
+	if len(resp.VolumeLocation.ReadOnlyVids) > 0 {
+		readOnlyVids = make(map[uint32]struct{}, len(resp.VolumeLocation.ReadOnlyVids))
+		for _, vid := range resp.VolumeLocation.ReadOnlyVids {
+			readOnlyVids[vid] = struct{}{}
+		}
+	}
+	var readOnlyCanDeleteVids map[uint32]struct{}
+	if len(resp.VolumeLocation.ReadOnlyCanDeleteVids) > 0 {
+		readOnlyCanDeleteVids = make(map[uint32]struct{}, len(resp.VolumeLocation.ReadOnlyCanDeleteVids))
+		for _, vid := range resp.VolumeLocation.ReadOnlyCanDeleteVids {
+			readOnlyCanDeleteVids[vid] = struct{}{}
+		}
+	}
+	for _, newVid := range resp.VolumeLocation.NewVids {
+		if _, isRemote := remoteVids[newVid]; isRemote {
+			continue
+		}
+		newLoc := loc
+		if _, isReadOnly := readOnlyVids[newVid]; isReadOnly {
+			newLoc.ReadOnly = true
+		}
+		if _, canDelete := readOnlyCanDeleteVids[newVid]; canDelete {
+			newLoc.ReadOnlyCanDelete = true
+		}
+		glog.V(2).Infof("%s.%s: %s masterClient adds volume %d", mc.FilerGroup, mc.clientType, newLoc.Url, newVid)
+		mc.addLocation(newVid, newLoc)
+	}
+	for _, remoteVid := range resp.VolumeLocation.RemoteVids {
+		remoteLoc := loc
+		remoteLoc.DataInRemote = true
+		if _, isReadOnly := readOnlyVids[remoteVid]; isReadOnly {
+			remoteLoc.ReadOnly = true
+		}
+		if _, canDelete := readOnlyCanDeleteVids[remoteVid]; canDelete {
+			remoteLoc.ReadOnlyCanDelete = true
+		}
+		glog.V(2).Infof("%s.%s: %s masterClient adds remote volume %d", mc.FilerGroup, mc.clientType, remoteLoc.Url, remoteVid)
+		mc.addLocation(remoteVid, remoteLoc)
+	}
+	for _, deletedVid := range resp.VolumeLocation.DeletedVids {
+		if _, moved := stillOnServer[deletedVid]; moved {
+			continue
+		}
+		glog.V(2).Infof("%s.%s: %s masterClient removes volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedVid)
+		mc.deleteLocation(deletedVid, loc)
+	}
+	stillOnServerEc := addedVids(resp.VolumeLocation.NewEcVids, resp.VolumeLocation.DeletedEcVids)
+	for _, newEcVid := range resp.VolumeLocation.NewEcVids {
+		glog.V(2).Infof("%s.%s: %s masterClient adds ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, newEcVid)
+		mc.addEcLocation(newEcVid, loc)
+	}
+	for _, deletedEcVid := range resp.VolumeLocation.DeletedEcVids {
+		if _, moved := stillOnServerEc[deletedEcVid]; moved {
+			continue
+		}
+		glog.V(2).Infof("%s.%s: %s masterClient removes ec volume %d", mc.FilerGroup, mc.clientType, loc.Url, deletedEcVid)
+		mc.deleteEcLocation(deletedEcVid, loc)
+	}
+	glog.V(1).Infof("updateVidMap(%s) %s.%s: %s volume add local: %d, remote: %d, del: %d, add ec: %d del ec: %d",
+		resp.VolumeLocation.DataCenter, mc.FilerGroup, mc.clientType, loc.Url,
+		len(resp.VolumeLocation.NewVids)-len(resp.VolumeLocation.RemoteVids),
+		len(resp.VolumeLocation.RemoteVids),
+		len(resp.VolumeLocation.DeletedVids), len(resp.VolumeLocation.NewEcVids),
+		len(resp.VolumeLocation.DeletedEcVids))
+}
+
+func (mc *MasterClient) WithClient(ctx context.Context, streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
+	getMasterF := func() pb.ServerAddress {
+		return mc.GetMaster(ctx)
+	}
+	return mc.WithClientCustomGetMaster(ctx, getMasterF, streamingMode, fn)
+}
+
+// WithClientCustomGetMaster bounds the wait for a master leader by ctx, so a
+// caller with a deadline is not parked for the length of an election. The dial
+// still gets context.Background(): fn brings its own RPC context, so nothing
+// here can attribute a cancellation to the shared connection.
+func (mc *MasterClient) WithClientCustomGetMaster(ctx context.Context, getMasterF func() pb.ServerAddress, streamingMode bool, fn func(client master_pb.SeaweedClient) error) error {
+	return util.RetryWithBackoff(ctx, "master grpc", util.RetryWaitTime, util.IsTransientError, func() error {
+		master := getMasterF()
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		return pb.WithMasterClient(context.Background(), streamingMode, master, mc.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+			return fn(client)
+		})
+	})
+}
+
+func (mc *MasterClient) getCurrentMaster() pb.ServerAddress {
+	mc.currentMasterLock.RLock()
+	defer mc.currentMasterLock.RUnlock()
+	return mc.currentMaster
+}
+
+func (mc *MasterClient) setCurrentMaster(master pb.ServerAddress) {
+	mc.currentMasterLock.Lock()
+	mc.currentMaster = master
+	mc.currentMasterLock.Unlock()
+}
+
+// GetMaster returns the current master address, blocking until connected.
+//
+// IMPORTANT: This method blocks until KeepConnectedToMaster successfully establishes
+// a connection to a master server. If KeepConnectedToMaster hasn't been started in a
+// background goroutine, this will block indefinitely (or until ctx is canceled).
+//
+// Typical initialization pattern:
+//
+//	mc := wdclient.NewMasterClient(...)
+//	go mc.KeepConnectedToMaster(ctx)  // Start connection management
+//	// ... later ...
+//	master := mc.GetMaster(ctx)       // Will block until connected
+//
+// If called before KeepConnectedToMaster establishes a connection, this may cause
+// unexpected timeouts in LookupVolumeIds and other operations that depend on it.
+func (mc *MasterClient) GetMaster(ctx context.Context) pb.ServerAddress {
+	mc.WaitUntilConnected(ctx)
+	return mc.getCurrentMaster()
+}
+
+// GetMasters returns all configured master addresses, blocking until connected.
+// See GetMaster() for important initialization contract details.
+func (mc *MasterClient) GetMasters(ctx context.Context) []pb.ServerAddress {
+	mc.WaitUntilConnected(ctx)
+	return mc.masters.GetInstances()
+}
+
+// ListMasterSet returns a set of configured master addresses keyed by their
+// canonical http form. Unlike GetMasters this does not wait for a connection,
+// so it is safe to call from admission paths that must stay non-blocking.
+func (mc *MasterClient) ListMasterSet() map[string]struct{} {
+	addrs := mc.masters.GetInstances()
+	set := make(map[string]struct{}, len(addrs))
+	for _, a := range addrs {
+		set[a.ToHttpAddress()] = struct{}{}
+	}
+	return set
+}
+
+// WaitUntilConnected blocks until a master connection is established or ctx is canceled.
+// This does NOT initiate connections - it only waits for KeepConnectedToMaster to succeed.
+func (mc *MasterClient) WaitUntilConnected(ctx context.Context) {
+	attempts := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			currentMaster := mc.getCurrentMaster()
+			if currentMaster != "" {
+				return
+			}
+			attempts++
+			if attempts%100 == 0 { // Log every 100 attempts (roughly every 20 seconds)
+				glog.V(0).Infof("%s.%s WaitUntilConnected still waiting for master connection (attempt %d)...", mc.FilerGroup, mc.clientType, attempts)
+			}
+			// Use select with time.After to respect context cancellation during sleep
+			sleepDuration := time.Duration(rand.Int31n(200)) * time.Millisecond
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(sleepDuration):
+				// continue to next iteration
+			}
+		}
+	}
+}
+
+func (mc *MasterClient) KeepConnectedToMaster(ctx context.Context) {
+	glog.V(1).Infof("%s.%s masterClient bootstraps with masters %v", mc.FilerGroup, mc.clientType, mc.masters)
+	reconnectCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			if isCanceledErr(ctx.Err()) {
+				glog.V(1).Infof("Connection to masters stopped: %v", ctx.Err())
+			} else {
+				glog.V(0).Infof("Connection to masters stopped: %v", ctx.Err())
+			}
+			return
+		default:
+			reconnectStart := time.Now()
+			if reconnectCount > 0 {
+				glog.V(0).Infof("%s.%s masterClient reconnection attempt #%d", mc.FilerGroup, mc.clientType, reconnectCount)
+			}
+			mc.tryAllMasters(ctx)
+			reconnectCount++
+			glog.V(1).Infof("%s.%s masterClient connection cycle completed in %v, sleeping before retry",
+				mc.FilerGroup, mc.clientType, time.Since(reconnectStart))
+			time.Sleep(time.Second)
+		}
+	}
+}
+
+func (mc *MasterClient) FindLeaderFromOtherPeers(myMasterAddress pb.ServerAddress) (leader string) {
+	for _, master := range mc.masters.GetInstances() {
+		if master == myMasterAddress {
+			continue
+		}
+		if grpcErr := pb.WithMasterClient(context.Background(), false, master, mc.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+			ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond)
+			defer cancel()
+			resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
+			if err != nil {
+				return err
+			}
+			leader = resp.Leader
+			return nil
+		}); grpcErr != nil {
+			glog.V(0).Infof("connect to %s: %v", master, grpcErr)
+		}
+		if leader != "" {
+			glog.V(0).Infof("existing leader is %s", leader)
+			return
+		}
+	}
+	glog.V(0).Infof("No existing leader found!")
+	return
+}

weed/wdclient/masterclient_move_test.go

@@ -0,0 +1,93 @@
+package wdclient
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func moveClient() *MasterClient {
+	return &MasterClient{vidMapClient: newVidMapClient(nil, "", 0)}
+}
+
+func moveResponse(newVids, deletedVids []uint32) *master_pb.KeepConnectedResponse {
+	return &master_pb.KeepConnectedResponse{VolumeLocation: &master_pb.VolumeLocation{
+		Url: "server:8080", PublicUrl: "server:8080", NewVids: newVids, DeletedVids: deletedVids,
+	}}
+}
+
+// A volume moved between a server's disks arrives added and removed at once,
+// and the server still has it.
+func TestMovedVolumeKeepsItsLocation(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveResponse([]uint32{1}, nil))
+	mc.updateVidMap(moveResponse([]uint32{1}, []uint32{1}))
+
+	locations, found := mc.GetLocations(1)
+	if !found || len(locations) != 1 {
+		t.Errorf("a volume that moved between its server's disks lost its location: found=%v %v", found, locations)
+	}
+}
+
+func TestRemovedVolumeLosesItsLocation(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveResponse([]uint32{1}, nil))
+	mc.updateVidMap(moveResponse(nil, []uint32{1}))
+
+	if locations, found := mc.GetLocations(1); found && len(locations) > 0 {
+		t.Errorf("a volume that left the server kept its location: %v", locations)
+	}
+}
+
+func TestReadOnlyVolumeUpdatesItsLocation(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveResponse([]uint32{1}, nil))
+	mc.updateVidMap(&master_pb.KeepConnectedResponse{VolumeLocation: &master_pb.VolumeLocation{
+		Url: "server:8080", PublicUrl: "server:8080", NewVids: []uint32{1}, ReadOnlyVids: []uint32{1},
+	}})
+
+	locations, found := mc.GetLocations(1)
+	if !found || len(locations) != 1 || !locations[0].ReadOnly {
+		t.Fatalf("read-only update = %v, want one read-only location", locations)
+	}
+}
+
+// Removals of other volumes in the same message must still apply.
+func TestRemovalsAlongsideAMoveStillApply(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveResponse([]uint32{1, 2}, nil))
+	mc.updateVidMap(moveResponse([]uint32{1}, []uint32{1, 2}))
+
+	if locations, found := mc.GetLocations(1); !found || len(locations) != 1 {
+		t.Errorf("the moved volume lost its location: found=%v %v", found, locations)
+	}
+	if locations, found := mc.GetLocations(2); found && len(locations) > 0 {
+		t.Errorf("a volume that left the server kept its location: %v", locations)
+	}
+}
+
+func moveEcResponse(newEcVids, deletedEcVids []uint32) *master_pb.KeepConnectedResponse {
+	return &master_pb.KeepConnectedResponse{VolumeLocation: &master_pb.VolumeLocation{
+		Url: "server:8080", PublicUrl: "server:8080", NewEcVids: newEcVids, DeletedEcVids: deletedEcVids,
+	}}
+}
+
+func TestEcVolumeMovedKeepsItsLocation(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveEcResponse([]uint32{7}, nil))
+	mc.updateVidMap(moveEcResponse([]uint32{7}, []uint32{7}))
+
+	if locations, found := mc.GetLocations(7); !found || len(locations) != 1 {
+		t.Errorf("an ec volume reported both ways lost its location: found=%v %v", found, locations)
+	}
+}
+
+func TestEcVolumeRemovedLosesItsLocation(t *testing.T) {
+	mc := moveClient()
+	mc.updateVidMap(moveEcResponse([]uint32{7}, nil))
+	mc.updateVidMap(moveEcResponse(nil, []uint32{7}))
+
+	if locations, found := mc.GetLocations(7); found && len(locations) > 0 {
+		t.Errorf("an ec volume that left the server kept its location: %v", locations)
+	}
+}

weed/wdclient/masterclient_retry_test.go

@@ -0,0 +1,119 @@
+package wdclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+// fakeLookupServer returns Unavailable for the first N calls, then succeeds.
+type fakeLookupServer struct {
+	master_pb.UnimplementedSeaweedServer
+	unavailableCount int32
+	callCount        atomic.Int32
+}
+
+func (s *fakeLookupServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
+	n := s.callCount.Add(1)
+	if n <= s.unavailableCount {
+		return nil, status.Errorf(codes.Unavailable, "master is warming up")
+	}
+	resp := &master_pb.LookupVolumeResponse{}
+	for _, vid := range req.VolumeOrFileIds {
+		resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
+			VolumeOrFileId: vid,
+			Locations: []*master_pb.Location{
+				{Url: "127.0.0.1:8080", PublicUrl: "127.0.0.1:8080"},
+			},
+		})
+	}
+	return resp, nil
+}
+
+func startFakeMasterServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, srv)
+	go func() { _ = grpcServer.Serve(lis) }()
+	t.Cleanup(grpcServer.GracefulStop)
+
+	_, port, _ := net.SplitHostPort(lis.Addr().String())
+	return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
+}
+
+func TestLookupVolumeIdsRetriesOnUnavailable(t *testing.T) {
+	srv := &fakeLookupServer{unavailableCount: 3}
+	addr := startFakeMasterServer(t, srv)
+
+	mc := NewMasterClient(
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		"", "test", "", "", "",
+		pb.ServerDiscovery{},
+	)
+	mc.setCurrentMaster(addr)
+	mc.grpcTimeout = 5 * time.Second
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	provider := &masterVolumeProvider{masterClient: mc}
+	result, err := provider.LookupVolumeIds(ctx, []string{"1"})
+
+	if err != nil {
+		t.Fatalf("expected success after retries, got error: %v", err)
+	}
+	if _, ok := result["1"]; !ok {
+		t.Error("expected volume 1 in result")
+	}
+	if calls := srv.callCount.Load(); calls != 4 {
+		t.Errorf("expected 4 calls (3 unavailable + 1 success), got %d", calls)
+	}
+}
+
+func TestLookupVolumeIdsStopsOnContextCancel(t *testing.T) {
+	srv := &fakeLookupServer{unavailableCount: 1000}
+	addr := startFakeMasterServer(t, srv)
+
+	mc := NewMasterClient(
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		"", "test", "", "", "",
+		pb.ServerDiscovery{},
+	)
+	mc.setCurrentMaster(addr)
+	mc.grpcTimeout = 5 * time.Second
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	provider := &masterVolumeProvider{masterClient: mc}
+	start := time.Now()
+	_, err := provider.LookupVolumeIds(ctx, []string{"1"})
+	elapsed := time.Since(start)
+
+	// Verify the error is from context deadline
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Fatalf("expected context.DeadlineExceeded, got: %v", err)
+	}
+	// Verify the loop actually retried (not just an immediate failure)
+	if calls := srv.callCount.Load(); calls <= 1 {
+		t.Errorf("expected multiple retry attempts, got %d calls", calls)
+	}
+	if elapsed > 5*time.Second {
+		t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed)
+	}
+}

weed/wdclient/masterclient_test.go

@@ -0,0 +1,186 @@
+package wdclient
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+// TestWaitUntilConnectedWithoutKeepConnected verifies that WaitUntilConnected
+// respects context cancellation when KeepConnectedToMaster is not running.
+// This tests the fix for https://github.com/seaweedfs/seaweedfs/issues/7721
+func TestWaitUntilConnectedWithoutKeepConnected(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	// Without KeepConnectedToMaster running, WaitUntilConnected should
+	// respect context cancellation and not block forever
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	mc.WaitUntilConnected(ctx)
+	elapsed := time.Since(start)
+
+	// Should have returned due to context timeout, not blocked forever
+	if elapsed > 200*time.Millisecond {
+		t.Errorf("WaitUntilConnected blocked for %v, expected to return on context timeout", elapsed)
+	}
+
+	// GetMaster should return empty since we never connected
+	master := mc.getCurrentMaster()
+	if master != "" {
+		t.Errorf("Expected empty master, got %s", master)
+	}
+}
+
+// TestWaitUntilConnectedReturnsImmediatelyWhenConnected verifies that
+// WaitUntilConnected returns immediately when a master is already set.
+func TestWaitUntilConnectedReturnsImmediatelyWhenConnected(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	// Simulate that KeepConnectedToMaster has already established a connection
+	mc.setCurrentMaster("localhost:9333")
+
+	ctx := context.Background()
+	start := time.Now()
+	mc.WaitUntilConnected(ctx)
+	elapsed := time.Since(start)
+
+	// Should return almost immediately (< 10ms)
+	if elapsed > 10*time.Millisecond {
+		t.Errorf("WaitUntilConnected took %v when master was already set, expected immediate return", elapsed)
+	}
+
+	// Verify master is returned
+	master := mc.getCurrentMaster()
+	if master != "localhost:9333" {
+		t.Errorf("Expected master localhost:9333, got %s", master)
+	}
+}
+
+// TestGetMasterRespectsContextCancellation verifies that GetMaster
+// respects context cancellation and doesn't block forever.
+func TestGetMasterRespectsContextCancellation(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	// GetMaster calls WaitUntilConnected internally
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	master := mc.GetMaster(ctx)
+	elapsed := time.Since(start)
+
+	// Should return on context timeout
+	if elapsed > 200*time.Millisecond {
+		t.Errorf("GetMaster blocked for %v, expected to return on context timeout", elapsed)
+	}
+
+	// Master should be empty since we never connected
+	if master != "" {
+		t.Errorf("Expected empty master when not connected, got %s", master)
+	}
+}
+
+// TestMasterClientFilerGroupLogging verifies the FilerGroup is properly set
+// and would be logged correctly (regression test for issue #7721 log message format)
+func TestMasterClientFilerGroupLogging(t *testing.T) {
+	filerGroup := "filer_1"
+	clientType := "s3"
+
+	mc := NewMasterClient(grpc.EmptyDialOption{}, filerGroup, clientType, "", "", "", pb.ServerDiscovery{})
+
+	if mc.FilerGroup != filerGroup {
+		t.Errorf("Expected FilerGroup %s, got %s", filerGroup, mc.FilerGroup)
+	}
+
+	if mc.clientType != clientType {
+		t.Errorf("Expected clientType %s, got %s", clientType, mc.clientType)
+	}
+}
+
+// TestWithClientStopsWaitingOnCanceledContext verifies that WithClient hands the
+// caller's context to the wait for a master leader, so a caller that has already
+// given up is not parked until an election finishes.
+func TestWithClientStopsWaitingOnCanceledContext(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	called := false
+	start := time.Now()
+	err := mc.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		called = true
+		return nil
+	})
+	elapsed := time.Since(start)
+
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("expected context.Canceled, got %v", err)
+	}
+	if called {
+		t.Error("callback ran without a master leader")
+	}
+	if elapsed > time.Second {
+		t.Errorf("WithClient blocked for %v with an already canceled context", elapsed)
+	}
+}
+
+// TestWithClientStopsWaitingOnDeadline verifies the same bound applies to a
+// deadline the caller set rather than an outright cancellation.
+func TestWithClientStopsWaitingOnDeadline(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	err := mc.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		return nil
+	})
+	elapsed := time.Since(start)
+
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("expected context.DeadlineExceeded, got %v", err)
+	}
+	if elapsed > time.Second {
+		t.Errorf("WithClient blocked for %v past a 100ms deadline", elapsed)
+	}
+}
+
+// TestWithClientStopsBackoffOnCancel verifies that a cancellation arriving while
+// the retry is backing off cuts the backoff short rather than sleeping it out.
+func TestWithClientStopsBackoffOnCancel(t *testing.T) {
+	mc := NewMasterClient(grpc.WithTransportCredentials(insecure.NewCredentials()), "test-group", "test-client", "", "", "", pb.ServerDiscovery{})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	getMasterF := func() pb.ServerAddress { return "localhost:19333" }
+
+	attempts := 0
+	start := time.Now()
+	err := mc.WithClientCustomGetMaster(ctx, getMasterF, false, func(client master_pb.SeaweedClient) error {
+		attempts++
+		cancel()
+		return errors.New("connection reset by peer")
+	})
+	elapsed := time.Since(start)
+
+	if attempts != 1 {
+		t.Errorf("expected 1 attempt, got %d", attempts)
+	}
+	if !errors.Is(err, context.Canceled) {
+		t.Errorf("expected context.Canceled, got %v", err)
+	}
+	if elapsed > time.Second {
+		t.Errorf("WithClientCustomGetMaster slept %v after the context was canceled", elapsed)
+	}
+}

weed/wdclient/vid_map.go

@@ -0,0 +1,431 @@
+package wdclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type HasLookupFileIdFunction interface {
+	GetLookupFileIdFunction() LookupFileIdFunctionType
+}
+
+type LookupFileIdFunctionType func(ctx context.Context, fileId string) (targetUrls []string, err error)
+
+type Location struct {
+	Url               string `json:"url,omitempty"`
+	PublicUrl         string `json:"publicUrl,omitempty"`
+	DataCenter        string `json:"dataCenter,omitempty"`
+	GrpcPort          int    `json:"grpcPort,omitempty"`
+	DataInRemote      bool   `json:"dataInRemote,omitempty"`
+	ReadOnly          bool   `json:"readOnly,omitempty"`
+	ReadOnlyCanDelete bool   `json:"readOnlyCanDelete,omitempty"`
+}
+
+func (l Location) ServerAddress() pb.ServerAddress {
+	return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort)
+}
+
+// locationsEntry is what a volume id maps to: the locations themselves plus the
+// generation they were learned in. An entry is immutable once stored; every
+// update installs a new one, so locations handed to a reader are never
+// rewritten underneath it.
+type locationsEntry struct {
+	locations  []Location
+	generation uint64
+}
+
+type vidMap struct {
+	sync.RWMutex
+	vid2Locations   map[uint32]*locationsEntry
+	ecVid2Locations map[uint32]*locationsEntry
+	// serverRefCount tracks how many vid locations (regular + EC) currently
+	// reference each volume server address. Maintaining it incrementally lets
+	// hasVolumeServer answer in O(1) instead of walking every volume entry.
+	// Keys are the canonical http form of pb.ServerAddress, so callers that
+	// pass either "host:port" or "host:port.grpc" find the same entry.
+	serverRefCount map[string]int
+	DataCenter     string
+	// generation counts resets. Each entry remembers the generation it was
+	// learned in, so history expires per volume rather than by keeping
+	// snapshot copies of the whole map.
+	generation uint64
+	// retainGenerations is how many resets an entry survives without being
+	// refreshed before reset drops it.
+	retainGenerations uint64
+}
+
+func newVidMap(dataCenter string, retainGenerations int) *vidMap {
+	if retainGenerations <= 0 {
+		retainGenerations = DefaultVidMapCacheSize
+	}
+	return &vidMap{
+		vid2Locations:     make(map[uint32]*locationsEntry),
+		ecVid2Locations:   make(map[uint32]*locationsEntry),
+		serverRefCount:    make(map[string]int),
+		DataCenter:        dataCenter,
+		retainGenerations: uint64(retainGenerations),
+	}
+}
+
+// locationServerKey returns the index key used by serverRefCount for a
+// Location. The key normalises away the optional grpc-port suffix so the
+// counter stays consistent with hasVolumeServer's lookup.
+func locationServerKey(loc Location) string {
+	return loc.ServerAddress().ToHttpAddress()
+}
+
+func (vc *vidMap) isSameDataCenter(loc *Location) bool {
+	if vc.DataCenter == "" || loc.DataCenter == "" || vc.DataCenter != loc.DataCenter {
+		return false
+	}
+	return true
+}
+
+// LookupVolumeServerUrl returns the cached volume-server URLs for vid in
+// preference order: same-DC local, same-DC remote-tier, then the other data
+// centers on the same footing. Within each group the order is randomized so
+// load spreads across equivalent servers.
+func (vc *vidMap) LookupVolumeServerUrl(vid string) (serverUrls []string, err error) {
+	id, err := strconv.Atoi(vid)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return nil, err
+	}
+
+	locations, found := vc.GetLocations(uint32(id))
+	if !found {
+		return nil, fmt.Errorf("volume %d not found", id)
+	}
+	var sameDcServers, otherDcServers []string
+	localUrls := make(map[string]bool)
+
+	for _, loc := range locations {
+		glog.V(4).Infof("lookup %s => %s, data in remote storage tier: %v", vid, loc.Url, loc.DataInRemote)
+
+		if !loc.DataInRemote {
+			localUrls[loc.Url] = true
+		}
+
+		if vc.isSameDataCenter(&loc) {
+			sameDcServers = append(sameDcServers, loc.Url)
+		} else {
+			otherDcServers = append(otherDcServers, loc.Url)
+		}
+	}
+	rand.Shuffle(len(sameDcServers), func(i, j int) {
+		sameDcServers[i], sameDcServers[j] = sameDcServers[j], sameDcServers[i]
+	})
+	rand.Shuffle(len(otherDcServers), func(i, j int) {
+		otherDcServers[i], otherDcServers[j] = otherDcServers[j], otherDcServers[i]
+	})
+	// Local replicas go first inside each data center, but never ahead of the
+	// data-center preference itself: a remote tier is often in the same region
+	// as the local replicas, so crossing a DC boundary to avoid it can cost
+	// more than the remote read it saves.
+	if len(localUrls) > 0 {
+		sameDcServers = util.ReorderToFront(localUrls, sameDcServers)
+		otherDcServers = util.ReorderToFront(localUrls, otherDcServers)
+	}
+	serverUrls = append(sameDcServers, otherDcServers...)
+	return
+}
+
+// LookupFileId resolves a "<vid>,<cookie>" file id to a list of HTTP read
+// URLs using the same DC-then-local ordering as LookupVolumeServerUrl.
+func (vc *vidMap) LookupFileId(ctx context.Context, fileId string) (fullUrls []string, err error) {
+	parts := strings.Split(fileId, ",")
+	if len(parts) != 2 {
+		return nil, errors.New("Invalid fileId " + fileId)
+	}
+	serverUrls, lookupError := vc.LookupVolumeServerUrl(parts[0])
+	if lookupError != nil {
+		return nil, lookupError
+	}
+	for _, serverUrl := range serverUrls {
+		fullUrls = append(fullUrls, "http://"+serverUrl+"/"+fileId)
+	}
+	return
+}
+
+// GetVidLocations returns the cached Location entries for vid as a string,
+// for callers that need richer per-server fields than raw URLs (e.g.
+// DataInRemote, PublicUrl).
+func (vc *vidMap) GetVidLocations(vid string) (locations []Location, err error) {
+	id, err := strconv.Atoi(vid)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return nil, fmt.Errorf("Unknown volume id %s", vid)
+	}
+	foundLocations, found := vc.GetLocations(uint32(id))
+	if found {
+		return foundLocations, nil
+	}
+	return nil, fmt.Errorf("volume id %s not found", vid)
+}
+
+// GetLocations returns the cached Location entries for vid as a uint32.
+// When both regular and EC entries are present, whichever was learned last
+// wins so a volume that switched between regular and EC encoding stops
+// answering from the stale copy. Returns found=false when nothing remains,
+// including when only an older-generation entry would otherwise apply.
+func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) {
+	vc.RLock()
+	defer vc.RUnlock()
+
+	regular, hasRegular := lookupEntry(vc.vid2Locations, vid)
+	ec, hasEc := lookupEntry(vc.ecVid2Locations, vid)
+
+	switch {
+	case hasRegular && hasEc:
+		// Whichever was learned last wins: once a volume is EC encoded, the
+		// regular copies a previous generation knew must stop answering for
+		// it, and a decoded volume must stop answering with its shards. A tie
+		// means one generation reported both, where the regular copies serve.
+		if ec.generation > regular.generation {
+			return ec.locations, true
+		}
+		return regular.locations, true
+	case hasRegular:
+		return regular.locations, true
+	case hasEc:
+		return ec.locations, true
+	}
+
+	// Nothing older to fall back to: a volume's history lives in its own entry,
+	// so a volume whose locations are all gone (a pod restarting, say) is a
+	// miss rather than a reason to serve what it used to have.
+	return nil, false
+}
+
+// lookupEntry returns vid's entry when it still holds locations. Callers must
+// hold the lock.
+func lookupEntry(vid2Locations map[uint32]*locationsEntry, vid uint32) (*locationsEntry, bool) {
+	entry, found := vid2Locations[vid]
+	if !found || len(entry.locations) == 0 {
+		return nil, false
+	}
+	return entry, true
+}
+
+func (vc *vidMap) GetLocationsClone(vid uint32) (locations []Location, found bool) {
+	locations, found = vc.GetLocations(vid)
+
+	if found {
+		// clone the locations in case the volume locations are changed below
+		existingLocations := make([]Location, len(locations))
+		copy(existingLocations, locations)
+		return existingLocations, found
+	}
+
+	return nil, false
+}
+
+// hasVolumeServer reports whether any tracked volume (regular or EC) is hosted
+// on addr, including volumes still held from earlier generations. Used to gate
+// admission of operations targeting a volume server.
+func (vc *vidMap) hasVolumeServer(addr pb.ServerAddress) bool {
+	key := addr.ToHttpAddress()
+	if key == "" {
+		return false
+	}
+	vc.RLock()
+	defer vc.RUnlock()
+	return vc.serverRefCount[key] > 0
+}
+
+func (vc *vidMap) addLocation(vid uint32, location Location) {
+	vc.Lock()
+	defer vc.Unlock()
+
+	glog.V(4).Infof("+ volume id %d: %+v", vid, location)
+
+	vc.addLocationToMap(vc.vid2Locations, vid, location)
+}
+
+func (vc *vidMap) addEcLocation(vid uint32, location Location) {
+	vc.Lock()
+	defer vc.Unlock()
+
+	glog.V(4).Infof("+ ec volume id %d: %+v", vid, location)
+
+	vc.addLocationToMap(vc.ecVid2Locations, vid, location)
+}
+
+// addLocationToMap records location for vid. The first write of a generation
+// replaces what an earlier one held instead of merging with it: after a reset
+// the new master is the authority, so a volume that moved must not keep
+// answering with the server it moved off. Callers must hold the write lock.
+//
+// If the URL is already present and the remote/local classification matches,
+// the entry is left untouched (same replica, same view). When the
+// classification flips -- e.g. a volume tiered to remote storage, or a
+// remote-backed replica restored locally -- the entry is rebuilt so
+// subsequent lookups pick up the new DataInRemote. The server reference key
+// only depends on the URL/grpc port, so it stays stable across the flip and
+// the refcount does not need to move.
+func (vc *vidMap) addLocationToMap(vid2Locations map[uint32]*locationsEntry, vid uint32, location Location) {
+	entry, found := vid2Locations[vid]
+	if !found || entry.generation != vc.generation {
+		if found {
+			vc.releaseEntry(entry)
+		}
+		vid2Locations[vid] = &locationsEntry{
+			locations:  []Location{location},
+			generation: vc.generation,
+		}
+		vc.incrementServerRef(locationServerKey(location))
+		return
+	}
+
+	for i, loc := range entry.locations {
+		if loc.Url == location.Url {
+			if loc.DataInRemote == location.DataInRemote && loc.ReadOnly == location.ReadOnly && loc.ReadOnlyCanDelete == location.ReadOnlyCanDelete {
+				return
+			}
+			// A reader holds the slice GetLocations handed it after the lock
+			// was dropped, so the replacement is copied rather than written
+			// into the array underneath it.
+			updated := make([]Location, len(entry.locations))
+			copy(updated, entry.locations)
+			updated[i] = location
+			entry.locations = updated
+			return
+		}
+	}
+
+	locations := make([]Location, 0, len(entry.locations)+1)
+	locations = append(locations, entry.locations...)
+	locations = append(locations, location)
+	vid2Locations[vid] = &locationsEntry{locations: locations, generation: entry.generation}
+	vc.incrementServerRef(locationServerKey(location))
+}
+
+func (vc *vidMap) deleteLocation(vid uint32, location Location) {
+	vc.Lock()
+	defer vc.Unlock()
+
+	glog.V(4).Infof("- volume id %d: %+v", vid, location)
+
+	vc.deleteLocationFromMap(vc.vid2Locations, vid, location)
+}
+
+func (vc *vidMap) deleteEcLocation(vid uint32, location Location) {
+	vc.Lock()
+	defer vc.Unlock()
+
+	glog.V(4).Infof("- ec volume id %d: %+v", vid, location)
+
+	vc.deleteLocationFromMap(vc.ecVid2Locations, vid, location)
+}
+
+// deleteLocationFromMap drops one location from vid's entry, and the entry
+// itself once its last location is gone. The generation is untouched: a delete
+// only speaks about the location it names, it does not make the rest of the
+// entry any fresher. Callers must hold the write lock.
+func (vc *vidMap) deleteLocationFromMap(vid2Locations map[uint32]*locationsEntry, vid uint32, location Location) {
+	entry, found := vid2Locations[vid]
+	if !found {
+		return
+	}
+
+	for i, loc := range entry.locations {
+		if loc.Url != location.Url {
+			continue
+		}
+		vc.decrementServerRef(locationServerKey(loc))
+		if len(entry.locations) == 1 {
+			delete(vid2Locations, vid)
+			return
+		}
+		remaining := make([]Location, 0, len(entry.locations)-1)
+		remaining = append(remaining, entry.locations[:i]...)
+		remaining = append(remaining, entry.locations[i+1:]...)
+		vid2Locations[vid] = &locationsEntry{locations: remaining, generation: entry.generation}
+		return
+	}
+}
+
+func (vc *vidMap) deleteVid(vid uint32) {
+	vc.Lock()
+	defer vc.Unlock()
+
+	if entry, found := vc.vid2Locations[vid]; found {
+		vc.releaseEntry(entry)
+		delete(vc.vid2Locations, vid)
+	}
+	if entry, found := vc.ecVid2Locations[vid]; found {
+		vc.releaseEntry(entry)
+		delete(vc.ecVid2Locations, vid)
+	}
+}
+
+// reset starts a new generation, as when the master changes and everything it
+// told us has to be relearned. Entries stay readable while they are relearned
+// and are dropped once they fall out of the retained window.
+func (vc *vidMap) reset() {
+	vc.Lock()
+	defer vc.Unlock()
+
+	vc.generation++
+	if vc.generation <= vc.retainGenerations {
+		return
+	}
+	oldest := vc.generation - vc.retainGenerations
+	vc.expire(vc.vid2Locations, oldest)
+	vc.expire(vc.ecVid2Locations, oldest)
+}
+
+// expire drops entries last refreshed before oldest. Callers must hold the
+// write lock.
+func (vc *vidMap) expire(vid2Locations map[uint32]*locationsEntry, oldest uint64) {
+	for vid, entry := range vid2Locations {
+		if entry.generation >= oldest {
+			continue
+		}
+		vc.releaseEntry(entry)
+		delete(vid2Locations, vid)
+	}
+}
+
+// releaseEntry drops the server references an entry holds. Callers must hold
+// the write lock.
+func (vc *vidMap) releaseEntry(entry *locationsEntry) {
+	for _, loc := range entry.locations {
+		vc.decrementServerRef(locationServerKey(loc))
+	}
+}
+
+// incrementServerRef increases the refcount for key. Empty keys are skipped
+// so a zero-value Location (which serialises to "") does not leak a permanent
+// bucket that hasVolumeServer and decrementServerRef both ignore. Callers
+// must hold vc's write lock.
+func (vc *vidMap) incrementServerRef(key string) {
+	if key == "" {
+		return
+	}
+	vc.serverRefCount[key]++
+}
+
+// decrementServerRef decreases the refcount for key and removes the entry
+// once it falls to zero. Callers must hold vc's write lock.
+func (vc *vidMap) decrementServerRef(key string) {
+	if key == "" {
+		return
+	}
+	if n, ok := vc.serverRefCount[key]; ok {
+		if n <= 1 {
+			delete(vc.serverRefCount, key)
+		} else {
+			vc.serverRefCount[key] = n - 1
+		}
+	}
+}

weed/wdclient/vid_map_generation_test.go

@@ -0,0 +1,330 @@
+package wdclient
+
+import (
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+func urlsOf(locations []Location) []string {
+	urls := make([]string, 0, len(locations))
+	for _, loc := range locations {
+		urls = append(urls, loc.Url)
+	}
+	return urls
+}
+
+// A volume that moved while we were talking to the previous master must answer
+// with where it is now, not with both servers: the first write of a generation
+// replaces the entry instead of merging into it.
+func TestAddLocationReplacesEarlierGeneration(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(3)
+	movedFrom := Location{Url: "10.0.0.1:8080"}
+	movedTo := Location{Url: "10.0.0.2:8080"}
+
+	vm.addLocation(vid, movedFrom)
+	vm.reset()
+	vm.addLocation(vid, movedTo)
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != movedTo.Url {
+		t.Fatalf("expected only %s after the move, got %v", movedTo.Url, urlsOf(locs))
+	}
+	if vm.hasVolumeServer(pb.ServerAddress(movedFrom.Url)) {
+		t.Errorf("server %s should no longer be referenced after the volume moved", movedFrom.Url)
+	}
+	if !vm.hasVolumeServer(pb.ServerAddress(movedTo.Url)) {
+		t.Errorf("server %s should be referenced after the volume moved", movedTo.Url)
+	}
+}
+
+// Replicas reported within one generation accumulate, and a repeated report of
+// the same server is not counted twice.
+func TestAddLocationMergesWithinGeneration(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(4)
+	first := Location{Url: "10.0.0.1:8080"}
+	second := Location{Url: "10.0.0.2:8080"}
+
+	vm.addLocation(vid, first)
+	vm.addLocation(vid, second)
+	vm.addLocation(vid, first)
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 2 {
+		t.Fatalf("expected both replicas, got %v", urlsOf(locs))
+	}
+
+	// One delete must be enough to evict the server: the duplicate report of
+	// `first` must not have taken a second reference.
+	vm.deleteLocation(vid, first)
+	if vm.hasVolumeServer(pb.ServerAddress(first.Url)) {
+		t.Errorf("server %s should be evicted after a single delete", first.Url)
+	}
+}
+
+// An entry survives exactly retainGenerations resets without being relearned.
+func TestResetRetainsUntilWindowExpires(t *testing.T) {
+	const retain = 2
+	vm := newVidMap("", retain)
+	vid := uint32(5)
+	location := Location{Url: "10.0.0.1:8080"}
+	vm.addLocation(vid, location)
+
+	for i := 0; i < retain; i++ {
+		vm.reset()
+		if _, found := vm.GetLocations(vid); !found {
+			t.Fatalf("location should still be retained after %d reset(s)", i+1)
+		}
+	}
+
+	vm.reset()
+	if _, found := vm.GetLocations(vid); found {
+		t.Errorf("location should be dropped after %d resets", retain+1)
+	}
+	if vm.hasVolumeServer(pb.ServerAddress(location.Url)) {
+		t.Errorf("expiry should release the server reference for %s", location.Url)
+	}
+	if len(vm.serverRefCount) != 0 {
+		t.Errorf("expiry left %d dangling server refcounts", len(vm.serverRefCount))
+	}
+}
+
+// Relearning an entry restarts its retention window.
+func TestResetWindowRestartsOnRelearn(t *testing.T) {
+	const retain = 2
+	vm := newVidMap("", retain)
+	vid := uint32(6)
+	location := Location{Url: "10.0.0.1:8080"}
+
+	for i := 0; i < retain+3; i++ {
+		vm.addLocation(vid, location)
+		vm.reset()
+	}
+
+	if _, found := vm.GetLocations(vid); !found {
+		t.Error("a location relearned every generation must never expire")
+	}
+}
+
+// A delete names one location; it must not make the rest of a stale entry look
+// freshly learned and so outlive its retention window.
+func TestDeleteLocationDoesNotRefreshGeneration(t *testing.T) {
+	const retain = 2
+	vm := newVidMap("", retain)
+	vid := uint32(7)
+	first := Location{Url: "10.0.0.1:8080"}
+	second := Location{Url: "10.0.0.2:8080"}
+
+	vm.addLocation(vid, first)
+	vm.addLocation(vid, second)
+	for i := 0; i < retain; i++ {
+		vm.reset()
+	}
+	vm.deleteLocation(vid, first)
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != second.Url {
+		t.Fatalf("expected %s to remain, got found=%v %v", second.Url, found, urlsOf(locs))
+	}
+
+	vm.reset()
+	if _, found := vm.GetLocations(vid); found {
+		t.Error("the surviving location was learned in the expired generation and should be dropped")
+	}
+}
+
+// Encoding a volume must stop the regular copies a previous generation knew
+// from answering for it, and decoding it must stop its shards from answering.
+func TestNewestGenerationWinsAcrossEcTransition(t *testing.T) {
+	regular := Location{Url: "10.0.0.1:8080"}
+	ecShard := Location{Url: "10.0.0.2:8080"}
+
+	t.Run("encoded", func(t *testing.T) {
+		vm := newVidMap("", DefaultVidMapCacheSize)
+		vm.addLocation(1, regular)
+		vm.reset()
+		vm.addEcLocation(1, ecShard)
+
+		locs, found := vm.GetLocations(1)
+		if !found || len(locs) != 1 || locs[0].Url != ecShard.Url {
+			t.Fatalf("expected the freshly learned EC shard, got found=%v %v", found, urlsOf(locs))
+		}
+	})
+
+	t.Run("decoded", func(t *testing.T) {
+		vm := newVidMap("", DefaultVidMapCacheSize)
+		vm.addEcLocation(1, ecShard)
+		vm.reset()
+		vm.addLocation(1, regular)
+
+		locs, found := vm.GetLocations(1)
+		if !found || len(locs) != 1 || locs[0].Url != regular.Url {
+			t.Fatalf("expected the freshly learned regular copy, got found=%v %v", found, urlsOf(locs))
+		}
+	})
+
+	t.Run("same generation prefers regular", func(t *testing.T) {
+		vm := newVidMap("", DefaultVidMapCacheSize)
+		vm.addEcLocation(1, ecShard)
+		vm.addLocation(1, regular)
+
+		locs, found := vm.GetLocations(1)
+		if !found || len(locs) != 1 || locs[0].Url != regular.Url {
+			t.Fatalf("expected the regular copy, got found=%v %v", found, urlsOf(locs))
+		}
+	})
+}
+
+// Losing the last location drops the entry: a client that never resets should
+// not accumulate one empty entry per volume it has ever seen deleted.
+func TestDeleteLastLocationDropsEntry(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(12)
+	location := Location{Url: "10.0.0.1:8080"}
+
+	vm.addLocation(vid, location)
+	vm.deleteLocation(vid, location)
+
+	if _, found := vm.GetLocations(vid); found {
+		t.Error("a volume with no locations left must not resolve")
+	}
+	if len(vm.vid2Locations) != 0 {
+		t.Errorf("expected the emptied entry to be dropped, got %v", vm.vid2Locations)
+	}
+	if len(vm.serverRefCount) != 0 {
+		t.Errorf("server refcounts leaked: %v", vm.serverRefCount)
+	}
+}
+
+// EC locations follow the same rules as regular ones, and back a volume whose
+// regular locations are gone.
+func TestEcLocationsFollowGenerationRules(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(8)
+	regular := Location{Url: "10.0.0.1:8080"}
+	ecShard := Location{Url: "10.0.0.2:8080"}
+	movedEcShard := Location{Url: "10.0.0.3:8080"}
+
+	vm.addLocation(vid, regular)
+	vm.addEcLocation(vid, ecShard)
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != regular.Url {
+		t.Fatalf("regular locations should win while they exist, got %v", urlsOf(locs))
+	}
+
+	// Regular copy goes away: the EC shards still serve the volume.
+	vm.deleteLocation(vid, regular)
+	locs, found = vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != ecShard.Url {
+		t.Fatalf("expected EC shard location, got found=%v %v", found, urlsOf(locs))
+	}
+
+	// EC shards move under a new master: no merging with the old report.
+	vm.reset()
+	vm.addEcLocation(vid, movedEcShard)
+	locs, found = vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != movedEcShard.Url {
+		t.Fatalf("expected only the moved EC shard, got found=%v %v", found, urlsOf(locs))
+	}
+}
+
+// Locations handed to a caller are never rewritten underneath it.
+func TestGetLocationsResultIsStable(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(9)
+	first := Location{Url: "10.0.0.1:8080"}
+	second := Location{Url: "10.0.0.2:8080"}
+
+	vm.addLocation(vid, first)
+	vm.addLocation(vid, second)
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 2 {
+		t.Fatalf("expected both replicas, got %v", urlsOf(locs))
+	}
+	snapshot := append([]Location(nil), locs...)
+
+	vm.deleteLocation(vid, first)
+	vm.addLocation(vid, Location{Url: "10.0.0.3:8080"})
+
+	for i := range snapshot {
+		if locs[i] != snapshot[i] {
+			t.Errorf("location %d changed under the caller: %v became %v", i, snapshot[i], locs[i])
+		}
+	}
+}
+
+func TestNewVidMapDefaultsRetention(t *testing.T) {
+	for _, retain := range []int{0, -1} {
+		if got := newVidMap("", retain).retainGenerations; got != DefaultVidMapCacheSize {
+			t.Errorf("newVidMap(%d) retention = %d, want %d", retain, got, DefaultVidMapCacheSize)
+		}
+	}
+}
+
+// Readers must never see a spurious miss for a volume that stays live, and the
+// bookkeeping must survive concurrent writers. Run with -race.
+func TestConcurrentResetAndUpdates(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	live := Location{Url: "10.0.0.1:8080"}
+	churn := Location{Url: "10.0.0.2:8080"}
+	const liveVid, churnVid = 1, 2
+
+	vm.addLocation(liveVid, live)
+
+	var wg sync.WaitGroup
+	stop := make(chan struct{})
+
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					if _, found := vm.GetLocations(liveVid); !found {
+						t.Error("a volume that is relearned every generation must always resolve")
+						return
+					}
+					vm.hasVolumeServer(pb.ServerAddress(live.Url))
+					vm.GetLocationsClone(churnVid)
+					runtime.Gosched()
+				}
+			}
+		}()
+	}
+
+	for i := 0; i < 4; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 200; j++ {
+				vm.addEcLocation(churnVid, churn)
+				vm.deleteEcLocation(churnVid, churn)
+				vm.deleteVid(churnVid)
+			}
+		}()
+	}
+
+	for i := 0; i < 300; i++ {
+		vm.reset()
+		vm.addLocation(liveVid, live)
+	}
+	close(stop)
+	wg.Wait()
+
+	if _, found := vm.GetLocations(liveVid); !found {
+		t.Fatal("live volume lost after the churn")
+	}
+	vm.deleteVid(liveVid)
+	if len(vm.serverRefCount) != 0 {
+		t.Errorf("server refcounts leaked: %v", vm.serverRefCount)
+	}
+}

weed/wdclient/vid_map_remote_transition_test.go

@@ -0,0 +1,122 @@
+package wdclient
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+)
+
+// When the master reports a tier transition (local ↔ remote) for an existing
+// replica on the same server URL, the cached DataInRemote must update. Otherwise
+// reads keep preferring a remote-backed replica or skip a newly-restored local
+// one. The server refcount must stay stable because the server key only depends
+// on the URL/grpc port.
+func TestAddLocationUpdatesDataInRemoteOnTransition(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(11)
+	server := Location{Url: "10.0.0.1:8080", DataCenter: "dc1", GrpcPort: 18080}
+
+	// First seen as local.
+	vm.addLocation(vid, server)
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].DataInRemote {
+		t.Fatalf("expected single local replica, got %+v", locs)
+	}
+
+	// Same URL flips to remote.
+	tiered := server
+	tiered.DataInRemote = true
+	vm.addLocation(vid, tiered)
+
+	locs, found = vm.GetLocations(vid)
+	if !found {
+		t.Fatalf("tier transition dropped the replica")
+	}
+	if len(locs) != 1 {
+		t.Fatalf("tier transition should replace in place, got %d entries: %+v", len(locs), locs)
+	}
+	if !locs[0].DataInRemote {
+		t.Errorf("DataInRemote did not flip to remote on tier-out: %+v", locs[0])
+	}
+	if locs[0].Url != server.Url || locs[0].DataCenter != server.DataCenter {
+		t.Errorf("replaced entry lost non-DataInRemote fields: %+v", locs[0])
+	}
+	if !vm.hasVolumeServer(pb.ServerAddress(server.Url)) {
+		t.Errorf("server ref should remain stable across DataInRemote flip")
+	}
+
+	// Restored locally: same URL, DataInRemote back to false.
+	restored := server
+	vm.addLocation(vid, restored)
+
+	locs, found = vm.GetLocations(vid)
+	if !found {
+		t.Fatalf("restore transition dropped the replica")
+	}
+	if len(locs) != 1 {
+		t.Fatalf("restore should replace in place, got %d entries: %+v", len(locs), locs)
+	}
+	if locs[0].DataInRemote {
+		t.Errorf("DataInRemote did not flip back to local on restore: %+v", locs[0])
+	}
+}
+
+// LookupVolumeServerUrl must reflect the latest DataInRemote so the local-first
+// ordering picks up newly-restored local replicas on the next read.
+func TestLookupVolumeServerUrlReflectsRemoteTransition(t *testing.T) {
+	vm := newVidMap("dc1", DefaultVidMapCacheSize)
+	vid := uint32(12)
+
+	remote := Location{Url: "10.0.0.1:8080", DataCenter: "dc1", DataInRemote: true}
+	vm.addLocation(vid, remote)
+
+	urls, err := vm.LookupVolumeServerUrl("12")
+	if err != nil {
+		t.Fatalf("lookup failed: %v", err)
+	}
+	if len(urls) != 1 || urls[0] != "10.0.0.1:8080" {
+		t.Fatalf("expected only the remote replica, got %v", urls)
+	}
+
+	// Tier restored: same URL, DataInRemote=false.
+	local := Location{Url: "10.0.0.1:8080", DataCenter: "dc1"}
+	vm.addLocation(vid, local)
+
+	urls, err = vm.LookupVolumeServerUrl("12")
+	if err != nil {
+		t.Fatalf("lookup after restore failed: %v", err)
+	}
+	if len(urls) != 1 || urls[0] != "10.0.0.1:8080" {
+		t.Fatalf("expected only the restored replica, got %v", urls)
+	}
+}
+
+// A tier flip must not write into the slice a concurrent lookup is still
+// walking: GetLocations hands out the entry's own slice and the caller reads
+// it after the lock is dropped.
+func TestTierFlipDoesNotRaceWithLookup(t *testing.T) {
+	vm := newVidMap("dc1", DefaultVidMapCacheSize)
+	vid := uint32(13)
+	vm.addLocation(vid, Location{Url: "10.0.0.1:8080", DataCenter: "dc1"})
+	vm.addLocation(vid, Location{Url: "10.0.0.2:8080", DataCenter: "dc1"})
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 2000; i++ {
+			vm.addLocation(vid, Location{Url: "10.0.0.1:8080", DataCenter: "dc1", DataInRemote: i%2 == 0})
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 2000; i++ {
+			if _, err := vm.LookupVolumeServerUrl("13"); err != nil {
+				t.Errorf("lookup failed mid-flip: %v", err)
+				return
+			}
+		}
+	}()
+	wg.Wait()
+}

weed/wdclient/vid_map_test.go

@@ -0,0 +1,166 @@
+package wdclient
+
+import (
+	"context"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"google.golang.org/grpc"
+)
+
+func TestLookupFileId(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", pb.ServerDiscovery{})
+	length := 5
+
+	//Construct a cache linked list of length 5
+	for i := 0; i < length; i++ {
+		mc.addLocation(uint32(i), Location{Url: strconv.FormatInt(int64(i), 10)})
+		mc.resetVidMap()
+	}
+	for i := 0; i < length; i++ {
+		locations, found := mc.GetLocations(uint32(i))
+		if !found || len(locations) != 1 || locations[0].Url != strconv.FormatInt(int64(i), 10) {
+			t.Fatalf("urls of vid=%d is not valid.", i)
+		}
+	}
+
+	//When continue to add nodes to the linked list, the previous node will be deleted, and the cache of the response will be gone.
+	for i := length; i < length+5; i++ {
+		mc.addLocation(uint32(i), Location{Url: strconv.FormatInt(int64(i), 10)})
+		mc.resetVidMap()
+	}
+	for i := 0; i < length; i++ {
+		locations, found := mc.GetLocations(uint32(i))
+		if found {
+			t.Fatalf("urls of vid[%d] should not exists, but found: %v", i, locations)
+		}
+	}
+
+	//The delete operation will be applied to all cache nodes
+	_, found := mc.GetLocations(uint32(length))
+	if !found {
+		t.Fatalf("urls of vid[%d] not found", length)
+	}
+
+	//If the locations of the current node exist, return directly
+	newUrl := "abc"
+	mc.addLocation(uint32(length), Location{Url: newUrl})
+	locations, found := mc.GetLocations(uint32(length))
+	if !found || locations[0].Url != newUrl {
+		t.Fatalf("urls of vid[%d] not found", length)
+	}
+
+	//After delete `abc`, cache nodes are searched
+	deleteLoc := Location{Url: newUrl}
+	mc.deleteLocation(uint32(length), deleteLoc)
+	locations, found = mc.GetLocations(uint32(length))
+	if found && locations[0].Url != strconv.FormatInt(int64(length), 10) {
+		t.Fatalf("urls of vid[%d] not expected", length)
+	}
+
+	//lock: concurrent test
+	var wg sync.WaitGroup
+	for i := 0; i < 20; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				for i := 0; i < 20; i++ {
+					_, _ = mc.GetLocations(uint32(i))
+				}
+			}
+		}()
+	}
+
+	for i := 0; i < 100; i++ {
+		mc.addLocation(uint32(i), Location{})
+	}
+	wg.Wait()
+}
+
+func TestConcurrentGetLocations(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", pb.ServerDiscovery{})
+	location := Location{Url: "TestDataRacing"}
+	mc.addLocation(1, location)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := sync.WaitGroup{}
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+					_, found := mc.GetLocations(1)
+					if !found {
+						cancel()
+						t.Error("vid map invalid due to data racing. ")
+						return
+					}
+				}
+			}
+		}()
+	}
+
+	//Simulate vidmap reset with cache when leader changes
+	for i := 0; i < 100; i++ {
+		mc.resetVidMap()
+		mc.addLocation(1, location)
+		time.Sleep(1 * time.Microsecond)
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestHasVolumeServer(t *testing.T) {
+	mc := NewMasterClient(grpc.EmptyDialOption{}, "", "", "", "", "", pb.ServerDiscovery{})
+
+	regular := Location{Url: "10.0.0.1:8080", GrpcPort: 18080}
+	ecOnly := Location{Url: "10.0.0.2:8080", GrpcPort: 18080}
+
+	mc.addLocation(7, regular)
+	mc.addEcLocation(9, ecOnly)
+
+	addr := func(u string) pb.ServerAddress { return pb.ServerAddress(u) }
+
+	if !mc.HasVolumeServer(addr("10.0.0.1:8080")) {
+		t.Fatalf("regular volume server must be known by http address")
+	}
+	if !mc.HasVolumeServer(addr("10.0.0.1:8080.18080")) {
+		t.Fatalf("regular volume server must be known by grpc-suffix address")
+	}
+	if !mc.HasVolumeServer(addr("10.0.0.2:8080")) {
+		t.Fatalf("ec-only volume server must be known")
+	}
+	if mc.HasVolumeServer(addr("127.0.0.1:1")) {
+		t.Fatalf("unknown address must not be known")
+	}
+
+	// Adding the same location twice must not double-count:
+	// deleting once should evict the server.
+	mc.addLocation(7, regular)
+	mc.deleteLocation(7, regular)
+	if mc.HasVolumeServer(addr("10.0.0.1:8080")) {
+		t.Fatalf("server should be evicted after deleteLocation")
+	}
+
+	// Removing the EC entry must also drop the index entry.
+	mc.deleteEcLocation(9, ecOnly)
+	if mc.HasVolumeServer(addr("10.0.0.2:8080")) {
+		t.Fatalf("server should be evicted after deleteEcLocation")
+	}
+
+	// deleteVid removes every reference held by that vid in one call.
+	mc.addLocation(11, regular)
+	mc.addEcLocation(11, regular)
+	mc.InvalidateCache("11,abc")
+	if mc.HasVolumeServer(addr("10.0.0.1:8080")) {
+		t.Fatalf("server should be evicted after InvalidateCache")
+	}
+}

weed/wdclient/vidmap_authoritative_test.go

@@ -0,0 +1,49 @@
+package wdclient
+
+import (
+	"context"
+	"reflect"
+	"testing"
+)
+
+type authoritativeLookupProvider struct {
+	calls  int
+	result map[string][]Location
+	err    error
+}
+
+func (p *authoritativeLookupProvider) LookupVolumeIds(_ context.Context, volumeIDs []string) (map[string][]Location, error) {
+	p.calls++
+	return p.result, p.err
+}
+
+func TestAuthoritativeVolumeLookupDoesNotTrustCachedLocations(t *testing.T) {
+	provider := &authoritativeLookupProvider{result: map[string][]Location{}}
+	client := newVidMapClient(provider, "", DefaultVidMapCacheSize)
+	client.addLocation(17, Location{Url: "retired-volume:8080"})
+
+	locations, err := client.LookupVolumeIdsAuthoritative(context.Background(), []string{"17"})
+	if err != nil {
+		t.Fatalf("authoritative lookup: %v", err)
+	}
+	if provider.calls != 1 {
+		t.Fatalf("provider calls = %d, want 1", provider.calls)
+	}
+	if len(locations) != 0 {
+		t.Fatalf("authoritative lookup returned stale cache: %#v", locations)
+	}
+}
+
+func TestAuthoritativeVolumeLookupReturnsProviderResult(t *testing.T) {
+	want := map[string][]Location{"17": {{Url: "current-volume:8080"}}}
+	provider := &authoritativeLookupProvider{result: want}
+	client := newVidMapClient(provider, "", DefaultVidMapCacheSize)
+
+	got, err := client.LookupVolumeIdsAuthoritative(context.Background(), []string{"17"})
+	if err != nil {
+		t.Fatalf("authoritative lookup: %v", err)
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("locations = %#v, want %#v", got, want)
+	}
+}

weed/wdclient/vidmap_client.go

@@ -0,0 +1,338 @@
+package wdclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/sync/singleflight"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// VolumeLocationProvider is the interface for looking up volume locations
+// This allows different implementations (master subscription, filer queries, etc.)
+type VolumeLocationProvider interface {
+	// LookupVolumeIds looks up volume locations for the given volume IDs
+	// Returns a map of volume ID to locations
+	LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error)
+}
+
+// vidMapClient provides volume location caching with pluggable lookup
+// It wraps the battle-tested vidMap with customizable volume lookup strategies
+type vidMapClient struct {
+	vidMap         *vidMap
+	provider       VolumeLocationProvider
+	vidLookupGroup singleflight.Group
+}
+
+const (
+	// DefaultVidMapCacheSize is the default number of resets a volume location
+	// survives without being relearned. This provides cache history when
+	// volumes move between servers.
+	DefaultVidMapCacheSize = 5
+)
+
+// newVidMapClient creates a new client with the given provider and data center
+func newVidMapClient(provider VolumeLocationProvider, dataCenter string, cacheSize int) *vidMapClient {
+	return &vidMapClient{
+		vidMap:   newVidMap(dataCenter, cacheSize),
+		provider: provider,
+	}
+}
+
+// GetLookupFileIdFunction returns a function that can be used to lookup file IDs
+func (vc *vidMapClient) GetLookupFileIdFunction() LookupFileIdFunctionType {
+	return vc.LookupFileIdWithFallback
+}
+
+// LookupFileIdWithFallback resolves a "<vid>,<cookie>" file id to a list of
+// HTTP read URLs, using the cached vidMap when populated and falling back to
+// the provider for a fresh lookup on miss. URLs are returned in preference
+// order: same-DC local, same-DC remote-tier, then the other data centers on
+// the same footing -- mirroring the cached vidMap path so both routes agree.
+// Concurrent misses for the same vid are coalesced via the singleflight group
+// on LookupVolumeIdsWithFallback.
+func (vc *vidMapClient) LookupFileIdWithFallback(ctx context.Context, fileId string) (fullUrls []string, err error) {
+	// Try cache first
+	dataCenter := vc.vidMap.DataCenter
+	fullUrls, err = vc.vidMap.LookupFileId(ctx, fileId)
+
+	// Cache hit - return immediately
+	if err == nil && len(fullUrls) > 0 {
+		return
+	}
+
+	// Cache miss - extract volume ID from file ID (format: "volumeId,needle_id_cookie")
+	if fileId == "" {
+		return nil, fmt.Errorf("empty fileId")
+	}
+	parts := strings.Split(fileId, ",")
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("invalid fileId %s", fileId)
+	}
+	volumeId := parts[0]
+
+	// Use shared lookup logic with batching and singleflight
+	vidLocations, err := vc.LookupVolumeIdsWithFallback(ctx, []string{volumeId})
+
+	// Check for partial results first (important for multi-volume batched lookups)
+	locations, found := vidLocations[volumeId]
+	if !found || len(locations) == 0 {
+		// Volume not found - return specific error with context from lookup if available
+		if err != nil {
+			return nil, fmt.Errorf("volume %s not found for fileId %s: %w", volumeId, fileId, err)
+		}
+		return nil, fmt.Errorf("volume %s not found for fileId %s", volumeId, fileId)
+	}
+
+	// Volume found successfully - ignore any errors about other volumes
+	// (not relevant for single-volume lookup, but defensive for future batching)
+
+	// Build HTTP URLs from locations, preferring same data center
+	var sameDcUrls, otherDcUrls []string
+	localUrls := make(map[string]bool)
+	for _, loc := range locations {
+		httpUrl := "http://" + loc.Url + "/" + fileId
+		glog.V(4).Infof("lookup %s => %s, data in remote storage tier: %v", fileId, loc.Url, loc.DataInRemote)
+		if !loc.DataInRemote {
+			localUrls[httpUrl] = true
+		}
+		if dataCenter != "" && dataCenter == loc.DataCenter {
+			sameDcUrls = append(sameDcUrls, httpUrl)
+		} else {
+			otherDcUrls = append(otherDcUrls, httpUrl)
+		}
+	}
+
+	// Shuffle to distribute load across volume servers
+	rand.Shuffle(len(sameDcUrls), func(i, j int) { sameDcUrls[i], sameDcUrls[j] = sameDcUrls[j], sameDcUrls[i] })
+	rand.Shuffle(len(otherDcUrls), func(i, j int) { otherDcUrls[i], otherDcUrls[j] = otherDcUrls[j], otherDcUrls[i] })
+
+	// Local replicas go first inside each data center, but never ahead of the
+	// data-center preference itself. Mirrors vidMap.LookupVolumeServerUrl so
+	// all client lookup paths agree.
+	if len(localUrls) > 0 {
+		sameDcUrls = util.ReorderToFront(localUrls, sameDcUrls)
+		otherDcUrls = util.ReorderToFront(localUrls, otherDcUrls)
+	}
+	fullUrls = append(sameDcUrls, otherDcUrls...)
+	return fullUrls, nil
+}
+
+// LookupVolumeIdsWithFallback looks up volume locations, querying provider if not in cache.
+// Uses singleflight to coalesce concurrent requests for the same batch of volumes.
+//
+// IMPORTANT: This function may return PARTIAL results with a non-nil error.
+// The result map contains successfully looked up volumes, while the error aggregates
+// failures for volumes that couldn't be found or had lookup errors.
+//
+// Callers MUST check both the result map AND the error:
+//   - result != nil && err == nil: All volumes found successfully
+//   - result != nil && err != nil: Some volumes found, some failed (check both)
+//   - result == nil && err != nil: Complete failure (connection error, etc.)
+//
+// Example usage:
+//
+//	locs, err := mc.LookupVolumeIdsWithFallback(ctx, []string{"1", "2", "999"})
+//	if len(locs) > 0 {
+//	    // Process successfully found volumes
+//	}
+//	if err != nil {
+//	    // Log/handle failed volumes
+//	}
+func (vc *vidMapClient) LookupVolumeIdsWithFallback(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
+	result := make(map[string][]Location)
+	var needsLookup []string
+	var lookupErrors []error
+
+	// Check cache first and parse volume IDs once
+	vidStringToUint := make(map[string]uint32, len(volumeIds))
+
+	for _, vidString := range volumeIds {
+		vid, err := strconv.ParseUint(vidString, 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("invalid volume id %s: %v", vidString, err)
+		}
+		vidStringToUint[vidString] = uint32(vid)
+
+		locations, found := vc.vidMap.GetLocations(uint32(vid))
+		if found && len(locations) > 0 {
+			result[vidString] = locations
+		} else {
+			needsLookup = append(needsLookup, vidString)
+		}
+	}
+
+	if len(needsLookup) == 0 {
+		return result, nil
+	}
+
+	// Batch query all missing volumes using singleflight on the batch key
+	// Sort for stable key to coalesce identical batches
+	sort.Strings(needsLookup)
+	batchKey := strings.Join(needsLookup, ",")
+
+	sfResult, err, _ := vc.vidLookupGroup.Do(batchKey, func() (interface{}, error) {
+		// Double-check cache for volumes that might have been populated while waiting
+		stillNeedLookup := make([]string, 0, len(needsLookup))
+		batchResult := make(map[string][]Location)
+
+		for _, vidString := range needsLookup {
+			vid := vidStringToUint[vidString] // Use pre-parsed value
+			if locations, found := vc.vidMap.GetLocations(vid); found && len(locations) > 0 {
+				batchResult[vidString] = locations
+			} else {
+				stillNeedLookup = append(stillNeedLookup, vidString)
+			}
+		}
+
+		if len(stillNeedLookup) == 0 {
+			return batchResult, nil
+		}
+
+		// Query provider with batched volume IDs
+		glog.V(2).Infof("Looking up %d volumes from provider: %v", len(stillNeedLookup), stillNeedLookup)
+
+		providerResults, err := vc.provider.LookupVolumeIds(ctx, stillNeedLookup)
+		if err != nil {
+			return batchResult, fmt.Errorf("provider lookup failed: %w", err)
+		}
+
+		// Update cache with results
+		for vidString, locations := range providerResults {
+			vid, err := strconv.ParseUint(vidString, 10, 32)
+			if err != nil {
+				glog.Warningf("Failed to parse volume id '%s': %v", vidString, err)
+				continue
+			}
+
+			for _, loc := range locations {
+				vc.addLocation(uint32(vid), loc)
+			}
+
+			if len(locations) > 0 {
+				batchResult[vidString] = locations
+			}
+		}
+
+		return batchResult, nil
+	})
+
+	if err != nil {
+		lookupErrors = append(lookupErrors, err)
+	}
+
+	// Merge singleflight batch results
+	if batchLocations, ok := sfResult.(map[string][]Location); ok {
+		for vid, locs := range batchLocations {
+			result[vid] = locs
+		}
+	}
+
+	// Check for volumes that still weren't found
+	for _, vidString := range needsLookup {
+		if _, found := result[vidString]; !found {
+			lookupErrors = append(lookupErrors, fmt.Errorf("volume %s not found", vidString))
+		}
+	}
+
+	// Return aggregated errors
+	return result, errors.Join(lookupErrors...)
+}
+
+// LookupVolumeIdsAuthoritative bypasses the asynchronously maintained vid map.
+// Use it when a stale positive result is unsafe, such as before serving cache
+// payload from a Volume that the master may have retired. The provider still
+// batches all requested IDs into one lookup, returns a nil map when it got no
+// answer, and reports the volumes the master does not serve as errors.
+func (vc *vidMapClient) LookupVolumeIdsAuthoritative(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
+	return vc.provider.LookupVolumeIds(ctx, volumeIds)
+}
+
+// Public methods for external access
+//
+// The vidMap itself is never replaced, so these all read the one map under its
+// own lock. Resets bump its generation instead of swapping in a fresh instance.
+
+// GetLocations safely retrieves volume locations
+func (vc *vidMapClient) GetLocations(vid uint32) (locations []Location, found bool) {
+	return vc.vidMap.GetLocations(vid)
+}
+
+// GetLocationsClone safely retrieves a clone of volume locations
+func (vc *vidMapClient) GetLocationsClone(vid uint32) (locations []Location, found bool) {
+	return vc.vidMap.GetLocationsClone(vid)
+}
+
+// GetVidLocations safely retrieves volume locations by string ID
+func (vc *vidMapClient) GetVidLocations(vid string) (locations []Location, err error) {
+	return vc.vidMap.GetVidLocations(vid)
+}
+
+// LookupFileId safely looks up URLs for a file ID
+func (vc *vidMapClient) LookupFileId(ctx context.Context, fileId string) (fullUrls []string, err error) {
+	return vc.vidMap.LookupFileId(ctx, fileId)
+}
+
+// LookupVolumeServerUrl safely looks up volume server URLs
+func (vc *vidMapClient) LookupVolumeServerUrl(vid string) (serverUrls []string, err error) {
+	return vc.vidMap.LookupVolumeServerUrl(vid)
+}
+
+// HasVolumeServer reports whether addr is currently a known volume server
+// (hosts at least one volume or EC shard) in the cached vid map. Used by
+// admission paths that must only contact peers learned from the master.
+func (vc *vidMapClient) HasVolumeServer(addr pb.ServerAddress) bool {
+	return vc.vidMap.hasVolumeServer(addr)
+}
+
+// GetDataCenter safely retrieves the data center
+func (vc *vidMapClient) GetDataCenter() string {
+	return vc.vidMap.DataCenter
+}
+
+// Thread-safe helpers for vidMap operations
+
+// addLocation adds a volume location
+func (vc *vidMapClient) addLocation(vid uint32, location Location) {
+	vc.vidMap.addLocation(vid, location)
+}
+
+// deleteLocation removes a volume location
+func (vc *vidMapClient) deleteLocation(vid uint32, location Location) {
+	vc.vidMap.deleteLocation(vid, location)
+}
+
+// addEcLocation adds an EC volume location
+func (vc *vidMapClient) addEcLocation(vid uint32, location Location) {
+	vc.vidMap.addEcLocation(vid, location)
+}
+
+// deleteEcLocation removes an EC volume location
+func (vc *vidMapClient) deleteEcLocation(vid uint32, location Location) {
+	vc.vidMap.deleteEcLocation(vid, location)
+}
+
+// resetVidMap starts a new generation, as when the master changes: what the
+// previous one told us stays readable until it is relearned or expires.
+func (vc *vidMapClient) resetVidMap() {
+	vc.vidMap.reset()
+}
+
+// InvalidateCache removes all cached locations for a volume ID
+func (vc *vidMapClient) InvalidateCache(fileId string) {
+	parts := strings.Split(fileId, ",")
+	vidString := parts[0]
+	vid, err := strconv.ParseUint(vidString, 10, 32)
+	if err != nil {
+		return
+	}
+	vc.vidMap.deleteVid(uint32(vid))
+}

weed/wdclient/vidmap_client_localfirst_test.go

@@ -0,0 +1,113 @@
+package wdclient
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+type testLocationProvider struct {
+	locations map[string][]Location
+}
+
+func (p *testLocationProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
+	result := make(map[string][]Location)
+	for _, vid := range volumeIds {
+		if locs, found := p.locations[vid]; found {
+			result[vid] = locs
+		}
+	}
+	return result, nil
+}
+
+// TestLookupFileIdWithFallbackLocalFirst ensures volumes whose data is still
+// local are tried before remote-tier replicas, on the provider (cache miss) path.
+func TestLookupFileIdWithFallbackLocalFirst(t *testing.T) {
+	vc := newVidMapClient(&testLocationProvider{
+		locations: map[string][]Location{
+			"5": {
+				{Url: "10.0.0.1:8080", DataInRemote: true},
+				{Url: "10.0.0.2:8080", DataInRemote: false},
+			},
+		},
+	}, "", 5)
+
+	urls, err := vc.LookupFileIdWithFallback(context.Background(), "5,abcdef0123456789")
+	if err != nil {
+		t.Fatalf("lookup failed: %v", err)
+	}
+	if len(urls) != 2 {
+		t.Fatalf("expected 2 urls, got %v", urls)
+	}
+	hasLocal, hasRemote := false, false
+	for _, u := range urls {
+		if strings.Contains(u, "10.0.0.2:8080") {
+			hasLocal = true
+		}
+		if strings.Contains(u, "10.0.0.1:8080") {
+			hasRemote = true
+		}
+	}
+	if !hasLocal {
+		t.Errorf("local replica missing from result: %v", urls)
+	}
+	if !hasRemote {
+		t.Errorf("remote replica missing from result: %v", urls)
+	}
+	if !strings.Contains(urls[0], "10.0.0.2:8080") {
+		t.Errorf("expected local replica first, got %v", urls)
+	}
+}
+
+// TestLookupFileIdWithFallbackAllRemote keeps shuffled order when every replica is remote.
+func TestLookupFileIdWithFallbackAllRemote(t *testing.T) {
+	vc := newVidMapClient(&testLocationProvider{
+		locations: map[string][]Location{
+			"6": {
+				{Url: "10.0.0.1:8080", DataInRemote: true},
+				{Url: "10.0.0.2:8080", DataInRemote: true},
+			},
+		},
+	}, "", 5)
+
+	urls, err := vc.LookupFileIdWithFallback(context.Background(), "6,abcdef0123456789")
+	if err != nil {
+		t.Fatalf("lookup failed: %v", err)
+	}
+	if len(urls) != 2 {
+		t.Fatalf("expected 2 urls, got %v", urls)
+	}
+}
+
+// TestLookupFileIdWithFallbackKeepsDataCenterFirst verifies that the local-first
+// ordering applies inside each data center and does not override the data-center
+// preference: a same-DC remote replica still beats an other-DC local one, because
+// the remote tier is usually nearer than another data center.
+func TestLookupFileIdWithFallbackKeepsDataCenterFirst(t *testing.T) {
+	vc := newVidMapClient(&testLocationProvider{
+		locations: map[string][]Location{
+			"7": {
+				{Url: "10.0.0.1:8080", DataCenter: "dc1", DataInRemote: true},
+				{Url: "10.0.0.2:8080", DataCenter: "dc1", DataInRemote: false},
+				{Url: "10.0.0.3:8080", DataCenter: "dc2", DataInRemote: false},
+				{Url: "10.0.0.4:8080", DataCenter: "dc2", DataInRemote: true},
+			},
+		},
+	}, "dc1", 5)
+
+	urls, err := vc.LookupFileIdWithFallback(context.Background(), "7,abcdef0123456789")
+	if err != nil {
+		t.Fatalf("lookup failed: %v", err)
+	}
+	if len(urls) != 4 {
+		t.Fatalf("expected 4 urls, got %v", urls)
+	}
+
+	// dc1 local, dc1 remote, dc2 local, dc2 remote.
+	want := []string{"10.0.0.2:8080", "10.0.0.1:8080", "10.0.0.3:8080", "10.0.0.4:8080"}
+	for i, host := range want {
+		if !strings.Contains(urls[i], host) {
+			t.Fatalf("position %d should be %s, got %v", i, host, urls)
+		}
+	}
+}

weed/wdclient/vidmap_invalidation_test.go

@@ -0,0 +1,172 @@
+package wdclient
+
+import (
+	"testing"
+)
+
+// TestInvalidateCacheValidFileId tests cache invalidation with a valid file ID
+func TestInvalidateCacheValidFileId(t *testing.T) {
+	// Create a simple vidMapClient (can use nil provider for this test)
+	vc := &vidMapClient{
+		vidMap: newVidMap("", DefaultVidMapCacheSize),
+	}
+
+	// Add some locations to the cache
+	vid := uint32(456)
+	vc.addLocation(vid, Location{Url: "http://server1:8080"})
+
+	if _, found := vc.GetLocations(vid); !found {
+		t.Fatal("Location should exist before invalidation")
+	}
+
+	// Call InvalidateCache with a properly formatted file ID
+	fileId := "456,abcdef123456"
+	vc.InvalidateCache(fileId)
+
+	// Verify the locations were removed
+	if _, found := vc.GetLocations(vid); found {
+		t.Errorf("Expected locations for vid %d to be removed after InvalidateCache", vid)
+	}
+}
+
+// TestInvalidateCacheInvalidFileId tests cache invalidation with invalid file IDs
+func TestInvalidateCacheInvalidFileId(t *testing.T) {
+	testCases := []struct {
+		name   string
+		fileId string
+	}{
+		{"empty file ID", ""},
+		{"no comma separator", "12345"},
+		{"non-numeric vid", "abc,defg"},
+		{"negative vid", "-1,abcd"},
+		{"oversized vid", "999999999999999999999,abcd"},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			vc := &vidMapClient{
+				vidMap: newVidMap("", DefaultVidMapCacheSize),
+			}
+
+			// Add a location to ensure the cache isn't empty
+			vc.addLocation(1, Location{Url: "http://server:8080"})
+
+			// This should not panic or cause errors
+			vc.InvalidateCache(tc.fileId)
+
+			// Verify the existing location is still there (not affected)
+			if _, found := vc.GetLocations(1); !found {
+				t.Errorf("InvalidateCache with invalid fileId '%s' should not affect other entries", tc.fileId)
+			}
+		})
+	}
+}
+
+// TestInvalidateCacheWithHistory tests that invalidation also drops locations
+// that were learned before the last reset and are still being retained.
+func TestInvalidateCacheWithHistory(t *testing.T) {
+	vid := uint32(789)
+
+	vc := &vidMapClient{
+		vidMap: newVidMap("", DefaultVidMapCacheSize),
+	}
+
+	// Learned from an earlier master, then kept across a reset
+	vc.addLocation(vid, Location{Url: "http://server1:8080"})
+	vc.resetVidMap()
+
+	if _, found := vc.GetLocations(vid); !found {
+		t.Fatal("Retained location should still be readable after a reset")
+	}
+
+	vc.InvalidateCache("789,xyz123")
+
+	if _, found := vc.GetLocations(vid); found {
+		t.Error("Expected retained location to be dropped by InvalidateCache")
+	}
+}
+
+// TestDeleteVidDropsRetainedGenerations tests that deleteVid removes a volume
+// no matter which generation it was last refreshed in.
+func TestDeleteVidDropsRetainedGenerations(t *testing.T) {
+	vid := uint32(999)
+
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vm.addLocation(vid, Location{Url: "http://server1:8080"})
+	vm.addEcLocation(vid, Location{Url: "http://server2:8080"})
+	vm.reset()
+	vm.reset()
+
+	if _, found := vm.GetLocations(vid); !found {
+		t.Fatal("Volume should still be readable before deletion")
+	}
+
+	vm.deleteVid(vid)
+
+	if _, found := vm.GetLocations(vid); found {
+		t.Error("Expected volume to be gone after deleteVid")
+	}
+	if vm.hasVolumeServer("http://server1:8080") || vm.hasVolumeServer("http://server2:8080") {
+		t.Error("Expected deleteVid to release the server references it held")
+	}
+}
+
+// TestGetLocationsEmptyArrayNoFallback tests that a volume known to have no
+// locations reports a miss instead of serving what an earlier generation held.
+// Covers the bug where a volume pod restarts, the vid map holds an empty array,
+// and lookups fall back to the stale locations from before the restart.
+func TestGetLocationsEmptyArrayNoFallback(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(10)
+	oldLocation := Location{Url: "10.131.1.28:8081"}
+	newLocation := Location{Url: "10.131.1.65:8081"}
+
+	// Volume initially has its old location, which is then retained across a reset
+	vm.addLocation(vid, oldLocation)
+	vm.reset()
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 || locs[0].Url != oldLocation.Url {
+		t.Fatalf("Expected to find old location, got found=%v locs=%v", found, locs)
+	}
+
+	// Volume server restarts and the old location is deleted
+	vm.deleteLocation(vid, oldLocation)
+
+	locs, found = vm.GetLocations(vid)
+	if found {
+		t.Errorf("Expected found=false for empty location array, got found=true with locs=%v", locs)
+	}
+	if locs != nil {
+		t.Errorf("Expected nil locations for empty array, got %v (should not serve stale history!)", locs)
+	}
+
+	// When the new location is added, it should be returned (not the stale one)
+	vm.addLocation(vid, newLocation)
+	locs, found = vm.GetLocations(vid)
+	if !found || len(locs) != 1 {
+		t.Fatalf("Expected to find new location, got found=%v locs=%v", found, locs)
+	}
+	if locs[0].Url != newLocation.Url {
+		t.Errorf("Expected new location %s, got %s (got stale history!)", newLocation.Url, locs[0].Url)
+	}
+}
+
+// TestGetLocationsRetainedAcrossReset tests that a volume the new master has
+// not mentioned yet is still served from what the previous one told us.
+func TestGetLocationsRetainedAcrossReset(t *testing.T) {
+	vm := newVidMap("", DefaultVidMapCacheSize)
+	vid := uint32(99)
+	retained := Location{Url: "cache-server:8081"}
+
+	vm.addLocation(vid, retained)
+	vm.reset()
+
+	locs, found := vm.GetLocations(vid)
+	if !found || len(locs) != 1 {
+		t.Fatalf("Expected to find retained location after reset, got found=%v locs=%v", found, locs)
+	}
+	if locs[0].Url != retained.Url {
+		t.Errorf("Expected retained location %s, got %s", retained.Url, locs[0].Url)
+	}
+}