snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/replication/sink/filersink/README.txt

@@ -0,0 +1,12 @@
+How replication works
+======
+
+All metadata changes within current cluster would be notified to a message queue.
+
+If the meta data change is from other clusters, this metadata would change would not be notified to the message queue.
+
+So active<=>active replication is possible.
+
+
+All metadata changes would be published as metadata changes.
+So all mounts listening for metadata changes will get updated.

weed/replication/sink/filersink/fetch_write.go

@@ -0,0 +1,709 @@
+package filersink
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/schollz/progressbar/v3"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// missingSourceChunkGrace is how long a chunk the source cannot produce keeps
+// being retried before it counts as gone for good. Long enough to outlast a
+// volume server restart or a master failover, after which every volume that
+// still exists is registered again.
+var missingSourceChunkGrace = 30 * time.Minute
+
+// errSourceChunkMissing is a permanent replication failure: the source cluster
+// no longer holds the chunk's data, so no later attempt can fetch it.
+var errSourceChunkMissing = errors.New("source chunk missing")
+
+// isSourceChunkMissing reports whether the source answered that it does not have
+// the chunk — no location for its volume, or a 404 from the volume server that
+// does. Both shapes also cover a volume that is merely offline, which is why the
+// gate below waits them out instead of trusting a single answer.
+func isSourceChunkMissing(err error) bool {
+	return errors.Is(err, source.ErrVolumeNotFound) || errors.Is(err, util_http.ErrNotFound)
+}
+
+// missingSourceChunkGate decides when to stop retrying a chunk the source cannot
+// produce, so a lookup race or a restarting volume server is waited out while a
+// vacuumed one is written off instead of retried forever. The wait is kept per
+// volume on the sink: a volume that is gone took every file it held with it, and
+// waiting it out once per file would stall the sync for as long as it holds files.
+type missingSourceChunkGate struct {
+	sink     *FilerSink
+	volumeId string
+	gaveUp   bool
+}
+
+func (g *missingSourceChunkGate) isPermanent(err error) bool {
+	if !isSourceChunkMissing(err) {
+		return false
+	}
+	if time.Since(g.sink.sourceVolumeMissingSince(g.volumeId)) < missingSourceChunkGrace {
+		return false
+	}
+	g.gaveUp = true
+	return true
+}
+
+// wrap marks err permanent once the gate gave up, so the sink can tell a source
+// that lost the data from one that is only unreachable right now.
+func (g *missingSourceChunkGate) wrap(err error) error {
+	if err == nil || !g.gaveUp {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errSourceChunkMissing, err)
+}
+
+func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_pb.FileChunk, path string, sourceMtimeNs int64) (replicatedChunks []*filer_pb.FileChunk, err error) {
+	if len(sourceChunks) == 0 {
+		return
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	// a simple progress bar. Not ideal. Fix me.
+	var bar *progressbar.ProgressBar
+	if len(sourceChunks) > 1 {
+		name := filepath.Base(path)
+		bar = progressbar.NewOptions64(int64(len(sourceChunks)),
+			progressbar.OptionClearOnFinish(),
+			progressbar.OptionOnCompletion(func() {
+				fmt.Fprint(os.Stderr, "\n")
+			}),
+			progressbar.OptionFullWidth(),
+			progressbar.OptionSetDescription(name),
+		)
+	}
+
+	replicatedChunks = make([]*filer_pb.FileChunk, len(sourceChunks))
+	var errLock sync.Mutex
+	setError := func(e error) {
+		if e == nil {
+			return
+		}
+		errLock.Lock()
+		if err == nil {
+			err = e
+		}
+		errLock.Unlock()
+	}
+	hasError := func() bool {
+		errLock.Lock()
+		defer errLock.Unlock()
+		return err != nil
+	}
+
+	var wg sync.WaitGroup
+	for chunkIndex, sourceChunk := range sourceChunks {
+		if hasError() {
+			break
+		}
+
+		if sourceChunk.IsChunkManifest {
+			replicatedChunk, replicateErr := fs.replicateOneManifestChunk(ctx, sourceChunk, path, sourceMtimeNs)
+			if replicateErr != nil {
+				setError(replicateErr)
+				break
+			}
+			replicatedChunks[chunkIndex] = replicatedChunk
+			if bar != nil {
+				bar.Add(1)
+			}
+			continue
+		}
+
+		wg.Add(1)
+		index, source := chunkIndex, sourceChunk
+		fs.executor.Execute(func() {
+			defer wg.Done()
+			var replicatedChunk *filer_pb.FileChunk
+			retryErr := util.Retry("replicate chunks", func() error {
+				chunk, e := fs.replicateOneChunk(source, path, sourceMtimeNs)
+				if e != nil {
+					return e
+				}
+				replicatedChunk = chunk
+				return nil
+			})
+			if retryErr != nil {
+				setError(retryErr)
+				return
+			}
+
+			replicatedChunks[index] = replicatedChunk
+			if bar != nil {
+				bar.Add(1)
+			}
+		})
+	}
+	wg.Wait()
+
+	return
+}
+
+func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
+
+	fileId, err := fs.fetchAndWrite(sourceChunk, path, sourceMtimeNs)
+	if err != nil {
+		return nil, fmt.Errorf("copy %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	return &filer_pb.FileChunk{
+		FileId:       fileId,
+		Offset:       sourceChunk.Offset,
+		Size:         sourceChunk.Size,
+		ModifiedTsNs: sourceChunk.ModifiedTsNs,
+		ETag:         sourceChunk.ETag,
+		SourceFileId: sourceChunk.GetFileIdString(),
+		CipherKey:    sourceChunk.CipherKey,
+		IsCompressed: sourceChunk.IsCompressed,
+		SseType:      sourceChunk.SseType,
+		SseMetadata:  sourceChunk.SseMetadata,
+	}, nil
+}
+
+func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
+	// LookupFileId can transiently fail during a write burst (volume not yet
+	// registered). Retry like the data-chunk path does; the gate stops on
+	// supersession or, when that cannot be checked, after a few attempts.
+	var resolvedChunks []*filer_pb.FileChunk
+	resolveName := fmt.Sprintf("resolve manifest %s", sourceChunk.GetFileIdString())
+	missingGate := fs.newMissingSourceChunkGate(sourceChunk.GetFileIdString())
+	err := util.RetryUntil(resolveName, func() error {
+		rc, e := filer.ResolveOneChunkManifest(ctx, fs.filerSource.LookupFileId, sourceChunk, nil)
+		if e != nil {
+			return e
+		}
+		resolvedChunks = rc
+		return nil
+	}, fs.manifestResolveRetryGate(path, sourceMtimeNs, sourceChunk.GetFileIdString(), missingGate))
+	if err != nil {
+		return nil, fmt.Errorf("resolve manifest %s: %w", sourceChunk.GetFileIdString(), missingGate.wrap(err))
+	}
+
+	replicatedResolvedChunks, err := fs.replicateChunks(ctx, resolvedChunks, path, sourceMtimeNs)
+	if err != nil {
+		return nil, fmt.Errorf("replicate manifest data chunks %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	manifestDataChunks := make([]*filer_pb.FileChunk, len(replicatedResolvedChunks))
+	for i, chunk := range replicatedResolvedChunks {
+		copied := *chunk
+		manifestDataChunks[i] = &copied
+	}
+	filer_pb.BeforeEntrySerialization(manifestDataChunks)
+	manifestData, err := proto.Marshal(&filer_pb.FileChunkManifest{
+		Chunks: manifestDataChunks,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal manifest %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	manifestFileId, err := fs.uploadManifestChunk(path, sourceMtimeNs, sourceChunk.GetFileIdString(), manifestData)
+	if err != nil {
+		return nil, err
+	}
+
+	return &filer_pb.FileChunk{
+		FileId:          manifestFileId,
+		Offset:          sourceChunk.Offset,
+		Size:            sourceChunk.Size,
+		ModifiedTsNs:    sourceChunk.ModifiedTsNs,
+		ETag:            sourceChunk.ETag,
+		SourceFileId:    sourceChunk.GetFileIdString(),
+		IsChunkManifest: true,
+	}, nil
+}
+
+// maxUnverifiableResolveAttempts bounds manifest-resolve retries when
+// supersession cannot be checked for the target path.
+const maxUnverifiableResolveAttempts = 3
+
+// manifestResolveRetryGate decides whether a failing manifest resolve keeps
+// retrying: stop when the source superseded the replayed version (the caller
+// skips it as lossless), stop once the source has been unable to produce the
+// manifest for the whole grace period, stop immediately on non-transient errors
+// (e.g. corrupt manifest data) so the configured metadata error policy applies,
+// and stop after a few attempts when supersession cannot be checked at all (incremental
+// dated target keys don't map back to a source path) — propagating lets
+// filer.backup decide with the event's real source key instead of spinning
+// here forever.
+func (fs *FilerSink) manifestResolveRetryGate(path string, sourceMtimeNs int64, chunkName string, missingGate *missingSourceChunkGate) func(error) bool {
+	_, canCheckSupersession := fs.targetPathToSourcePath(path)
+	attempts := 0
+	return func(resolveErr error) (shouldContinue bool) {
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source manifest %s for %s: %v", chunkName, path, resolveErr)
+			return false
+		}
+		if missingGate.isPermanent(resolveErr) {
+			glog.Errorf("source has not had manifest %s for %s in %v, giving up on it: %v",
+				chunkName, path, missingSourceChunkGrace, resolveErr)
+			return false
+		}
+		if !isTransientResolveError(resolveErr) {
+			glog.V(0).Infof("resolve manifest %s for %s: non-transient error, propagating: %v", chunkName, path, resolveErr)
+			return false
+		}
+		attempts++
+		if !canCheckSupersession && attempts >= maxUnverifiableResolveAttempts {
+			glog.V(0).Infof("resolve manifest %s for %s: supersession unverifiable, propagating after %d attempts: %v", chunkName, path, attempts, resolveErr)
+			return false
+		}
+		glog.V(0).Infof("resolve manifest %s for %s: %v", chunkName, path, resolveErr)
+		return true
+	}
+}
+
+// isTransientResolveError reports whether a manifest resolve failure belongs to
+// the classes that clear on their own during a write burst — volume-lookup
+// races and network interruptions. Anything else (corrupt manifest data, bad
+// file ids) is permanent and must propagate rather than retry forever.
+func isTransientResolveError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if isRetryableNetworkError(err) {
+		return true
+	}
+	errStr := err.Error()
+	return strings.Contains(errStr, "LookupFileId") ||
+		(strings.Contains(errStr, "volume id") && strings.Contains(errStr, "not found"))
+}
+
+func (fs *FilerSink) uploadManifestChunk(path string, sourceMtimeNs int64, sourceFileId string, manifestData []byte) (fileId string, err error) {
+	uploader, err := fs.getUploader()
+	if err != nil {
+		glog.V(0).Infof("upload manifest data %v: %v", sourceFileId, err)
+		return "", fmt.Errorf("upload manifest data: %w", err)
+	}
+
+	retryName := fmt.Sprintf("replicate manifest chunk %s", sourceFileId)
+	err = util.RetryUntil(retryName, func() error {
+		uploadOption := &operation.UploadOption{
+			Filename:          "",
+			Cipher:            false,
+			IsInputCompressed: false,
+			MimeType:          "application/octet-stream",
+			PairMap:           nil,
+			RetryForever:      false,
+		}
+		if fs.writeChunkByFiler {
+			uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)
+		}
+		currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(
+			fs,
+			&filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: fs.replication,
+				Collection:  fs.collection,
+				TtlSec:      fs.ttlSec,
+				DataCenter:  fs.dataCenter,
+				DiskType:    fs.diskType,
+				Path:        path,
+			},
+			uploadOption,
+			bytes.NewReader(manifestData),
+		)
+		if uploadErr != nil {
+			return fmt.Errorf("upload manifest data: %w", uploadErr)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload manifest result: %v", uploadResult.Error)
+		}
+
+		fileId = currentFileId
+		return nil
+	}, func(uploadErr error) (shouldContinue bool) {
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source manifest %s for %s: %v", sourceFileId, path, uploadErr)
+			return false
+		}
+		glog.V(0).Infof("replicate manifest %s for %s: %v", sourceFileId, path, uploadErr)
+		return true
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return fileId, nil
+}
+
+func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (fileId string, err error) {
+	uploader, err := fs.getUploader()
+	if err != nil {
+		glog.V(0).Infof("upload source data %v: %v", sourceChunk.GetFileIdString(), err)
+		return "", fmt.Errorf("upload data: %w", err)
+	}
+
+	transferStatus := &ChunkTransferStatus{
+		ChunkTransferSnapshot: ChunkTransferSnapshot{
+			ChunkFileId: sourceChunk.GetFileIdString(),
+			Path:        path,
+			Status:      "downloading",
+		},
+	}
+	fs.activeTransfers.Store(sourceChunk.GetFileIdString(), transferStatus)
+	defer fs.activeTransfers.Delete(sourceChunk.GetFileIdString())
+
+	transientBackoff := time.Duration(0)
+	_, canCheckSupersession := fs.targetPathToSourcePath(path)
+	missingGate := fs.newMissingSourceChunkGate(sourceChunk.GetFileIdString())
+	var partialData []byte
+	var savedFilename string
+	var savedHeader http.Header
+	var savedSourceUrl string
+	retryName := fmt.Sprintf("replicate chunk %s", sourceChunk.GetFileIdString())
+	err = util.RetryUntil(retryName, func() error {
+		filename, header, resp, readErr := fs.filerSource.ReadPart(sourceChunk.GetFileIdString(), int64(len(partialData)))
+		if readErr != nil {
+			return fmt.Errorf("read part %s: %w", sourceChunk.GetFileIdString(), readErr)
+		}
+		defer util_http.CloseResponse(resp)
+
+		// Save metadata from first successful response
+		if len(partialData) == 0 {
+			savedFilename = filename
+			savedHeader = header
+			if resp.Request != nil && resp.Request.URL != nil {
+				savedSourceUrl = resp.Request.URL.String()
+			}
+		}
+
+		// Read the response body
+		data, readBodyErr := io.ReadAll(resp.Body)
+		if readBodyErr != nil {
+			// Keep whatever bytes we received before the error
+			partialData = append(partialData, data...)
+			return fmt.Errorf("read body: %w", readBodyErr)
+		}
+
+		// Combine with previously accumulated partial data
+		var fullData []byte
+		if len(partialData) > 0 {
+			fullData = append(partialData, data...)
+			glog.V(0).Infof("resumed reading %s, got %d + %d = %d bytes",
+				sourceChunk.GetFileIdString(), len(partialData), len(data), len(fullData))
+			partialData = nil
+		} else {
+			fullData = data
+		}
+
+		if err := validateReplicatedReadSize(sourceChunk, len(fullData)); err != nil {
+			return err
+		}
+		fs.sourceServed(sourceChunk.GetFileIdString())
+
+		transferStatus.mu.Lock()
+		transferStatus.BytesReceived = int64(len(fullData))
+		transferStatus.Status = "uploading"
+		transferStatus.mu.Unlock()
+
+		uploadOption := &operation.UploadOption{
+			Filename:          savedFilename,
+			Cipher:            false,
+			IsInputCompressed: "gzip" == savedHeader.Get("Content-Encoding"),
+			MimeType:          savedHeader.Get("Content-Type"),
+			PairMap:           nil,
+			RetryForever:      false,
+			SourceUrl:         savedSourceUrl,
+		}
+		if fs.writeChunkByFiler {
+			uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)
+		}
+		currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(
+			fs,
+			&filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: fs.replication,
+				Collection:  fs.collection,
+				TtlSec:      fs.ttlSec,
+				DataCenter:  fs.dataCenter,
+				DiskType:    fs.diskType,
+				Path:        path,
+			},
+			uploadOption,
+			util.NewBytesReader(fullData),
+		)
+		if uploadErr != nil {
+			return fmt.Errorf("upload data: %w", uploadErr)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload result: %v", uploadResult.Error)
+		}
+
+		transientBackoff = 0
+		fileId = currentFileId
+		return nil
+	}, func(retryErr error) (shouldContinue bool) {
+		if errors.Is(retryErr, errChunkSizeMismatch) {
+			glog.V(0).Infof("permanent size mismatch replicating %s for %s: %v",
+				sourceChunk.GetFileIdString(), path, retryErr)
+			transferStatus.mu.Lock()
+			transferStatus.LastErr = retryErr.Error()
+			transferStatus.mu.Unlock()
+			return false
+		}
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source %s for %s: %v", sourceChunk.GetFileIdString(), path, retryErr)
+			return false
+		}
+		transferStatus.mu.Lock()
+		transferStatus.LastErr = retryErr.Error()
+		transferStatus.mu.Unlock()
+		if isSourceChunkMissing(retryErr) && !canCheckSupersession {
+			glog.V(0).Infof("source does not have %s for %s, supersession unverifiable, propagating: %v",
+				sourceChunk.GetFileIdString(), path, retryErr)
+			return false
+		}
+		if missingGate.isPermanent(retryErr) {
+			glog.Errorf("source has not had %s for %s in %v, giving up on it: %v",
+				sourceChunk.GetFileIdString(), path, missingSourceChunkGrace, retryErr)
+			return false
+		}
+		if isRetryableNetworkError(retryErr) {
+			transientBackoff = nextTransientBackoff(transientBackoff)
+			transferStatus.mu.Lock()
+			transferStatus.BytesReceived = int64(len(partialData))
+			transferStatus.Status = fmt.Sprintf("waiting %v", transientBackoff)
+			transferStatus.mu.Unlock()
+			glog.V(0).Infof("connection interrupted while replicating %s for %s (%d bytes received so far), backing off %v: %v",
+				sourceChunk.GetFileIdString(), path, len(partialData), transientBackoff, retryErr)
+			time.Sleep(transientBackoff)
+			transferStatus.mu.Lock()
+			transferStatus.Status = "downloading"
+			transferStatus.mu.Unlock()
+		} else {
+			glog.V(0).Infof("replicate %s for %s: %v", sourceChunk.GetFileIdString(), path, retryErr)
+		}
+		return true
+	})
+	if err != nil {
+		return "", missingGate.wrap(err)
+	}
+
+	return fileId, nil
+}
+
+const maxTransientBackoff = 2 * time.Minute
+
+// nextTransientBackoff escalates 10s -> doubling -> 2m cap so an overloaded
+// destination can recover instead of being hammered by the RetryUntil loop.
+func nextTransientBackoff(current time.Duration) time.Duration {
+	if current < 10*time.Second {
+		return 10 * time.Second
+	}
+	current *= 2
+	if current > maxTransientBackoff {
+		current = maxTransientBackoff
+	}
+	return current
+}
+
+func isEofError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF)
+}
+
+// isRetryableNetworkError reports whether err is a transient network failure worth
+// a backoff-and-retry. A bare io.EOF counts here but not in util.IsTransientError:
+// mid-transfer it means the chunk read was truncated, not that a stream ended
+// cleanly.
+func isRetryableNetworkError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return isEofError(err) || util.IsTransientError(err)
+}
+
+// errChunkSizeMismatch is a permanent (non-retriable) replication failure.
+var errChunkSizeMismatch = errors.New("chunk size mismatch")
+
+func validateReplicatedReadSize(sourceChunk *filer_pb.FileChunk, readSize int) error {
+	if uint64(readSize) != sourceChunk.Size {
+		return fmt.Errorf("%w: read %s got %d bytes, source metadata says %d",
+			errChunkSizeMismatch, sourceChunk.GetFileIdString(),
+			readSize, sourceChunk.Size)
+	}
+	return nil
+}
+
+func (fs *FilerSink) newMissingSourceChunkGate(fileId string) *missingSourceChunkGate {
+	return &missingSourceChunkGate{sink: fs, volumeId: filer.VolumeId(fileId)}
+}
+
+// sourceVolumeMissingSince returns when the sink first found volumeId
+// unlocatable, recording now on the first sighting.
+func (fs *FilerSink) sourceVolumeMissingSince(volumeId string) time.Time {
+	since, _ := fs.missingVolumes.LoadOrStore(volumeId, time.Now())
+	return since.(time.Time)
+}
+
+// sourceServed records a chunk the source did serve: the probe
+// sourceStillServesChunks re-checks, and proof its volume is locatable again.
+func (fs *FilerSink) sourceServed(fileId string) {
+	fs.lastServedFileId.Store(&fileId)
+	fs.missingVolumes.Delete(filer.VolumeId(fileId))
+}
+
+// sourceStillServesChunks reports whether the source cluster can still produce the
+// last chunk it served. A volume with no locations reads the same whether it was
+// vacuumed away or every replica is down, so before an entry is written off the
+// sink re-reads a file id the source did serve: if that one cannot be produced
+// either the source is in an outage, and skipping would drop live files wholesale.
+// It is a read and not a lookup because a lookup only proves the master still has
+// the topology, not that a volume server answers.
+func (fs *FilerSink) sourceStillServesChunks() bool {
+	if fs.filerSource == nil {
+		return false
+	}
+	probe := fs.lastServedFileId.Load()
+	if probe == nil {
+		return false
+	}
+	_, _, resp, err := fs.filerSource.ReadPart(*probe, 0)
+	if err != nil {
+		glog.V(0).Infof("source cannot serve %s either, so it is not only the one chunk: %v", *probe, err)
+		return false
+	}
+	util_http.CloseResponse(resp)
+	return true
+}
+
+// hasSourceNewerVersion reports whether the source's current entry for targetPath
+// has moved past sourceMtimeNs — gone, or a strictly-newer mtime — meaning the
+// version being replayed is stale. The lookup runs regardless of sourceMtimeNs so
+// a deleted source is detected even when the replayed mtime is epoch/unset.
+func (fs *FilerSink) hasSourceNewerVersion(targetPath string, sourceMtimeNs int64) bool {
+	sourcePath, ok := fs.targetPathToSourcePath(targetPath)
+	if !ok {
+		return false
+	}
+	return SourceSupersedes(context.Background(), fs.filerSource, sourcePath, sourceMtimeNs)
+}
+
+// SourceSupersedes reports whether the live source's entry at sourcePath has
+// moved past the replayed version (mtimeNs) — deleted or strictly newer — so
+// skipping it is lossless. Exported for filer.backup's non-filer-sink decision.
+func SourceSupersedes(ctx context.Context, filerSource *source.FilerSource, sourcePath util.FullPath, mtimeNs int64) bool {
+	if filerSource == nil {
+		return false
+	}
+	sourceEntry, _, _, err := filer_pb.GetEntry(ctx, filerSource, sourcePath)
+	return sourceSupersedes(sourcePath, sourceEntry, err, mtimeNs)
+}
+
+// sourceSupersedes reports whether the replayed version (sourceMtimeNs) is stale:
+// true if the source entry is gone or strictly newer, false on a same/older version
+// or any non-"not found" lookup error (so a transient failure never skips a live
+// file). GetEntry signals missing as ErrNotFound — possibly wrapped or a plain gRPC
+// string, occasionally a nil entry — all treated as gone.
+func sourceSupersedes(sourcePath util.FullPath, sourceEntry *filer_pb.Entry, lookupErr error, sourceMtimeNs int64) bool {
+	if lookupErr != nil {
+		if errors.Is(lookupErr, filer_pb.ErrNotFound) || strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
+			glog.V(1).Infof("source entry %s no longer exists: %v", sourcePath, lookupErr)
+			return true
+		}
+		glog.V(1).Infof("lookup source entry %s: %v", sourcePath, lookupErr)
+		return false
+	}
+	if sourceEntry == nil {
+		glog.V(1).Infof("source entry %s no longer exists", sourcePath)
+		return true
+	}
+
+	// source still holds the entry; only a strictly-newer mtime proves staleness,
+	// and only when there is a valid replayed mtime to compare against.
+	if sourceMtimeNs <= 0 {
+		return false
+	}
+	return getEntryMtimeNs(sourceEntry) > sourceMtimeNs
+}
+
+func (fs *FilerSink) targetPathToSourcePath(targetPath string) (util.FullPath, bool) {
+	if fs.filerSource == nil {
+		return "", false
+	}
+
+	// Incremental sink keys carry a date prefix (sinkDir/YYYY-MM-DD/relPath) that
+	// can't be reversed to a source path; report unmappable rather than build a path
+	// under a nonexistent dated dir, which would read as ErrNotFound and skip a live entry.
+	if fs.isIncremental {
+		return "", false
+	}
+
+	normalizePath := func(p string) string {
+		p = strings.TrimSuffix(p, "/")
+		if p == "" {
+			return "/"
+		}
+		return p
+	}
+
+	sourceRoot := normalizePath(fs.filerSource.Dir)
+	targetRoot := normalizePath(fs.dir)
+	targetPath = normalizePath(targetPath)
+
+	var relative string
+	switch {
+	case targetRoot == "/":
+		relative = strings.TrimPrefix(targetPath, "/")
+	case targetPath == targetRoot:
+		relative = ""
+	case strings.HasPrefix(targetPath, targetRoot+"/"):
+		relative = targetPath[len(targetRoot)+1:]
+	default:
+		return "", false
+	}
+
+	if relative == "" {
+		return util.FullPath(sourceRoot), true
+	}
+	return util.FullPath(sourceRoot).Child(relative), true
+}
+
+var _ = filer_pb.FilerClient(&FilerSink{})
+
+func (fs *FilerSink) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	return pb.WithGrpcClient(context.Background(), streamingMode, fs.signature, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, false, fs.grpcDialOption)
+
+}
+
+func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+func (fs *FilerSink) GetDataCenter() string {
+	return fs.dataCenter
+}

weed/replication/sink/filersink/fetch_write_test.go

@@ -0,0 +1,675 @@
+package filersink
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func TestMain(m *testing.M) {
+	util_http.InitGlobalHttpClient()
+	os.Exit(m.Run())
+}
+
+func TestTargetPathToSourcePath(t *testing.T) {
+	tests := []struct {
+		name        string
+		targetRoot  string
+		sourceRoot  string
+		targetPath  string
+		incremental bool
+		wantPath    util.FullPath
+		wantOK      bool
+	}{
+		{
+			name:       "basic mapping",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/target/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			// incremental keys carry a date prefix that can't be reversed; unmappable
+			name:        "incremental sink is unmappable",
+			targetRoot:  "/target",
+			sourceRoot:  "/source",
+			targetPath:  "/target/2026-06-09/path/file.txt",
+			incremental: true,
+			wantPath:    "",
+			wantOK:      false,
+		},
+		{
+			name:       "trailing slash roots",
+			targetRoot: "/target/",
+			sourceRoot: "/source/",
+			targetPath: "/target/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			name:       "root target mapping",
+			targetRoot: "/",
+			sourceRoot: "/source",
+			targetPath: "/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			name:       "target root itself",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/target",
+			wantPath:   "/source",
+			wantOK:     true,
+		},
+		{
+			name:       "outside target root",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/other/path/file.txt",
+			wantPath:   "",
+			wantOK:     false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fs := &FilerSink{
+				dir:           tc.targetRoot,
+				isIncremental: tc.incremental,
+				filerSource: &source.FilerSource{
+					Dir: tc.sourceRoot,
+				},
+			}
+
+			gotPath, ok := fs.targetPathToSourcePath(tc.targetPath)
+			if ok != tc.wantOK {
+				t.Fatalf("ok mismatch: got %v, want %v", ok, tc.wantOK)
+			}
+			if gotPath != tc.wantPath {
+				t.Fatalf("path mismatch: got %q, want %q", gotPath, tc.wantPath)
+			}
+		})
+	}
+}
+
+// FilerSink must reject chunks whose received byte count disagrees with the
+// source filer metadata, instead of silently writing 0-byte needles with the
+// source size in the destination metadata.
+func TestValidateReplicatedChunkSize(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+
+	tests := []struct {
+		name         string
+		expectedSize uint64
+		readSize     int
+		wantErr      bool
+	}{
+		{
+			name:         "healthy",
+			expectedSize: 5171,
+			readSize:     5171,
+			wantErr:      false,
+		},
+		{
+			name:         "legitimately empty file",
+			expectedSize: 0,
+			readSize:     0,
+			wantErr:      false,
+		},
+		{
+			name:         "zero-byte read for non-empty source",
+			expectedSize: 5171,
+			readSize:     0,
+			wantErr:      true,
+		},
+		{
+			name:         "short read",
+			expectedSize: 5171,
+			readSize:     100,
+			wantErr:      true,
+		},
+		{
+			name:         "over-read (server returned more than metadata)",
+			expectedSize: 5171,
+			readSize:     8192,
+			wantErr:      true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			chunk := &filer_pb.FileChunk{FileId: fid, Size: tc.expectedSize}
+
+			gotErr := validateReplicatedReadSize(chunk, tc.readSize)
+
+			if tc.wantErr {
+				if gotErr == nil {
+					t.Fatalf("expected error, got nil (read=%d expected=%d)",
+						tc.readSize, tc.expectedSize)
+				}
+				if !errors.Is(gotErr, errChunkSizeMismatch) {
+					t.Fatalf("expected errChunkSizeMismatch, got %v", gotErr)
+				}
+				if !strings.Contains(gotErr.Error(), fid) {
+					t.Fatalf("error %q does not mention chunk id %q", gotErr, fid)
+				}
+				return
+			}
+			if gotErr != nil {
+				t.Fatalf("unexpected read-size error: %v", gotErr)
+			}
+		})
+	}
+}
+
+// End-to-end regression :
+// a source volume that responds 200 OK with Content-Length: 0
+// for a chunk that filer metadata claims is 5171 bytes must be rejected
+// by fetchAndWrite with a (non-retriable) size mismatch error,
+// instead of being silently propagated to the destination as a 0-byte needle.
+func TestFetchAndWriteRejectsZeroByteSource(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+	const expectedSize uint64 = 5171
+
+	// Shorten retry backoff so a fail-fast test that briefly enters the retry
+	// loop doesn't pay the production 1s+ wait. Scoped to this test so any
+	// future test in the package keeps the production constant.
+	prevRetryWaitTime := util.RetryWaitTime
+	util.RetryWaitTime = 100 * time.Millisecond
+	t.Cleanup(func() { util.RetryWaitTime = prevRetryWaitTime })
+
+	var hits atomic.Int32
+	sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.WriteHeader(http.StatusOK)
+		// Intentionally write no body — mimic the buggy volume response.
+	}))
+	defer sourceServer.Close()
+
+	serverAddr := strings.TrimPrefix(sourceServer.URL, "http://")
+
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(serverAddr, serverAddr, "/", true); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		address:     serverAddr,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	sourceChunk := &filer_pb.FileChunk{
+		FileId: fid,
+		Size:   expectedSize,
+	}
+
+	done := make(chan struct {
+		fileId string
+		err    error
+	}, 1)
+	go func() {
+		gotFileId, gotErr := fs.fetchAndWrite(sourceChunk, "/dst/index.bin", 0)
+		done <- struct {
+			fileId string
+			err    error
+		}{gotFileId, gotErr}
+	}()
+
+	select {
+	case result := <-done:
+		if result.err == nil {
+			t.Fatalf("expected size mismatch error, got nil (fileId=%q)", result.fileId)
+		}
+		if !errors.Is(result.err, errChunkSizeMismatch) {
+			t.Fatalf("expected errChunkSizeMismatch, got %v", result.err)
+		}
+		if !strings.Contains(result.err.Error(), "5171") {
+			t.Fatalf("error %q does not mention expected size 5171", result.err)
+		}
+		if !strings.Contains(result.err.Error(), fid) {
+			t.Fatalf("error %q does not mention chunk id %q", result.err, fid)
+		}
+		if h := hits.Load(); h != 1 {
+			t.Fatalf("expected exactly 1 source hit (fail-fast), got %d", h)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("fetchAndWrite did not return within 5s (retry loop not aborted on size mismatch); hits=%d", hits.Load())
+	}
+}
+
+type timeoutErr struct{}
+
+func (timeoutErr) Error() string   { return "synthetic timeout" }
+func (timeoutErr) Timeout() bool   { return true }
+func (timeoutErr) Temporary() bool { return true }
+
+// A transient network failure (interrupted read, idle-deadline timeout while
+// the destination reads the upload body, reset/broken pipe) must route through
+// the escalating backoff so an overloaded destination can recover instead of
+// being hammered. The volume server returns its idle timeout as a JSON error
+// string, so the text path matters as much as the net.Error interface.
+func TestIsRetryableNetworkError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"eof", io.EOF, true},
+		{"unexpected eof", io.ErrUnexpectedEOF, true},
+		{"volume idle timeout json", fmt.Errorf("upload result: read tcp 10.0.0.1:8082->10.0.0.1:54848: i/o timeout"), true},
+		{"volume idle timeout capitalized", fmt.Errorf("Upload result: read tcp 10.0.0.1:8082->10.0.0.1:54848: I/O timeout"), true},
+		{"connection reset", fmt.Errorf("upload data: write tcp ...: connection reset by peer"), true},
+		{"connection reset capitalized", fmt.Errorf("Connection reset by peer"), true},
+		{"broken pipe", fmt.Errorf("broken pipe"), true},
+		{"broken pipe capitalized", fmt.Errorf("Broken pipe"), true},
+		{"net.Error timeout", fmt.Errorf("dial: %w", timeoutErr{}), true},
+		{"size mismatch is permanent", errChunkSizeMismatch, false},
+		{"unrelated error", errors.New("not found"), false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isRetryableNetworkError(tc.err); got != tc.want {
+				t.Fatalf("isRetryableNetworkError(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+// Lock in that the errChunkSizeMismatch sentinel survives the wrap in
+// replicateOneChunk + pass-through in util.Retry, so filer_sink.go's
+// errors.Is check actually fires.
+func TestReplicateChunksPreservesSizeMismatchSentinel(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+	const expectedSize uint64 = 5171
+
+	sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer sourceServer.Close()
+
+	serverAddr := strings.TrimPrefix(sourceServer.URL, "http://")
+
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(serverAddr, serverAddr, "/", true); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		address:     serverAddr,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	sourceChunks := []*filer_pb.FileChunk{{FileId: fid, Size: expectedSize}}
+
+	_, err := fs.replicateChunks(nil, sourceChunks, "/dst/index.bin", 0)
+	if err == nil {
+		t.Fatal("expected error from replicateChunks, got nil")
+	}
+	if !errors.Is(err, errChunkSizeMismatch) {
+		t.Fatalf("error chain broken: errors.Is(err, errChunkSizeMismatch) = false; got %v", err)
+	}
+}
+
+// sourceSupersedes decides whether to skip a stale replayed event. The replayed
+// mtime is fixed; the table varies what the source lookup returned.
+func TestSourceSupersedes(t *testing.T) {
+	const eventNs int64 = 5_000_000_500 // the version being replayed (sec 5, ns 500)
+
+	withMtime := func(sec int64, ns int32) *filer_pb.Entry {
+		return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: sec, MtimeNs: ns}}
+	}
+
+	tests := []struct {
+		name      string
+		entry     *filer_pb.Entry
+		lookupErr error
+		want      bool
+	}{
+		// deleted on source: ErrNotFound in several shapes, all read as gone -> skip
+		{"not-found sentinel", nil, filer_pb.ErrNotFound, true},
+		{"not-found wrapped", nil, fmt.Errorf("lookup /x: %w", filer_pb.ErrNotFound), true},
+		{"not-found as string (gRPC)", nil, errors.New("rpc error: " + filer_pb.ErrNotFound.Error()), true},
+		{"nil entry, nil error", nil, nil, true},
+		// transient lookup failure must NOT skip a possibly-live file
+		{"network error", nil, errors.New("dial tcp: i/o timeout"), false},
+		// live entry: compare full-ns mtime against the replayed version
+		{"source strictly newer", withMtime(5, 600), nil, true},
+		{"source same version", withMtime(5, 500), nil, false},
+		{"source older (out-of-order replay)", withMtime(5, 400), nil, false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := sourceSupersedes("/source/x/config", tc.entry, tc.lookupErr, eventNs)
+			if got != tc.want {
+				t.Fatalf("sourceSupersedes = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// An epoch/unset replayed mtime (0) must not block "gone" detection: a deleted
+// source still reports superseded so the event is skipped instead of wedging on
+// permanent retries. A live source stays not-superseded — no valid mtime to compare.
+func TestSourceSupersedesEpochMtime(t *testing.T) {
+	live := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 600}}
+	if !sourceSupersedes("/source/x", nil, filer_pb.ErrNotFound, 0) {
+		t.Fatal("epoch-mtime deleted source must be reported gone")
+	}
+	if sourceSupersedes("/source/x", live, nil, 0) {
+		t.Fatal("epoch-mtime live source must not be reported superseded")
+	}
+}
+
+// An incremental sink's dated target keys cannot be mapped back to a source
+// path, so supersession is unverifiable: the gate must stop after the bounded
+// attempts and propagate instead of spinning forever.
+func TestManifestResolveRetryGateUnverifiableSupersessionBounded(t *testing.T) {
+	fs := &FilerSink{isIncremental: true, dir: "/backup"}
+	gate := fs.manifestResolveRetryGate("/backup/2026-07-10/buckets/x/f.pt", 123, "3,01abc", fs.newMissingSourceChunkGate("3,01abc"))
+	resolveErr := errors.New("LookupFileId volume id 3: not found")
+	for i := 1; i < maxUnverifiableResolveAttempts; i++ {
+		if !gate(resolveErr) {
+			t.Fatalf("attempt %d: gate must keep retrying before the bound", i)
+		}
+	}
+	if gate(resolveErr) {
+		t.Error("gate must propagate once the bound is reached with supersession unverifiable")
+	}
+}
+
+// Non-transient resolve errors (corrupt manifest data, bad file ids) must
+// propagate immediately — before any attempt counting or supersession
+// mapping — so the configured metadata error policy applies, instead of
+// retrying until the source is superseded.
+func TestManifestResolveRetryGateNonTransientPropagates(t *testing.T) {
+	fs := &FilerSink{dir: "/backup"}
+	gate := fs.manifestResolveRetryGate("/backup/buckets/x/f.pt", 123, "3,01abc", fs.newMissingSourceChunkGate("3,01abc"))
+	permanentErrs := []error{
+		errors.New("fail to unmarshal manifest 3,01abc: proto: cannot parse invalid wire-format data"),
+		errors.New("invalid fileId abc"),
+	}
+	for _, err := range permanentErrs {
+		if gate(err) {
+			t.Errorf("non-transient error must propagate immediately: %v", err)
+		}
+	}
+	if !gate(errors.New("LookupFileId volume id 3: not found")) {
+		t.Error("transient lookup race must keep retrying on the first attempt")
+	}
+	if isTransientResolveError(nil) {
+		t.Error("isTransientResolveError(nil) must be false")
+	}
+}
+
+// sourceFilerServer answers as a source filer that still holds the entry,
+// unchanged, while its master locates the chunk's volume only for the volume ids
+// in resolvable, which it serves from volumeUrl. With none listed it is a cluster
+// that has vacuumed the volume away — or lost every replica of it.
+type sourceFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mtime      int64
+	volumeUrl  string
+	resolvable []string
+}
+
+func (s *sourceFilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
+	locationsMap := make(map[string]*filer_pb.Locations)
+	for _, vid := range req.VolumeIds {
+		if slices.Contains(s.resolvable, vid) {
+			locationsMap[vid] = &filer_pb.Locations{
+				Locations: []*filer_pb.Location{{Url: s.volumeUrl}},
+			}
+		}
+	}
+	return &filer_pb.LookupVolumeResponse{LocationsMap: locationsMap}, nil
+}
+
+func (s *sourceFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: &filer_pb.Entry{
+		Name:       req.Name,
+		Attributes: &filer_pb.FuseAttributes{Mtime: s.mtime},
+	}}, nil
+}
+
+// liveVolume serves a chunk read the way a healthy volume server would, so the
+// sink's probe finds the source still producing data.
+func liveVolume(t *testing.T) string {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("chunk bytes"))
+	}))
+	t.Cleanup(server.Close)
+	return strings.TrimPrefix(server.URL, "http://")
+}
+
+func startSourceFiler(t *testing.T, mtime int64, volumeUrl string, resolvable ...string) *source.FilerSource {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	server := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(server, &sourceFilerServer{mtime: mtime, volumeUrl: volumeUrl, resolvable: resolvable})
+	go server.Serve(listener)
+	t.Cleanup(server.Stop)
+
+	address := listener.Addr().String()
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(address, address, "/src", false); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+	filerSrc.SetGrpcDialOption(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	return filerSrc
+}
+
+// A chunk the source cluster cannot produce must stop being retried once the
+// grace period is up. Before, the retry loop ran forever: the sync job never
+// completed, so it held its slot and pinned the offset watermark at the event
+// ahead of it, and filer.sync never checkpointed again.
+func TestFetchAndWriteStopsOnMissingSourceChunk(t *testing.T) {
+	prevRetryWaitTime, prevGrace := util.RetryWaitTime, missingSourceChunkGrace
+	util.RetryWaitTime = 100 * time.Millisecond
+	missingSourceChunkGrace = 500 * time.Millisecond
+	t.Cleanup(func() {
+		util.RetryWaitTime, missingSourceChunkGrace = prevRetryWaitTime, prevGrace
+	})
+
+	filerSrc := startSourceFiler(t, 5, "")
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	done := make(chan error, 1)
+	go func() {
+		_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10}, "/dst/x.bin", 5*int64(time.Second))
+		done <- fetchErr
+	}()
+
+	select {
+	case fetchErr := <-done:
+		if !errors.Is(fetchErr, errSourceChunkMissing) {
+			t.Fatalf("expected errSourceChunkMissing, got %v", fetchErr)
+		}
+		if !errors.Is(fetchErr, source.ErrVolumeNotFound) {
+			t.Fatalf("expected the underlying lookup failure to survive, got %v", fetchErr)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("fetchAndWrite is still retrying a chunk the source cannot produce")
+	}
+}
+
+// The gate waits out the shapes a restarting volume server produces, and only
+// gives up once the source has been saying "gone" for the whole grace period.
+// The wait belongs to the volume: a vacuumed one takes every file it held with
+// it, and waiting it out per file would stall the sync for as long as it holds
+// files.
+func TestMissingSourceChunkGate(t *testing.T) {
+	volumeGone := fmt.Errorf("read part 5617,01abc: %w", source.ErrVolumeNotFound)
+	needleGone := fmt.Errorf("read part 5617,01abc: 404 Not Found: %w", util_http.ErrNotFound)
+
+	fs := &FilerSink{}
+	gate := fs.newMissingSourceChunkGate("5617,01abc")
+	for _, err := range []error{volumeGone, needleGone} {
+		if gate.isPermanent(err) {
+			t.Fatalf("must keep retrying inside the grace period: %v", err)
+		}
+	}
+	if got := gate.wrap(volumeGone); errors.Is(got, errSourceChunkMissing) {
+		t.Fatalf("must not mark permanent while still retrying: %v", got)
+	}
+
+	// an unrelated failure is not the source answering "gone": it must not start a wait
+	other := fs.newMissingSourceChunkGate("5618,01abc")
+	other.isPermanent(errors.New("connection reset by peer"))
+	if _, found := fs.missingVolumes.Load("5618"); found {
+		t.Fatal("a non-missing error must not start the volume's wait")
+	}
+
+	// a second file in the same volume inherits that wait instead of restarting it
+	fs.missingVolumes.Store("5617", time.Now().Add(-2*missingSourceChunkGrace))
+	second := fs.newMissingSourceChunkGate("5617,02def")
+	if !second.isPermanent(volumeGone) {
+		t.Fatal("a later file must inherit the wait its volume already served")
+	}
+	wrapped := second.wrap(volumeGone)
+	if !errors.Is(wrapped, errSourceChunkMissing) || !errors.Is(wrapped, source.ErrVolumeNotFound) {
+		t.Fatalf("wrapped error lost a sentinel: %v", wrapped)
+	}
+	if second.wrap(nil) != nil {
+		t.Fatal("a successful retry must not be turned into an error")
+	}
+
+	// the volume answering again clears the wait it had served
+	fs.sourceServed("5617,09fff")
+	if _, found := fs.missingVolumes.Load("5617"); found {
+		t.Fatal("a served chunk must clear its volume's wait")
+	}
+}
+
+// Once the source has lost a chunk for good, holding the sync offset for its
+// entry only stops every later event from ever being checkpointed: the bytes are
+// not coming back. Skip it loudly instead — but only while the source is
+// demonstrably still serving other chunks, since a cluster that cannot locate
+// anything is having an outage and skipping would drop live files wholesale.
+func TestOnReplicateChunkErrorMissingSourceChunk(t *testing.T) {
+	const probe = "5616,01abc"
+	liveEntry := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5}}
+	missing := fmt.Errorf("copy 5617,02def: %w: %w", errSourceChunkMissing, source.ErrVolumeNotFound)
+
+	t.Run("source still serving", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, liveVolume(t), "5616"), dir: "/dst"}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); err != nil {
+			t.Fatalf("expected the entry to be skipped, got %v", err)
+		}
+	})
+
+	t.Run("source locating nothing", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, ""), dir: "/dst"}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("expected the error to be held for a retry, got %v", err)
+		}
+	})
+
+	t.Run("probe locates but cannot be read", func(t *testing.T) {
+		gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "Not Found", http.StatusNotFound)
+		}))
+		defer gone.Close()
+		fs := &FilerSink{
+			filerSource: startSourceFiler(t, 5, strings.TrimPrefix(gone.URL, "http://"), "5616"),
+			dir:         "/dst",
+		}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("a volume the master lists but no server answers must not license a skip, got %v", err)
+		}
+	})
+
+	t.Run("nothing served yet", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, liveVolume(t), "5616"), dir: "/dst"}
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("expected the error to be held with no probe to check, got %v", err)
+		}
+	})
+}
+
+// An incremental sink's dated target keys cannot be mapped back to a source path,
+// so nothing here can tell a vacuumed chunk from a superseded one. Waiting out the
+// grace period would stall every such entry for half an hour; propagate instead
+// and let filer.backup decide with the event's real source key.
+func TestFetchAndWriteMissingSourceChunkUnverifiableSupersession(t *testing.T) {
+	prevRetryWaitTime := util.RetryWaitTime
+	util.RetryWaitTime = 100 * time.Millisecond
+	t.Cleanup(func() { util.RetryWaitTime = prevRetryWaitTime })
+
+	fs := &FilerSink{
+		filerSource:   startSourceFiler(t, 5, ""),
+		dir:           "/backup",
+		isIncremental: true,
+		executor:      util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	done := make(chan error, 1)
+	go func() {
+		_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10},
+			"/backup/2026-07-10/x.bin", 5*int64(time.Second))
+		done <- fetchErr
+	}()
+
+	select {
+	case fetchErr := <-done:
+		if !errors.Is(fetchErr, source.ErrVolumeNotFound) {
+			t.Fatalf("expected the lookup failure to propagate, got %v", fetchErr)
+		}
+		if errors.Is(fetchErr, errSourceChunkMissing) {
+			t.Fatalf("must not write the chunk off without checking supersession: %v", fetchErr)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("fetchAndWrite waited out the grace period with supersession unverifiable")
+	}
+}

weed/replication/sink/filersink/filer_sink.go

@@ -0,0 +1,668 @@
+package filersink
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ChunkTransferSnapshot is a lock-free, copyable view of a chunk transfer's
+// progress, returned by ActiveTransfers.
+type ChunkTransferSnapshot struct {
+	ChunkFileId   string
+	Path          string
+	BytesReceived int64
+	Status        string // "downloading", "uploading", or "waiting 10s" etc.
+	LastErr       string
+}
+
+// ChunkTransferStatus tracks the progress of a single chunk being replicated.
+// Fields are guarded by mu: ChunkFileId and Path are immutable after creation,
+// while BytesReceived, Status, and LastErr are updated by fetchAndWrite and
+// read by ActiveTransfers.
+type ChunkTransferStatus struct {
+	mu sync.RWMutex
+	ChunkTransferSnapshot
+}
+
+type FilerSink struct {
+	filerSource       *source.FilerSource
+	grpcAddress       string
+	dir               string
+	replication       string
+	collection        string
+	ttlSec            int32
+	diskType          string
+	dataCenter        string
+	grpcDialOption    grpc.DialOption
+	address           string
+	writeChunkByFiler bool
+	isIncremental     bool
+	executor          *util.LimitedConcurrentExecutor
+	signature         int32
+	activeTransfers   sync.Map // chunkFileId -> *ChunkTransferStatus
+	uploader          *operation.Uploader
+	// lastServedFileId is the most recent chunk the source did serve, the probe
+	// sourceStillServesChunks re-checks before writing an entry off.
+	lastServedFileId atomic.Pointer[string]
+	missingVolumes   sync.Map // source volume id -> time.Time first found unlocatable
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &FilerSink{})
+}
+
+func (fs *FilerSink) GetName() string {
+	return "filer"
+}
+
+func (fs *FilerSink) GetSinkToDirectory() string {
+	return fs.dir
+}
+
+func (fs *FilerSink) GetDestinationIdentity() string {
+	return fs.grpcAddress + "\x00" + fs.dir
+}
+
+func (fs *FilerSink) IsIncremental() bool {
+	return fs.isIncremental
+}
+
+func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error {
+	fs.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	fs.dataCenter = configuration.GetString(prefix + "dataCenter")
+	fs.signature = util.RandomInt32()
+	return fs.DoInitialize(
+		"",
+		configuration.GetString(prefix+"grpcAddress"),
+		configuration.GetString(prefix+"directory"),
+		configuration.GetString(prefix+"replication"),
+		configuration.GetString(prefix+"collection"),
+		configuration.GetInt(prefix+"ttlSec"),
+		configuration.GetString(prefix+"disk"),
+		security.LoadClientTLS(util.GetViper(), "grpc.client"),
+		false)
+}
+
+func (fs *FilerSink) SetSourceFiler(s *source.FilerSource) {
+	fs.filerSource = s
+}
+
+// SetUploader sets a custom uploader for this sink, used when the target
+// cluster requires different TLS certificates than the global config.
+// Must be called during initialization, before any replication goroutines
+// start, since it writes fs.uploader without synchronization.
+func (fs *FilerSink) SetUploader(uploader *operation.Uploader) {
+	fs.uploader = uploader
+}
+
+func (fs *FilerSink) getUploader() (*operation.Uploader, error) {
+	if fs.uploader != nil {
+		return fs.uploader, nil
+	}
+	return operation.NewUploader()
+}
+
+func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
+	replication string, collection string, ttlSec int, diskType string, grpcDialOption grpc.DialOption, writeChunkByFiler bool) (err error) {
+	fs.address = address
+	if fs.address == "" {
+		fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
+	}
+	fs.grpcAddress = grpcAddress
+	fs.dir = dir
+	fs.replication = replication
+	fs.collection = collection
+	fs.ttlSec = int32(ttlSec)
+	fs.diskType = diskType
+	fs.grpcDialOption = grpcDialOption
+	fs.writeChunkByFiler = writeChunkByFiler
+	fs.executor = util.NewLimitedConcurrentExecutor(32)
+	return nil
+}
+
+// SetChunkConcurrency replaces the chunk replication executor with one using the
+// given concurrency limit. Must be called during initialization, before any
+// replication goroutines start, since it replaces fs.executor without
+// synchronization.
+func (fs *FilerSink) SetChunkConcurrency(concurrency int) {
+	if concurrency > 0 {
+		fs.executor = util.NewLimitedConcurrentExecutor(concurrency)
+	}
+}
+
+// ActiveTransfers returns an immutable snapshot of all in-progress chunk transfers.
+func (fs *FilerSink) ActiveTransfers() []ChunkTransferSnapshot {
+	var transfers []ChunkTransferSnapshot
+	fs.activeTransfers.Range(func(key, value any) bool {
+		t := value.(*ChunkTransferStatus)
+		t.mu.RLock()
+		transfers = append(transfers, t.ChunkTransferSnapshot)
+		t.mu.RUnlock()
+		return true
+	})
+	return transfers
+}
+
+func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	dir, name := util.FullPath(key).DirAndName()
+
+	glog.V(4).Infof("delete entry: %v", key)
+	err := filer_pb.Remove(context.Background(), fs, dir, name, deleteIncludeChunks, true, true, true, signatures)
+	if err != nil {
+		glog.V(0).Infof("delete entry %s: %v", key, err)
+		return fmt.Errorf("delete entry %s: %v", key, err)
+	}
+	return nil
+}
+
+var _ sink.EntryMover = (*FilerSink)(nil)
+
+// MoveEntry relocates oldKey to newKey on the target filer via AtomicRenameEntry:
+// a metadata-only move that relocates a whole subtree in one transaction, so a
+// directory rename never leaves descendants missing and chunks are neither
+// re-copied nor leaked.
+//
+// When the move fails because the old path is genuinely gone on the sink — a
+// descendant the parent rename already relocated, or one never replicated —
+// there is nothing to move, so it creates the new path instead (CreateEntry
+// short-circuits when the entry is already there, and never deletes). Existence
+// is re-checked with a direct lookup rather than inferred from the rename error,
+// so a rolled-back move that left the old entry intact propagates for retry
+// instead of being mistaken for "gone".
+func (fs *FilerSink) MoveEntry(oldKey, newKey string, newEntry *filer_pb.Entry, signatures []int32) error {
+	oldDir, oldName := util.FullPath(oldKey).DirAndName()
+	newDir, newName := util.FullPath(newKey).DirAndName()
+
+	err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: oldDir,
+			OldName:      oldName,
+			NewDirectory: newDir,
+			NewName:      newName,
+			Signatures:   signatures,
+		})
+		return err
+	})
+	if err == nil {
+		return nil
+	}
+	if missing, lookupErr := fs.entryMissing(oldKey); lookupErr == nil && missing {
+		glog.V(2).Infof("move %s => %s: old path gone, creating %s", oldKey, newKey, newKey)
+		return fs.CreateEntry(newKey, newEntry, signatures)
+	}
+	return fmt.Errorf("move %s => %s: %w", oldKey, newKey, err)
+}
+
+// entryMissing reports whether key has no entry on the target filer. A lookup
+// not-found (sentinel or the gRPC string form) means missing; any other lookup
+// error is returned so the caller does not treat an unknown state as missing.
+func (fs *FilerSink) entryMissing(key string) (bool, error) {
+	dir, name := util.FullPath(key).DirAndName()
+	missing := false
+	err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		})
+		if lookupErr == nil {
+			return nil
+		}
+		// The string check is a deliberate compatibility fallback: a cross-cluster
+		// gRPC error often arrives as a plain status string that no longer wraps the
+		// filer_pb.ErrNotFound sentinel, so errors.Is alone would miss it.
+		if errors.Is(lookupErr, filer_pb.ErrNotFound) || strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
+			missing = true
+			return nil
+		}
+		return lookupErr
+	})
+	return missing, err
+}
+
+func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+
+	return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		dir, name := util.FullPath(key).DirAndName()
+
+		// look up existing entry
+		lookupRequest := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+		// glog.V(1).Infof("lookup: %v", lookupRequest)
+		if resp, err := filer_pb.LookupEntry(context.Background(), client, lookupRequest); err == nil {
+			if destinationSatisfiesSource(resp.Entry, entry) {
+				glog.V(3).Infof("skip overwriting %s", key)
+				return nil
+			}
+			glog.Warningf("re-replicating %s: destination not current (dst %d chunks, src %d chunks)",
+				key, len(resp.Entry.GetChunks()), len(entry.GetChunks()))
+		}
+
+		replicatedChunks, err := fs.replicateChunks(context.Background(), entry.GetChunks(), key, getEntryMtimeNs(entry))
+
+		if err != nil {
+			// don't swallow: committing mismatched bytes would propagate corruption
+			if errors.Is(err, errChunkSizeMismatch) {
+				return fs.onCorruptChunk(key, entry, err)
+			}
+			return fs.onReplicateChunkError(key, entry, err)
+		}
+
+		// glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks)
+
+		request := &filer_pb.CreateEntryRequest{
+			Directory: dir,
+			Entry: &filer_pb.Entry{
+				Name:        name,
+				IsDirectory: entry.IsDirectory,
+				Attributes:  entry.Attributes,
+				Extended:    entry.Extended,
+				Chunks:      replicatedChunks,
+				Content:     entry.Content,
+				RemoteEntry: entry.RemoteEntry,
+			},
+			IsFromOtherCluster: true,
+			Signatures:         signatures,
+		}
+
+		glog.V(3).Infof("create: %v", request)
+		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+			glog.V(0).Infof("create entry %s: %v", key, err)
+			return fmt.Errorf("create entry %s: %v", key, err)
+		}
+
+		return nil
+	})
+}
+
+func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+
+	dir, name := util.FullPath(key).DirAndName()
+
+	// read existing entry
+	var existingEntry *filer_pb.Entry
+	err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+
+		glog.V(4).Infof("lookup entry: %v", request)
+		resp, err := filer_pb.LookupEntry(context.Background(), client, request)
+		if err != nil {
+			glog.V(0).Infof("lookup %s: %v", key, err)
+			return err
+		}
+
+		existingEntry = resp.Entry
+
+		return nil
+	})
+
+	if err != nil {
+		return false, fmt.Errorf("lookup %s: %v", key, err)
+	}
+
+	glog.V(4).Infof("oldEntry %+v, newEntry %+v, existingEntry: %+v", oldEntry, newEntry, existingEntry)
+
+	// the supersession checks below must look up where the incoming entry lives
+	// now, which for a rename is newParentPath/newEntry.Name, not the old key.
+	targetKey := updatedEntryKey(key, newParentPath, newEntry)
+
+	switch chooseUpdateAction(existingEntry, oldEntry, newEntry) {
+	case updateSkip:
+		// a newer, complete version already landed (out-of-order); leave it
+		glog.V(2).Infof("late updates %s", key)
+		return true, nil
+	case updateRepair:
+		glog.Warningf("repair %s: destination diverged from source (dst %d chunks, src %d chunks); re-replicating full source content",
+			key, len(existingEntry.GetChunks()), len(newEntry.GetChunks()))
+		replicatedChunks, err := fs.replicateChunks(context.Background(), newEntry.GetChunks(), targetKey, getEntryMtimeNs(newEntry))
+		if err != nil {
+			if errors.Is(err, errChunkSizeMismatch) {
+				return true, fs.onCorruptChunk(targetKey, newEntry, err)
+			}
+			return true, fs.onReplicateChunkError(targetKey, newEntry, err)
+		}
+		existingEntry.Chunks = replicatedChunks
+		existingEntry.Attributes = newEntry.Attributes
+		existingEntry.Extended = newEntry.Extended
+		existingEntry.HardLinkId = newEntry.HardLinkId
+		existingEntry.HardLinkCounter = newEntry.HardLinkCounter
+		existingEntry.Content = newEntry.Content
+		existingEntry.RemoteEntry = newEntry.RemoteEntry
+	default:
+		// source-side chunks resolve via source filer; sink volume IDs may collide.
+		deletedChunks, newChunks, err := compareChunks(context.Background(), filer.LookupFn(fs.filerSource), oldEntry, newEntry)
+		if err != nil {
+			return true, fmt.Errorf("replicate %s compare chunks error: %v", key, err)
+		}
+
+		// delete the chunks that are deleted from the source
+		if deleteIncludeChunks {
+			// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
+			existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.GetChunks(), deletedChunks)
+		}
+
+		// replicate the chunks that are new in the source
+		replicatedChunks, err := fs.replicateChunks(context.Background(), newChunks, targetKey, getEntryMtimeNs(newEntry))
+		if err != nil {
+			if errors.Is(err, errChunkSizeMismatch) {
+				return true, fs.onCorruptChunk(targetKey, newEntry, err)
+			}
+			return true, fs.onReplicateChunkError(targetKey, newEntry, err)
+		}
+		existingEntry.Chunks = append(existingEntry.GetChunks(), replicatedChunks...)
+		existingEntry.Attributes = newEntry.Attributes
+		existingEntry.Extended = newEntry.Extended
+		existingEntry.HardLinkId = newEntry.HardLinkId
+		existingEntry.HardLinkCounter = newEntry.HardLinkCounter
+		existingEntry.Content = newEntry.Content
+		existingEntry.RemoteEntry = newEntry.RemoteEntry
+	}
+
+	// save updated meta data
+	return true, fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.UpdateEntryRequest{
+			Directory:          newParentPath,
+			Entry:              existingEntry,
+			IsFromOtherCluster: true,
+			Signatures:         signatures,
+		}
+
+		if _, err := client.UpdateEntry(context.Background(), request); err != nil {
+			return fmt.Errorf("update existingEntry %s: %v", key, err)
+		}
+
+		return nil
+	})
+
+}
+func compareChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) {
+	aData, aMeta, aErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, oldEntry.GetChunks(), 0, math.MaxInt64, nil)
+	if aErr != nil {
+		return nil, nil, aErr
+	}
+	bData, bMeta, bErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, newEntry.GetChunks(), 0, math.MaxInt64, nil)
+	if bErr != nil {
+		return nil, nil, bErr
+	}
+
+	deletedChunks = append(deletedChunks, filer.DoMinusChunks(aData, bData)...)
+	deletedChunks = append(deletedChunks, filer.DoMinusChunks(aMeta, bMeta)...)
+
+	newChunks = append(newChunks, filer.DoMinusChunks(bData, aData)...)
+	newChunks = append(newChunks, filer.DoMinusChunks(bMeta, aMeta)...)
+
+	return
+}
+
+// getEntryMtimeNs returns the mtime at full nanosecond precision so versions
+// rewritten within the same second can be ordered. int64 ns is safe until ~2262.
+func getEntryMtimeNs(entry *filer_pb.Entry) int64 {
+	if entry == nil || entry.Attributes == nil {
+		return 0
+	}
+	return entry.Attributes.Mtime*int64(1e9) + int64(entry.Attributes.MtimeNs)
+}
+
+// EntryMtimeNs exposes getEntryMtimeNs for filer.backup's supersession check.
+func EntryMtimeNs(entry *filer_pb.Entry) int64 {
+	return getEntryMtimeNs(entry)
+}
+
+// onCorruptChunk handles a permanent chunk size-mismatch (fetched source bytes
+// disagree with source metadata). If the source already holds a newer version the
+// event is a stale replay whose chunk was overwritten and GC'd — skip it (lossless:
+// meta events are full snapshots, so a later event re-carries the current chunks).
+// Otherwise the live version is corrupt, so return the error and halt loudly.
+func (fs *FilerSink) onCorruptChunk(key string, entry *filer_pb.Entry, err error) error {
+	if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
+		glog.Warningf("skip stale entry %s with superseded corrupt chunk: %v", key, err)
+		return nil
+	}
+	glog.Errorf("refuse to replicate entry with corrupt chunk %s: %v", key, err)
+	return err
+}
+
+// onReplicateChunkError handles a non-size-mismatch replicateChunks failure.
+// Skip (return nil) when the live source has moved past this version — deleted
+// or strictly-newer mtime — so a later event carries the current content and the
+// skip is lossless, or when the source has lost the chunk for good: it can never
+// serve those bytes to anyone, and holding the offset for it only stops every
+// later event from ever being checkpointed. Otherwise propagate so the offset
+// stays put and the event is retried; returning nil would silently drop the file.
+func (fs *FilerSink) onReplicateChunkError(key string, entry *filer_pb.Entry, err error) error {
+	if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
+		glog.Warningf("skip stale entry %s, source superseded during replicate: %v", key, err)
+		return nil
+	}
+	if errors.Is(err, errSourceChunkMissing) && fs.sourceStillServesChunks() {
+		glog.Errorf("skip %s: the source cluster cannot produce its data, so it stays unreplicated: %v", key, err)
+		return nil
+	}
+	return fmt.Errorf("replicate entry chunks %s: %w", key, err)
+}
+
+// updateAction decides how UpdateEntry reconciles an incoming source version with
+// the destination's current entry.
+type updateAction int
+
+const (
+	// updateNormal applies the source diff: destination at or behind the source.
+	updateNormal updateAction = iota
+	// updateSkip leaves the destination untouched: a newer complete version already landed.
+	updateSkip
+	// updateRepair re-replicates full source over a destination strictly shorter than
+	// the source despite a newer mtime — a truncated copy from a failed replication.
+	updateRepair
+)
+
+// updatedEntryKey returns the sink path the incoming entry lands at: for a rename
+// newParentPath/newEntry.Name, else key's parent + name. Supersession must use this,
+// not the pre-rename key, which would look deleted on the source and skip a live event.
+func updatedEntryKey(key, newParentPath string, newEntry *filer_pb.Entry) string {
+	targetDir := newParentPath
+	if targetDir == "" {
+		targetDir, _ = util.FullPath(key).DirAndName()
+	}
+	return string(util.NewFullPath(targetDir, newEntry.Name))
+}
+
+func chooseUpdateAction(existing, oldEntry, incoming *filer_pb.Entry) updateAction {
+	exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
+	if exNs > inNs {
+		// Destination is strictly newer: incoming is a stale/out-of-order replay of
+		// an older source version, so never roll the destination back to it — even
+		// when the newer destination is smaller (a legitimate truncate). A newer copy
+		// that is itself incomplete is reconciled by -initialSnapshot, not by an
+		// older event.
+		return updateSkip
+	}
+	// Destination at or behind incoming. updateNormal applies the incremental
+	// oldEntry->incoming diff, valid only when the destination already holds
+	// oldEntry's chunks. A cheap, allocation-free positional identity match proves
+	// that for the common (in-sync) path; only when it fails — divergence, chunk
+	// reordering, or an out-of-band destination — do we run the precise O(n log n)
+	// checks.
+	if existing != nil && oldEntry != nil && !destinationMatchesReference(existing, oldEntry) {
+		// Truncated/gapped: the destination does not cover every byte range oldEntry
+		// holds. Range containment (not a scalar byte count, attr FileSize, or ETag)
+		// catches it for any destination, and is safe at equal mtime since a complete
+		// destination always covers oldEntry.
+		if !coversReference(existing, oldEntry) {
+			return updateRepair
+		}
+		// Stale superseded chunk at full coverage: a chunk this backup wrote whose
+		// source oldEntry no longer references. Restricted to backup-written chunks
+		// (out-of-band rsync/direct chunks are ignored) AND to a strictly-older
+		// destination — at equal mtime same-second versions cannot be ordered, so
+		// defer rather than risk a rollback (reliable ordering is a separate fix).
+		if exNs < inNs && hasStaleBackupChunk(existing, oldEntry) {
+			return updateRepair
+		}
+	}
+	return updateNormal
+}
+
+// destinationSatisfiesSource reports whether an existing destination entry already
+// satisfies the incoming source entry, so CreateEntry can skip the write. Coverage
+// is checked first: a chunk-truncated destination (covering fewer bytes than the
+// source) must be re-replicated even when its attr.Md5-backed ETag still matches
+// the source, so ETag equality must never bypass the coverage check. A destination
+// that is at least as complete is skippable when it has identical content (ETag) or
+// is a newer-or-equal version.
+func destinationSatisfiesSource(existing, incoming *filer_pb.Entry) bool {
+	if existing == nil {
+		return false
+	}
+	exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
+	if exNs > inNs {
+		return true // destination strictly newer: incoming is older → keep destination
+	}
+	if !destinationMatchesReference(existing, incoming) {
+		if !coversReference(existing, incoming) {
+			return false // missing a byte range the source covers (truncated/gapped)
+		}
+		if exNs < inNs && hasStaleBackupChunk(existing, incoming) {
+			return false // a provably-older destination holds a superseded backup chunk
+		}
+	}
+	if filer.ETag(existing) == filer.ETag(incoming) {
+		return true // identical content
+	}
+	return exNs >= inNs // equal mtime → keep (deferred); strictly older → re-replicate
+}
+
+// destinationMatchesReference is a cheap, allocation-free sufficient check that the
+// destination already holds exactly the reference's chunks: equal count and, in
+// order, each destination chunk was replicated from the corresponding reference
+// chunk (SourceFileId == reference FileId) at the same byte range (Offset/Size).
+// True ⇒ full coverage and no stale chunk, so updateNormal/skip is safe and the
+// O(n log n) range and identity checks can be skipped — the common in-sync path.
+// False is never a false positive: a reordered, diverged, or out-of-band
+// (rsync/direct) destination just falls back to the precise checks.
+func destinationMatchesReference(existing, reference *filer_pb.Entry) bool {
+	ex, ref := existing.GetChunks(), reference.GetChunks()
+	if len(ex) != len(ref) {
+		return false
+	}
+	for i := range ref {
+		if ex[i].SourceFileId == "" || ex[i].SourceFileId != ref[i].GetFileIdString() {
+			return false
+		}
+		// Require the same byte range too. replicateOneChunk copies the source
+		// chunk's Offset/Size verbatim, so SourceFileId identity already implies an
+		// identical range today; verifying it here keeps this fast path sound even
+		// if replication ever re-chunks (split/coalesce), instead of silently
+		// depending on that invariant from another file.
+		if ex[i].Offset != ref[i].Offset || ex[i].Size != ref[i].Size {
+			return false
+		}
+	}
+	return true
+}
+
+// hasStaleBackupChunk reports whether existing holds a chunk that this backup
+// wrote (SourceFileId set) whose source chunk `reference` no longer references.
+// That means an overwrite/deletion event was missed: the destination still carries
+// the superseded source chunk's bytes even though total coverage looks full, so the
+// incremental diff cannot fix it. Chunks without SourceFileId (seeded out-of-band,
+// e.g. rsync or a direct write) are ignored, so such destinations keep their normal
+// path and are never re-copied on this signal.
+func hasStaleBackupChunk(existing, reference *filer_pb.Entry) bool {
+	if existing == nil || reference == nil {
+		return false
+	}
+	refFids := make(map[string]bool, len(reference.GetChunks()))
+	for _, c := range reference.GetChunks() {
+		refFids[c.GetFileIdString()] = true
+	}
+	for _, c := range existing.GetChunks() {
+		if c.SourceFileId == "" {
+			continue // not backup-written → do not judge staleness
+		}
+		if !refFids[c.SourceFileId] {
+			return true
+		}
+	}
+	return false
+}
+
+// mergedIntervals returns the entry's chunk byte ranges, sorted and merged
+// (overlapping or touching ranges combined). Chunk manifests are counted by their
+// own offset/size span, which already covers the range they wrap, so no manifest
+// resolution is needed.
+func mergedIntervals(entry *filer_pb.Entry) [][2]int64 {
+	if entry == nil {
+		return nil
+	}
+	chunks := entry.GetChunks()
+	if len(chunks) == 0 {
+		return nil
+	}
+	ivs := make([][2]int64, 0, len(chunks))
+	for _, c := range chunks {
+		ivs = append(ivs, [2]int64{int64(c.Offset), int64(c.Offset) + int64(c.Size)})
+	}
+	sort.Slice(ivs, func(i, j int) bool { return ivs[i][0] < ivs[j][0] })
+	merged := make([][2]int64, 0, len(ivs))
+	for _, iv := range ivs {
+		if n := len(merged); n > 0 && iv[0] <= merged[n-1][1] {
+			if iv[1] > merged[n-1][1] {
+				merged[n-1][1] = iv[1]
+			}
+			continue
+		}
+		merged = append(merged, iv)
+	}
+	return merged
+}
+
+// coversReference reports whether existing's chunks cover every byte range that
+// reference's chunks cover. Equal total coverage is not sufficient proof — extra
+// chunks at other offsets can hide a missing range — so containment is verified
+// range by range against the merged interval sets.
+func coversReference(existing, reference *filer_pb.Entry) bool {
+	ex := mergedIntervals(existing)
+	i := 0
+	for _, r := range mergedIntervals(reference) {
+		lo, hi := r[0], r[1]
+		for lo < hi {
+			for i < len(ex) && ex[i][1] <= lo {
+				i++
+			}
+			if i >= len(ex) || ex[i][0] > lo {
+				return false // a byte in reference is not covered by existing
+			}
+			lo = ex[i][1]
+		}
+	}
+	return true
+}

weed/replication/sink/filersink/filer_sink_test.go

@@ -0,0 +1,342 @@
+package filersink
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// entry builds a file entry with the given mtime and size for the update-action
+// decision, which only looks at those two attributes.
+func entry(mtime int64, size uint64) *filer_pb.Entry {
+	return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: size}}
+}
+
+// entryNs adds a sub-second component to check same-second ordering.
+func entryNs(mtime int64, ns int32, size uint64) *filer_pb.Entry {
+	return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, MtimeNs: ns, FileSize: size}}
+}
+
+// entryChunks builds an entry with nChunks contiguous 1-byte chunks at a given
+// mtime, so coveredBytes == nChunks. Different counts give different coverage,
+// which is what the truncation-repair routing keys on.
+func entryChunks(mtime int64, nChunks int) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: uint64(nChunks)}}
+	for i := range nChunks {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{
+			FileId: "3," + string(rune('a'+i)), Offset: int64(i), Size: 1,
+		})
+	}
+	return e
+}
+
+// entryMd5 is like entryChunks but also sets attr.Md5, so filer.ETag returns the
+// stored Md5 (the S3 PutObject case) instead of a chunk-derived ETag. Used to show
+// that coverage — not the Md5-backed ETag — drives the truncation decision.
+func entryMd5(mtime int64, nChunks int, md5 []byte) *filer_pb.Entry {
+	e := entryChunks(mtime, nChunks)
+	e.Attributes.Md5 = md5
+	return e
+}
+
+// srcEntry builds a source-side entry at mtime whose chunks carry the given
+// FileIds — the identities a backup destination's SourceFileId is matched against.
+func srcEntry(mtime int64, fids ...string) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for i, f := range fids {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{FileId: f, Offset: int64(i), Size: 1})
+	}
+	return e
+}
+
+// dstEntry builds a destination entry at mtime. An empty src marks a chunk
+// written out-of-band (no SourceFileId, e.g. rsync/direct); otherwise it is a
+// backup chunk copied from that source FileId.
+func dstEntry(mtime int64, srcFids ...string) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for i, s := range srcFids {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{FileId: "dst", SourceFileId: s, Offset: int64(i), Size: 1})
+	}
+	return e
+}
+
+// ivEntry builds an entry at mtime with chunks at the given [offset,end) byte
+// ranges — for coverage/containment tests where exact ranges matter.
+func ivEntry(mtime int64, ivs ...[2]int64) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for _, iv := range ivs {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{Offset: iv[0], Size: uint64(iv[1] - iv[0])})
+	}
+	return e
+}
+
+// getEntryMtimeNs must order versions written within the same second, which
+// plain second-grained mtime cannot. It also must be nil-safe.
+func TestGetEntryMtimeNs(t *testing.T) {
+	if got := getEntryMtimeNs(nil); got != 0 {
+		t.Fatalf("nil entry: got %d, want 0", got)
+	}
+	if got := getEntryMtimeNs(&filer_pb.Entry{}); got != 0 {
+		t.Fatalf("nil attributes: got %d, want 0", got)
+	}
+
+	mk := func(sec int64, ns int32) *filer_pb.Entry {
+		return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: sec, MtimeNs: ns}}
+	}
+	if got, want := getEntryMtimeNs(mk(5, 200)), int64(5_000_000_200); got != want {
+		t.Fatalf("full-ns: got %d, want %d", got, want)
+	}
+	// same second, later sub-second component must compare strictly greater
+	if !(getEntryMtimeNs(mk(5, 200)) > getEntryMtimeNs(mk(5, 100))) {
+		t.Fatalf("same-second ordering failed: %d not > %d",
+			getEntryMtimeNs(mk(5, 200)), getEntryMtimeNs(mk(5, 100)))
+	}
+}
+
+// onCorruptChunk must surface the error, not skip, when it can't confirm
+// supersession (no source filer here) — otherwise it could drop a live file.
+func TestOnCorruptChunkRefusesWhenSupersessionUnconfirmed(t *testing.T) {
+	fs := &FilerSink{} // filerSource nil => hasSourceNewerVersion is always false
+	sentinel := errors.New("corrupt chunk: read 0 bytes, source says 107")
+
+	got := fs.onCorruptChunk("/buckets/x/config",
+		&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 200}},
+		sentinel)
+
+	if !errors.Is(got, sentinel) {
+		t.Fatalf("expected the error to be returned (refuse), got %v", got)
+	}
+}
+
+// onReplicateChunkError must NOT swallow a transient/partial replicate failure
+// (a volume-id lookup race or unretried manifest-chunk resolve during a write
+// burst) when it cannot confirm the source has moved on. Here filerSource is
+// nil, so hasSourceNewerVersion is always false — the "supersession unconfirmed"
+// case. Returning nil would let the metadata subscription advance its offset
+// past an event whose file was never written, silently dropping it (the
+// filer.backup partial-landing bug). It must propagate so the offset stays put
+// and the event is reprocessed. Genuine source-404 (gone/superseded) skipping is
+// covered by TestSourceSupersedes, which gates the return-nil branch.
+func TestOnReplicateChunkErrorPropagatesWhenSupersessionUnconfirmed(t *testing.T) {
+	fs := &FilerSink{} // filerSource nil => hasSourceNewerVersion always false
+	sentinel := errors.New("replicate manifest data chunks 7,02def: volume id 7 not found")
+
+	got := fs.onReplicateChunkError("/buckets/x/model.pt",
+		&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 200}},
+		sentinel)
+
+	if !errors.Is(got, sentinel) {
+		t.Fatalf("expected the transient replicate error to be propagated (not swallowed), got %v", got)
+	}
+}
+
+// chooseUpdateAction decides skip vs repair vs normal. A destination left
+// truncated by an earlier failed replication can carry a newer mtime (the
+// source preserved an old mtime while the partial copy was written recently),
+// so a pure mtime guard would strand the corruption. A shorter destination with
+// a newer mtime must be repaired, not skipped.
+func TestChooseUpdateAction(t *testing.T) {
+	tests := []struct {
+		name     string
+		existing *filer_pb.Entry
+		oldEntry *filer_pb.Entry
+		incoming *filer_pb.Entry
+		want     updateAction
+	}{
+		{"nil existing entry", nil, entry(100, 100), entry(200, 100), updateNormal},
+		{"destination older, catching up", entry(100, 50), entry(100, 50), entry(200, 100), updateNormal},
+		{"same mtime", entry(200, 50), entry(200, 50), entry(200, 100), updateNormal},
+		{"destination newer and complete", entry(300, 100), entry(100, 100), entry(200, 100), updateSkip},
+		{"destination newer and larger", entry(300, 200), entry(100, 100), entry(200, 100), updateSkip},
+		// out-of-order: a newer (smaller) destination must NOT be rolled back by an
+		// older, larger replayed event — even though it does not cover oldEntry.
+		{"newer smaller dest, older larger event -> skip", ivEntry(300, [2]int64{0, 5}), ivEntry(100, [2]int64{0, 10}), ivEntry(100, [2]int64{0, 10}), updateSkip},
+		// same second: sub-second ordering must still pick the winner
+		{"same second, destination newer and complete", entryNs(5, 200, 100), entryNs(5, 100, 100), entryNs(5, 100, 100), updateSkip},
+		{"same second, destination older", entryNs(5, 100, 100), entryNs(5, 100, 100), entryNs(5, 200, 100), updateNormal},
+		// chunk-incompleteness (the truncation bug): destination is at/behind the
+		// incoming mtime but covers fewer bytes than oldEntry, so the incremental
+		// diff cannot heal it — must repair in full, not append. entryChunks(n) has
+		// n contiguous 1-byte chunks → coverage n.
+		{"behind, covers less than oldEntry -> repair", entryChunks(200, 2), entryChunks(200, 4), entryChunks(200, 4), updateRepair},
+		{"behind, matches oldEntry coverage -> normal", entryChunks(200, 3), entryChunks(200, 3), entryChunks(200, 5), updateNormal},
+		{"behind, exceeds oldEntry coverage -> normal", entryChunks(200, 5), entryChunks(200, 3), entryChunks(200, 5), updateNormal},
+		// range containment, not scalar coverage: equal total bytes but a missing
+		// range (the destination has an extra range at another offset) → repair.
+		{"behind, equal total but range gap -> repair", ivEntry(200, [2]int64{0, 10}, [2]int64{30, 40}), ivEntry(200, [2]int64{0, 20}), ivEntry(200, [2]int64{0, 20}), updateRepair},
+		// same-range stale: full coverage but a backup-written chunk whose source
+		// oldEntry no longer references → repair only when the destination is
+		// provably older. Equal-mtime is deferred (cannot order same-second
+		// versions). Out-of-band (rsync) chunks are never stale-checked.
+		{"older, stale backup chunk -> repair", dstEntry(100, "A", "X"), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateRepair},
+		{"equal-mtime stale -> deferred (normal)", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+		{"older, rsync chunks -> normal", dstEntry(100, "", ""), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+		// in-sync fast path: destination holds exactly oldEntry's chunks → normal
+		{"in-sync (fast path) -> normal", dstEntry(100, "A", "B"), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := chooseUpdateAction(tc.existing, tc.oldEntry, tc.incoming); got != tc.want {
+				t.Fatalf("chooseUpdateAction = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// coversReference must verify range containment, not just equal total bytes: a
+// destination with the same byte count but a missing range (extra chunk elsewhere)
+// does not cover the reference.
+func TestCoversReference(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"empty reference", ivEntry(0), ivEntry(0), true},
+		{"reference no chunks", ivEntry(0, [2]int64{0, 10}), ivEntry(0), true},
+		{"exact", ivEntry(0, [2]int64{0, 10}), ivEntry(0, [2]int64{0, 10}), true},
+		{"superset", ivEntry(0, [2]int64{0, 20}), ivEntry(0, [2]int64{0, 10}), true},
+		{"truncated tail", ivEntry(0, [2]int64{0, 5}), ivEntry(0, [2]int64{0, 10}), false},
+		{"equal total, range gap", ivEntry(0, [2]int64{0, 10}, [2]int64{30, 40}), ivEntry(0, [2]int64{0, 20}), false},
+		{"contiguous parts cover", ivEntry(0, [2]int64{0, 10}, [2]int64{10, 20}), ivEntry(0, [2]int64{0, 20}), true},
+		{"overlapping parts cover", ivEntry(0, [2]int64{0, 12}, [2]int64{8, 20}), ivEntry(0, [2]int64{0, 20}), true},
+		{"existing empty, reference has bytes", ivEntry(0), ivEntry(0, [2]int64{0, 1}), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := coversReference(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("coversReference = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// destinationSatisfiesSource gates CreateEntry's skip. Coverage/identity must be
+// checked before ETag: a truncated or stale destination whose attr.Md5 still
+// matches the source (so filer.ETag is equal) must NOT be skipped, or
+// -initialSnapshot would leave it permanently diverged.
+func TestDestinationSatisfiesSource(t *testing.T) {
+	md5A := []byte("0123456789abcdef")
+	md5B := []byte("fedcba9876543210")
+	cases := []struct {
+		name               string
+		existing, incoming *filer_pb.Entry
+		want               bool
+	}{
+		{"nil existing", nil, entryChunks(200, 4), false},
+		{"identical complete -> skip", entryChunks(200, 4), entryChunks(200, 4), true},
+		{"truncated (no md5) -> replicate", entryChunks(200, 2), entryChunks(200, 4), false},
+		// regression: same attr.Md5 (equal filer.ETag) but fewer chunks must repair
+		{"md5-backed truncated -> replicate", entryMd5(200, 2, md5A), entryMd5(200, 4, md5A), false},
+		{"md5 equal and complete -> skip", entryMd5(200, 4, md5A), entryMd5(200, 4, md5A), true},
+		// complete but different content: keep newer, replace older
+		{"complete, dest newer -> skip", entryMd5(300, 4, md5A), entryMd5(200, 4, md5B), true},
+		{"complete, dest older -> replicate", entryMd5(200, 4, md5A), entryMd5(300, 4, md5B), false},
+		// same-range stale backup chunk: replicate only when provably older; an
+		// equal-mtime stale dest is deferred (skip), and rsync chunks are not
+		// stale-checked.
+		{"older, stale backup chunk -> replicate", dstEntry(100, "A", "X"), srcEntry(200, "A", "B"), false},
+		{"equal-mtime stale -> skip (deferred)", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), true},
+		{"rsync complete -> skip", dstEntry(200, "", ""), srcEntry(200, "A", "B"), true},
+		{"in-sync (fast path) -> skip", dstEntry(200, "A", "B"), srcEntry(200, "A", "B"), true},
+		// out-of-order: newer (smaller) destination must not be rolled back by an
+		// older, larger source replay.
+		{"newer smaller dest, older larger source -> skip", ivEntry(300, [2]int64{0, 5}), ivEntry(100, [2]int64{0, 10}), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := destinationSatisfiesSource(tc.existing, tc.incoming); got != tc.want {
+				t.Fatalf("destinationSatisfiesSource = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// destinationMatchesReference is the allocation-free fast path: true only when the
+// destination holds exactly the reference's chunks in order (by SourceFileId). It
+// must be conservative — never a false positive — so reorders, count mismatches,
+// and out-of-band (no SourceFileId) chunks all return false.
+func TestDestinationMatchesReference(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"both empty", dstEntry(0), srcEntry(0), true},
+		{"exact in order", dstEntry(0, "A", "B"), srcEntry(0, "A", "B"), true},
+		{"count mismatch", dstEntry(0, "A"), srcEntry(0, "A", "B"), false},
+		{"reordered", dstEntry(0, "B", "A"), srcEntry(0, "A", "B"), false},
+		{"stale identity", dstEntry(0, "A", "X"), srcEntry(0, "A", "B"), false},
+		{"out-of-band chunk", dstEntry(0, "A", ""), srcEntry(0, "A", "B"), false},
+		// Same identity but a different byte range must not fast-path as a match,
+		// so the precise checks still run if replication ever re-chunks.
+		{"shifted offset, same identity",
+			&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 0}, Chunks: []*filer_pb.FileChunk{
+				{FileId: "dst", SourceFileId: "A", Offset: 0, Size: 1},
+				{FileId: "dst", SourceFileId: "B", Offset: 5, Size: 1},
+			}}, srcEntry(0, "A", "B"), false},
+		{"larger size, same identity",
+			&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 0}, Chunks: []*filer_pb.FileChunk{
+				{FileId: "dst", SourceFileId: "A", Offset: 0, Size: 2},
+			}}, srcEntry(0, "A"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := destinationMatchesReference(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("destinationMatchesReference = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// hasStaleBackupChunk flags a destination that still holds a backup-written chunk
+// (SourceFileId set) the source reference no longer lists — a missed
+// overwrite/deletion. Out-of-band chunks (no SourceFileId, e.g. rsync) are ignored.
+func TestHasStaleBackupChunk(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"nil existing", nil, srcEntry(200, "A"), false},
+		{"in sync", dstEntry(200, "A", "B"), srcEntry(200, "A", "B"), false},
+		{"stale backup chunk", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), true},
+		{"rsync chunks ignored", dstEntry(200, "", ""), srcEntry(200, "A", "B"), false},
+		{"synced backup + rsync", dstEntry(200, "A", ""), srcEntry(200, "A", "B"), false},
+		{"stale backup among rsync", dstEntry(200, "X", ""), srcEntry(200, "A", "B"), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := hasStaleBackupChunk(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("hasStaleBackupChunk = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// updatedEntryKey must resolve the incoming entry's new path. For a rename the
+// supersession check has to target newParentPath/newEntry.Name, not the old key,
+// or the renamed-away old path looks deleted on the source and skips a live event.
+func TestUpdatedEntryKey(t *testing.T) {
+	named := func(name string) *filer_pb.Entry { return &filer_pb.Entry{Name: name} }
+	tests := []struct {
+		name          string
+		key           string
+		newParentPath string
+		entry         *filer_pb.Entry
+		want          string
+	}{
+		{"content update, no new parent", "/dst/a.txt", "", named("a.txt"), "/dst/a.txt"},
+		{"rename same dir", "/dst/old.txt", "/dst", named("new.txt"), "/dst/new.txt"},
+		{"rename to subdir", "/dst/old.txt", "/dst/sub", named("new.txt"), "/dst/sub/new.txt"},
+		{"root parent", "/old.txt", "/", named("new.txt"), "/new.txt"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := updatedEntryKey(tc.key, tc.newParentPath, tc.entry); got != tc.want {
+				t.Fatalf("updatedEntryKey = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}