snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/replication/repl_util/replication_util.go

@@ -0,0 +1,105 @@
+package repl_util
+
+import (
+	"context"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// CopyFromChunkViews copies chunk data with optional SSE decryption.
+// If entry has SSE-encrypted chunks, data is decrypted before writing.
+func CopyFromChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error, entry *filer_pb.Entry) error {
+	if entry != nil {
+		sseType, err := detectSSEType(entry)
+		if err != nil {
+			return err
+		}
+		if sseType != filer_pb.SSEType_NONE {
+			return copyWithDecryption(filerSource, entry, writeFunc)
+		}
+	}
+	return copyChunkViews(chunkViews, filerSource, writeFunc)
+}
+
+func copyWithDecryption(filerSource *source.FilerSource, entry *filer_pb.Entry, writeFunc func(data []byte) error) error {
+	reader := filer.NewFileReader(filerSource, entry)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		CloseReader(reader)
+		return err
+	}
+	defer CloseMaybeDecryptedReader(reader, decrypted)
+	buf := make([]byte, 128*1024)
+	for {
+		n, readErr := decrypted.Read(buf)
+		if n > 0 {
+			if writeErr := writeFunc(buf[:n]); writeErr != nil {
+				return writeErr
+			}
+		}
+		if readErr == io.EOF {
+			return nil
+		}
+		if readErr != nil {
+			return readErr
+		}
+	}
+}
+
+// CloseReader closes r if it implements io.Closer.
+func CloseReader(r io.Reader) {
+	if closer, ok := r.(io.Closer); ok {
+		closer.Close()
+	}
+}
+
+// CloseMaybeDecryptedReader closes the decrypted reader if it implements io.Closer,
+// otherwise falls back to closing the original reader.
+func CloseMaybeDecryptedReader(original, decrypted io.Reader) {
+	if closer, ok := decrypted.(io.Closer); ok {
+		closer.Close()
+	} else {
+		CloseReader(original)
+	}
+}
+
+func copyChunkViews(chunkViews *filer.IntervalList[*filer.ChunkView], filerSource *source.FilerSource, writeFunc func(data []byte) error) error {
+
+	for x := chunkViews.Front(); x != nil; x = x.Next {
+		chunk := x.Value
+
+		fileUrls, err := filerSource.LookupFileId(context.Background(), chunk.FileId)
+		if err != nil {
+			return err
+		}
+
+		var writeErr error
+		var shouldRetry bool
+		jwt := filer.ChunkReadJwt(fileUrls, chunk.FileId)
+
+		for _, fileUrl := range fileUrls {
+			shouldRetry, err = util_http.ReadUrlAsStream(context.Background(), fileUrl, jwt, chunk.CipherKey, chunk.IsGzipped, chunk.IsFullChunk(), chunk.OffsetInChunk, int(chunk.ViewSize), func(data []byte) {
+				writeErr = writeFunc(data)
+			})
+			if err != nil {
+				glog.V(1).Infof("read from %s: %v", fileUrl, err)
+			} else if writeErr != nil {
+				glog.V(1).Infof("copy from %s: %v", fileUrl, writeErr)
+			} else {
+				break
+			}
+		}
+		if shouldRetry && err != nil {
+			return err
+		}
+		if writeErr != nil {
+			return writeErr
+		}
+	}
+	return nil
+}

weed/replication/repl_util/sse_init.go

@@ -0,0 +1,55 @@
+package repl_util
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/kms"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var (
+	sseInitMu      sync.Mutex
+	sseInitialized bool
+)
+
+// InitializeSSEForReplication sets up SSE-S3 and SSE-KMS decryption so that
+// replication sinks can transparently decrypt encrypted objects.
+// SSE-S3 is initialized from the filer (KEK stored on filer).
+// SSE-KMS is initialized from Viper config (security.toml [kms] section or
+// WEED_KMS_* environment variables).
+// SSE-C cannot be decrypted (customer key not available) and will error at
+// decryption time.
+//
+// Safe to call multiple times; only the first successful initialization takes
+// effect. Failed attempts do not prevent future retries.
+func InitializeSSEForReplication(filerSource filer_pb.FilerClient) error {
+	sseInitMu.Lock()
+	defer sseInitMu.Unlock()
+	if sseInitialized {
+		return nil
+	}
+
+	// Initialize SSE-S3 key manager from filer
+	if err := s3api.GetSSES3KeyManager().InitializeWithFiler(filerSource); err != nil {
+		return err
+	}
+
+	// Attempt KMS initialization from Viper config.
+	// KMS configuration is typically in the S3 config file which the
+	// replication commands don't load directly. Support loading from
+	// security.toml [kms] section or WEED_KMS_* environment variables.
+	loader := kms.NewConfigLoader(util.GetViper())
+	if err := loader.LoadConfigurations(); err != nil {
+		glog.Warningf("KMS initialization from config failed: %v (SSE-KMS decryption will not be available)", err)
+	} else if err := loader.ValidateConfiguration(); err != nil {
+		glog.Warningf("KMS configuration validation failed: %v (SSE-KMS decryption will not be available)", err)
+	} else {
+		glog.V(0).Infof("KMS initialized for replication")
+	}
+
+	sseInitialized = true
+	return nil
+}

weed/replication/repl_util/sse_reader.go

@@ -0,0 +1,150 @@
+package repl_util
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+// MaybeDecryptReader wraps reader with SSE decryption if the entry has encrypted chunks.
+// Returns the original reader unchanged if no SSE encryption is detected.
+func MaybeDecryptReader(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry == nil {
+		return reader, nil
+	}
+
+	sseType, err := detectSSEType(entry)
+	if err != nil {
+		return nil, err
+	}
+	if sseType == filer_pb.SSEType_NONE {
+		return reader, nil
+	}
+
+	switch sseType {
+	case filer_pb.SSEType_SSE_S3:
+		return decryptSSES3(reader, entry)
+	case filer_pb.SSEType_SSE_KMS:
+		return decryptSSEKMS(reader, entry)
+	case filer_pb.SSEType_SSE_C:
+		return nil, fmt.Errorf("SSE-C encrypted object cannot be decrypted during replication (customer key not available)")
+	}
+
+	return nil, fmt.Errorf("unsupported SSE type: %v", sseType)
+}
+
+// MaybeDecryptContent decrypts inline entry content if SSE-encrypted.
+// Returns the original content unchanged if no SSE encryption is detected.
+func MaybeDecryptContent(content []byte, entry *filer_pb.Entry) ([]byte, error) {
+	if entry == nil || len(content) == 0 {
+		return content, nil
+	}
+
+	sseType, err := detectSSEType(entry)
+	if err != nil {
+		return nil, err
+	}
+	if sseType == filer_pb.SSEType_NONE {
+		return content, nil
+	}
+
+	reader := bytes.NewReader(content)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(decrypted)
+}
+
+func detectSSEType(entry *filer_pb.Entry) (filer_pb.SSEType, error) {
+	// Check chunk metadata first
+	var detected filer_pb.SSEType
+	for _, chunk := range entry.GetChunks() {
+		if chunk.SseType != filer_pb.SSEType_NONE {
+			if detected == filer_pb.SSEType_NONE {
+				detected = chunk.SseType
+			} else if chunk.SseType != detected {
+				return filer_pb.SSEType_NONE, fmt.Errorf("mixed SSE types in chunks: %v and %v", detected, chunk.SseType)
+			}
+		}
+	}
+	if detected != filer_pb.SSEType_NONE {
+		return detected, nil
+	}
+
+	// Fall back to extended metadata for inline objects (no chunks)
+	if entry.Extended != nil {
+		hasS3 := len(entry.Extended[s3_constants.SeaweedFSSSES3Key]) > 0
+		hasKMS := len(entry.Extended[s3_constants.SeaweedFSSSEKMSKey]) > 0
+		hasC := len(entry.Extended[s3_constants.SeaweedFSSSEIV]) > 0
+		count := 0
+		if hasS3 {
+			count++
+		}
+		if hasKMS {
+			count++
+		}
+		if hasC {
+			count++
+		}
+		if count > 1 {
+			return filer_pb.SSEType_NONE, fmt.Errorf("conflicting SSE metadata in entry: multiple SSE key types present")
+		}
+		if hasS3 {
+			return filer_pb.SSEType_SSE_S3, nil
+		}
+		if hasKMS {
+			return filer_pb.SSEType_SSE_KMS, nil
+		}
+		if hasC {
+			return filer_pb.SSEType_SSE_C, nil
+		}
+	}
+	return filer_pb.SSEType_NONE, nil
+}
+
+func decryptSSES3(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry.Extended == nil {
+		return nil, fmt.Errorf("SSE-S3 encrypted entry has no extended metadata")
+	}
+
+	keyData := entry.Extended[s3_constants.SeaweedFSSSES3Key]
+	if len(keyData) == 0 {
+		return nil, fmt.Errorf("SSE-S3 key metadata not found in entry")
+	}
+
+	keyManager := s3api.GetSSES3KeyManager()
+	sseS3Key, err := s3api.DeserializeSSES3Metadata(keyData, keyManager)
+	if err != nil {
+		return nil, fmt.Errorf("deserialize SSE-S3 metadata: %w", err)
+	}
+
+	iv, err := s3api.GetSSES3IV(entry, sseS3Key, keyManager)
+	if err != nil {
+		return nil, fmt.Errorf("get SSE-S3 IV: %w", err)
+	}
+
+	return s3api.CreateSSES3DecryptedReader(reader, sseS3Key, iv)
+}
+
+func decryptSSEKMS(reader io.Reader, entry *filer_pb.Entry) (io.Reader, error) {
+	if entry.Extended == nil {
+		return nil, fmt.Errorf("SSE-KMS encrypted entry has no extended metadata")
+	}
+
+	kmsMetadata := entry.Extended[s3_constants.SeaweedFSSSEKMSKey]
+	if len(kmsMetadata) == 0 {
+		return nil, fmt.Errorf("SSE-KMS key metadata not found in entry")
+	}
+
+	sseKMSKey, err := s3api.DeserializeSSEKMSMetadata(kmsMetadata)
+	if err != nil {
+		return nil, fmt.Errorf("deserialize SSE-KMS metadata: %w", err)
+	}
+
+	return s3api.CreateSSEKMSDecryptedReader(reader, sseKMSKey)
+}

weed/replication/repl_util/sse_reader_test.go

@@ -0,0 +1,534 @@
+package repl_util
+
+import (
+	"bytes"
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/kms"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestDetectSSEType(t *testing.T) {
+	tests := []struct {
+		name      string
+		entry     *filer_pb.Entry
+		wantType  filer_pb.SSEType
+		wantError bool
+	}{
+		{
+			name:     "no chunks no extended",
+			entry:    &filer_pb.Entry{},
+			wantType: filer_pb.SSEType_NONE,
+		},
+		{
+			name: "plaintext chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_NONE},
+					{SseType: filer_pb.SSEType_NONE},
+				},
+			},
+			wantType: filer_pb.SSEType_NONE,
+		},
+		{
+			name: "uniform SSE-S3 chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+					{SseType: filer_pb.SSEType_SSE_S3},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+		{
+			name: "uniform SSE-KMS chunks",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_KMS},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_KMS,
+		},
+		{
+			name: "mixed chunk SSE types",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+					{SseType: filer_pb.SSEType_SSE_KMS},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "inline SSE-S3 via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSES3Key: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+		{
+			name: "inline SSE-KMS via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEKMSKey: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_KMS,
+		},
+		{
+			name: "inline SSE-C via extended",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEIV: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_C,
+		},
+		{
+			name: "conflicting extended metadata",
+			entry: &filer_pb.Entry{
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSES3Key:  {0x01},
+					s3_constants.SeaweedFSSSEKMSKey: {0x02},
+				},
+			},
+			wantError: true,
+		},
+		{
+			name: "chunks take precedence over extended",
+			entry: &filer_pb.Entry{
+				Chunks: []*filer_pb.FileChunk{
+					{SseType: filer_pb.SSEType_SSE_S3},
+				},
+				Extended: map[string][]byte{
+					s3_constants.SeaweedFSSSEKMSKey: {0x01},
+				},
+			},
+			wantType: filer_pb.SSEType_SSE_S3,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := detectSSEType(tt.entry)
+			if tt.wantError {
+				if err == nil {
+					t.Fatalf("expected error, got type %v", got)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if got != tt.wantType {
+				t.Errorf("got %v, want %v", got, tt.wantType)
+			}
+		})
+	}
+}
+
+func TestMaybeDecryptReader_Plaintext(t *testing.T) {
+	content := []byte("hello world")
+	entry := &filer_pb.Entry{}
+	reader := bytes.NewReader(content)
+
+	got, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	result, err := io.ReadAll(got)
+	if err != nil {
+		t.Fatalf("ReadAll error: %v", err)
+	}
+	if !bytes.Equal(result, content) {
+		t.Errorf("got %q, want %q", result, content)
+	}
+}
+
+func TestMaybeDecryptReader_NilEntry(t *testing.T) {
+	content := []byte("hello")
+	reader := bytes.NewReader(content)
+
+	got, err := MaybeDecryptReader(reader, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	result, err := io.ReadAll(got)
+	if err != nil {
+		t.Fatalf("ReadAll error: %v", err)
+	}
+	if !bytes.Equal(result, content) {
+		t.Errorf("got %q, want %q", result, content)
+	}
+}
+
+func TestMaybeDecryptReader_SSEC_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEIV: {0x01},
+		},
+	}
+	reader := bytes.NewReader([]byte("data"))
+
+	_, err := MaybeDecryptReader(reader, entry)
+	if err == nil {
+		t.Fatal("expected error for SSE-C")
+	}
+}
+
+func TestMaybeDecryptContent_Plaintext(t *testing.T) {
+	content := []byte("hello world")
+	entry := &filer_pb.Entry{}
+
+	got, err := MaybeDecryptContent(content, entry)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !bytes.Equal(got, content) {
+		t.Errorf("got %q, want %q", got, content)
+	}
+}
+
+func TestMaybeDecryptContent_NilEntry(t *testing.T) {
+	content := []byte("data")
+	got, err := MaybeDecryptContent(content, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !bytes.Equal(got, content) {
+		t.Errorf("got %q, want %q", got, content)
+	}
+}
+
+func TestMaybeDecryptContent_Empty(t *testing.T) {
+	got, err := MaybeDecryptContent(nil, &filer_pb.Entry{})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got != nil {
+		t.Errorf("expected nil, got %v", got)
+	}
+}
+
+func TestMaybeDecryptContent_SSEC_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEIV: {0x01},
+		},
+	}
+
+	_, err := MaybeDecryptContent([]byte("data"), entry)
+	if err == nil {
+		t.Fatal("expected error for SSE-C")
+	}
+}
+
+func TestMaybeDecryptContent_MixedExtended_Error(t *testing.T) {
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key:  {0x01},
+			s3_constants.SeaweedFSSSEKMSKey: {0x02},
+		},
+	}
+
+	_, err := MaybeDecryptContent([]byte("data"), entry)
+	if err == nil {
+		t.Fatal("expected error for conflicting SSE metadata")
+	}
+}
+
+// --- SSE-S3 integration tests ---
+// These tests exercise the full MaybeDecryptReader/MaybeDecryptContent path
+// for SSE-S3: detectSSEType → decryptSSES3 → DeserializeSSES3Metadata →
+// GetSSES3IV → CreateSSES3DecryptedReader. A test KEK is injected via
+// WEED_S3_SSE_KEK env var and a mock filer client.
+
+// testFilerClient is a minimal filer_pb.FilerClient mock that returns
+// ErrNotFound for all lookups (no KEK on filer — we use env var instead).
+type testFilerClient struct{}
+
+func (c *testFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fmt.Errorf("%w", filer_pb.ErrNotFound)
+}
+func (c *testFilerClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.Url }
+func (c *testFilerClient) GetDataCenter() string                     { return "" }
+
+// setupTestSSES3 initializes the global SSE-S3 key manager with a test KEK
+// via the WEED_S3_SSE_KEK env var and returns the KEK bytes + cleanup func.
+func setupTestSSES3(t *testing.T) (kek []byte, cleanup func()) {
+	t.Helper()
+
+	kek = make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, kek); err != nil {
+		t.Fatal(err)
+	}
+
+	// Force Viper to pick up the new env var
+	os.Setenv("WEED_S3_SSE_KEK", hex.EncodeToString(kek))
+
+	// Reset Viper cache so it reads the new env var
+	v := util.GetViper()
+	v.AutomaticEnv()
+
+	// Re-initialize the global key manager with the KEK from env
+	km := s3api.GetSSES3KeyManager()
+	if err := km.InitializeWithFiler(&testFilerClient{}); err != nil {
+		os.Unsetenv("WEED_S3_SSE_KEK")
+		t.Fatalf("InitializeWithFiler: %v", err)
+	}
+
+	return kek, func() {
+		os.Unsetenv("WEED_S3_SSE_KEK")
+		// Re-initialize with no KEK to clear the super key
+		km.InitializeWithFiler(&testFilerClient{})
+	}
+}
+
+func TestMaybeDecryptReader_SSES3(t *testing.T) {
+	_, cleanup := setupTestSSES3(t)
+	defer cleanup()
+
+	plaintext := []byte("SSE-S3 encrypted content for testing round-trip decryption")
+
+	// Generate a DEK and encrypt
+	sseKey, err := s3api.GenerateSSES3Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
+	if err != nil {
+		t.Fatalf("encrypt: %v", err)
+	}
+	ciphertext, err := io.ReadAll(encReader)
+	if err != nil {
+		t.Fatalf("read ciphertext: %v", err)
+	}
+
+	// Build serialized SSE-S3 metadata (uses the global key manager to
+	// envelope-encrypt the DEK with the test KEK)
+	sseKey.IV = encIV
+	metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
+	if err != nil {
+		t.Fatalf("serialize metadata: %v", err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key: metadataBytes,
+		},
+	}
+
+	// Test full path: MaybeDecryptReader → decryptSSES3 → DeserializeSSES3Metadata → CreateSSES3DecryptedReader
+	decrypted, err := MaybeDecryptReader(bytes.NewReader(ciphertext), entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptReader: %v", err)
+	}
+	result, err := io.ReadAll(decrypted)
+	if err != nil {
+		t.Fatalf("ReadAll: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+func TestMaybeDecryptContent_SSES3(t *testing.T) {
+	_, cleanup := setupTestSSES3(t)
+	defer cleanup()
+
+	plaintext := []byte("inline SSE-S3 content")
+
+	// Generate a DEK and encrypt inline content
+	sseKey, err := s3api.GenerateSSES3Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+	encReader, encIV, err := s3api.CreateSSES3EncryptedReader(bytes.NewReader(plaintext), sseKey)
+	if err != nil {
+		t.Fatalf("encrypt: %v", err)
+	}
+	ciphertext, err := io.ReadAll(encReader)
+	if err != nil {
+		t.Fatalf("read ciphertext: %v", err)
+	}
+
+	sseKey.IV = encIV
+	metadataBytes, err := s3api.SerializeSSES3Metadata(sseKey)
+	if err != nil {
+		t.Fatalf("serialize metadata: %v", err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSES3Key: metadataBytes,
+		},
+	}
+
+	// Test full path: MaybeDecryptContent → MaybeDecryptReader → decryptSSES3
+	result, err := MaybeDecryptContent(ciphertext, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptContent: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-S3 round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+// --- SSE-KMS integration tests ---
+
+// testKMSProvider is a minimal KMSProvider mock for testing.
+type testKMSProvider struct {
+	keyID     string
+	plaintext []byte // the DEK plaintext returned by Decrypt
+}
+
+func (p *testKMSProvider) GenerateDataKey(_ context.Context, _ *kms.GenerateDataKeyRequest) (*kms.GenerateDataKeyResponse, error) {
+	return nil, nil
+}
+
+func (p *testKMSProvider) Decrypt(_ context.Context, _ *kms.DecryptRequest) (*kms.DecryptResponse, error) {
+	return &kms.DecryptResponse{
+		KeyID:     p.keyID,
+		Plaintext: append([]byte(nil), p.plaintext...), // return a copy
+	}, nil
+}
+
+func (p *testKMSProvider) DescribeKey(_ context.Context, _ *kms.DescribeKeyRequest) (*kms.DescribeKeyResponse, error) {
+	return nil, nil
+}
+
+func (p *testKMSProvider) GetKeyID(_ context.Context, keyIdentifier string) (string, error) {
+	return p.keyID, nil
+}
+
+func (p *testKMSProvider) Close() error { return nil }
+
+func TestMaybeDecryptReader_SSEKMS(t *testing.T) {
+	plaintext := []byte("SSE-KMS encrypted content for testing")
+
+	// Generate a random DEK and IV
+	dek := make([]byte, 32)
+	iv := make([]byte, aes.BlockSize)
+	if _, err := io.ReadFull(rand.Reader, dek); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		t.Fatal(err)
+	}
+
+	// Encrypt with AES-CTR (same cipher mode as SSE-KMS)
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ciphertext := make([]byte, len(plaintext))
+	cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
+
+	// Set up a mock KMS provider that returns our DEK
+	keyID := "test-kms-key-1"
+	encryptedDEK := []byte("fake-encrypted-dek") // mock doesn't validate
+	kms.SetGlobalKMSProvider(&testKMSProvider{
+		keyID:     keyID,
+		plaintext: dek,
+	})
+	defer kms.SetGlobalKMSProvider(nil)
+
+	// Build serialized KMS metadata
+	kmsMetadata := s3api.SSEKMSMetadata{
+		Algorithm:        s3_constants.SSEAlgorithmKMS,
+		KeyID:            keyID,
+		EncryptedDataKey: base64.StdEncoding.EncodeToString(encryptedDEK),
+		IV:               base64.StdEncoding.EncodeToString(iv),
+	}
+	metadataBytes, err := json.Marshal(kmsMetadata)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
+		},
+	}
+
+	// Test MaybeDecryptReader
+	reader := bytes.NewReader(ciphertext)
+	decrypted, err := MaybeDecryptReader(reader, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptReader: %v", err)
+	}
+	result, err := io.ReadAll(decrypted)
+	if err != nil {
+		t.Fatalf("ReadAll: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
+	}
+}
+
+func TestMaybeDecryptContent_SSEKMS(t *testing.T) {
+	plaintext := []byte("inline SSE-KMS content")
+
+	dek := make([]byte, 32)
+	iv := make([]byte, aes.BlockSize)
+	if _, err := io.ReadFull(rand.Reader, dek); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		t.Fatal(err)
+	}
+
+	block, err := aes.NewCipher(dek)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ciphertext := make([]byte, len(plaintext))
+	cipher.NewCTR(block, iv).XORKeyStream(ciphertext, plaintext)
+
+	keyID := "test-kms-key-2"
+	kms.SetGlobalKMSProvider(&testKMSProvider{
+		keyID:     keyID,
+		plaintext: dek,
+	})
+	defer kms.SetGlobalKMSProvider(nil)
+
+	kmsMetadata := s3api.SSEKMSMetadata{
+		Algorithm:        s3_constants.SSEAlgorithmKMS,
+		KeyID:            keyID,
+		EncryptedDataKey: base64.StdEncoding.EncodeToString([]byte("fake-encrypted-dek")),
+		IV:               base64.StdEncoding.EncodeToString(iv),
+	}
+	metadataBytes, err := json.Marshal(kmsMetadata)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	entry := &filer_pb.Entry{
+		Extended: map[string][]byte{
+			s3_constants.SeaweedFSSSEKMSKey: metadataBytes,
+		},
+	}
+
+	result, err := MaybeDecryptContent(ciphertext, entry)
+	if err != nil {
+		t.Fatalf("MaybeDecryptContent: %v", err)
+	}
+	if !bytes.Equal(result, plaintext) {
+		t.Errorf("SSE-KMS round-trip failed: got %q, want %q", result, plaintext)
+	}
+}

weed/replication/replicator.go

@@ -0,0 +1,186 @@
+package replication
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+type Replicator struct {
+	sink        sink.ReplicationSink
+	source      *source.FilerSource
+	excludeDirs []string
+}
+
+func NewReplicator(sourceConfig util.Configuration, configPrefix string, dataSink sink.ReplicationSink) *Replicator {
+
+	source := &source.FilerSource{}
+	source.Initialize(sourceConfig, configPrefix)
+
+	if err := repl_util.InitializeSSEForReplication(source); err != nil {
+		glog.Warningf("SSE initialization failed: %v (encrypted objects may fail to replicate)", err)
+	}
+
+	dataSink.SetSourceFiler(source)
+
+	return &Replicator{
+		sink:        dataSink,
+		source:      source,
+		excludeDirs: sourceConfig.GetStringSlice(configPrefix + "excludeDirectories"),
+	}
+}
+
+func (r *Replicator) Replicate(ctx context.Context, key string, message *filer_pb.EventNotification) error {
+	if message.IsFromOtherCluster && r.sink.GetName() == "filer" {
+		return nil
+	}
+
+	oldEntry := message.OldEntry
+	newEntry := message.NewEntry
+	newParentPath := message.NewParentPath
+
+	oldInSource := util.IsEqualOrUnder(key, r.source.Dir) && !r.isExcluded(key)
+
+	// For rename events (both old and new entry present), check both paths
+	// against the source directory. Convert cross-boundary renames to
+	// create or delete so the sink stays consistent.
+	if oldEntry != nil && newEntry != nil {
+		newFullPath, targetParent := metadataEventTarget(key, newEntry, newParentPath)
+		newInSource := util.IsEqualOrUnder(newFullPath, r.source.Dir) && !r.isExcluded(newFullPath)
+
+		if !oldInSource && !newInSource {
+			return nil
+		}
+		if !oldInSource {
+			// Rename into watched directory: treat as create
+			oldEntry = nil
+			key = newFullPath
+			newParentPath = targetParent
+		} else if !newInSource {
+			// Rename out of watched directory: treat as delete
+			newEntry = nil
+			newParentPath = ""
+		}
+	} else if !oldInSource {
+		glog.V(4).Infof("skipping %v outside of %v", key, r.source.Dir)
+		return nil
+	}
+
+	var dateKey string
+	if r.sink.IsIncremental() {
+		var mTime int64
+		if newEntry != nil {
+			mTime = newEntry.Attributes.Mtime
+		} else if oldEntry != nil {
+			mTime = oldEntry.Attributes.Mtime
+		}
+		dateKey = time.Unix(mTime, 0).Format("2006-01-02")
+	}
+	oldSinkKey := r.sourceToSinkKey(key, dateKey)
+	glog.V(3).Infof("replicate %s => %s", key, oldSinkKey)
+
+	newSinkKey := oldSinkKey
+	newSinkParentPath := newParentPath
+	if oldEntry != nil && newEntry != nil {
+		targetSourceKey, targetSourceParent := metadataEventTarget(key, newEntry, newParentPath)
+		newSinkKey = r.sourceToSinkKey(targetSourceKey, dateKey)
+		newSinkParentPath = r.sourceToSinkPath(targetSourceParent, dateKey)
+	} else if newParentPath != "" && util.IsEqualOrUnder(newParentPath, r.source.Dir) {
+		newSinkParentPath = r.sourceToSinkPath(newParentPath, dateKey)
+	}
+
+	if oldEntry != nil && newEntry == nil {
+		glog.V(4).Infof("deleting %v", oldSinkKey)
+		return r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
+	}
+	if oldEntry == nil && newEntry != nil {
+		glog.V(4).Infof("creating %v", oldSinkKey)
+		return r.sink.CreateEntry(oldSinkKey, newEntry, message.Signatures)
+	}
+	if oldEntry == nil && newEntry == nil {
+		glog.V(0).Infof("weird message %+v", message)
+		return nil
+	}
+
+	if oldSinkKey != newSinkKey {
+		// A real move: the path changed. UpdateEntry cannot move an entry.
+		if mover, ok := r.sink.(sink.EntryMover); ok {
+			glog.V(4).Infof("moving %v => %v", oldSinkKey, newSinkKey)
+			return mover.MoveEntry(oldSinkKey, newSinkKey, newEntry, message.Signatures)
+		}
+		// Sinks without a native move: create at the new key first, then delete the
+		// old, so a crash between the two leaves the entry visible under both names
+		// rather than gone.
+		glog.V(4).Infof("creating renamed %v", newSinkKey)
+		if err := r.sink.CreateEntry(newSinkKey, newEntry, message.Signatures); err != nil {
+			return fmt.Errorf("create renamed entry %v: %w", newSinkKey, err)
+		}
+		return r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, false, message.Signatures)
+	}
+
+	// oldSinkKey == newSinkKey: pure in-place update (same path, content/attrs changed).
+	foundExisting, err := r.sink.UpdateEntry(oldSinkKey, oldEntry, newSinkParentPath, newEntry, message.DeleteChunks, message.Signatures)
+	if foundExisting {
+		glog.V(4).Infof("updated %v", oldSinkKey)
+		return err
+	}
+	err = r.sink.DeleteEntry(oldSinkKey, oldEntry.IsDirectory, false, message.Signatures)
+	if err != nil {
+		return fmt.Errorf("delete old entry %v: %w", oldSinkKey, err)
+	}
+	glog.V(4).Infof("creating missing %v", newSinkKey)
+	return r.sink.CreateEntry(newSinkKey, newEntry, message.Signatures)
+}
+
+func (r *Replicator) isExcluded(path string) bool {
+	for _, excludeDir := range r.excludeDirs {
+		if util.IsEqualOrUnder(path, excludeDir) {
+			return true
+		}
+	}
+	return false
+}
+
+func (r *Replicator) sourceToSinkKey(sourceKey, dateKey string) string {
+	return util.Join(r.sink.GetSinkToDirectory(), dateKey, sourceKey[len(r.source.Dir):])
+}
+
+func (r *Replicator) sourceToSinkPath(sourcePath, dateKey string) string {
+	return util.Join(r.sink.GetSinkToDirectory(), dateKey, sourcePath[len(r.source.Dir):])
+}
+
+func metadataEventTarget(key string, newEntry *filer_pb.Entry, newParentPath string) (targetKey, targetParent string) {
+	if newEntry == nil {
+		return "", ""
+	}
+
+	targetParent = newParentPath
+	if targetParent == "" {
+		targetParent, _ = util.FullPath(key).DirAndName()
+	}
+
+	return util.Join(targetParent, newEntry.Name), targetParent
+}
+
+func ReadFilerSignature(grpcDialOption grpc.DialOption, filer pb.ServerAddress) (filerSignature int32, readErr error) {
+	if readErr = pb.WithFilerClient(false, 0, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		if resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}); err != nil {
+			return fmt.Errorf("GetFilerConfiguration %s: %v", filer, err)
+		} else {
+			filerSignature = resp.Signature
+		}
+		return nil
+	}); readErr != nil {
+		return 0, readErr
+	}
+	return filerSignature, nil
+}

weed/replication/replicator_test.go

@@ -0,0 +1,411 @@
+package replication
+
+import (
+	"context"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var _ sink.ReplicationSink = (*recordingSink)(nil)
+
+type deleteCall struct {
+	key         string
+	isDirectory bool
+}
+
+type createCall struct {
+	key string
+}
+
+type updateCall struct {
+	key           string
+	newParentPath string
+}
+
+type recordingSink struct {
+	name                string
+	sinkToDirectory     string
+	incremental         bool
+	updateFoundExisting bool
+
+	deleteCalls []deleteCall
+	createCalls []createCall
+	updateCalls []updateCall
+
+	// ordered records the sink method names in call order so tests can assert
+	// create-before-delete sequencing.
+	ordered []string
+}
+
+func (s *recordingSink) GetName() string {
+	return s.name
+}
+
+func (s *recordingSink) Initialize(util.Configuration, string) error {
+	return nil
+}
+
+func (s *recordingSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+	s.deleteCalls = append(s.deleteCalls, deleteCall{key: key, isDirectory: isDirectory})
+	s.ordered = append(s.ordered, "delete")
+	return nil
+}
+
+func (s *recordingSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+	s.createCalls = append(s.createCalls, createCall{key: key})
+	s.ordered = append(s.ordered, "create")
+	return nil
+}
+
+func (s *recordingSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
+	s.updateCalls = append(s.updateCalls, updateCall{key: key, newParentPath: newParentPath})
+	s.ordered = append(s.ordered, "update")
+	return s.updateFoundExisting, nil
+}
+
+func (s *recordingSink) GetSinkToDirectory() string {
+	return s.sinkToDirectory
+}
+
+func (s *recordingSink) GetDestinationIdentity() string {
+	return s.sinkToDirectory
+}
+
+func (s *recordingSink) SetSourceFiler(*source.FilerSource) {}
+
+func (s *recordingSink) IsIncremental() bool {
+	return s.incremental
+}
+
+type moveCall struct {
+	oldKey string
+	newKey string
+}
+
+// movingSink is a recordingSink that also implements sink.EntryMover, modeling
+// a sink (like the filer) with a native atomic move.
+type movingSink struct {
+	*recordingSink
+	moveCalls []moveCall
+}
+
+var _ sink.EntryMover = (*movingSink)(nil)
+
+func (s *movingSink) MoveEntry(oldKey, newKey string, newEntry *filer_pb.Entry, signatures []int32) error {
+	s.moveCalls = append(s.moveCalls, moveCall{oldKey: oldKey, newKey: newKey})
+	s.ordered = append(s.ordered, "move")
+	return nil
+}
+
+func TestReplicateRenameUsesTargetKeyForNonFilerSink(t *testing.T) {
+	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/source"},
+	}
+
+	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "renamed.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewParentPath: "/source/new",
+	})
+	if err != nil {
+		t.Fatalf("Replicate rename: %v", err)
+	}
+
+	if len(s.updateCalls) != 0 {
+		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
+	}
+	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
+		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
+	}
+	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
+		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
+	}
+	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
+		t.Fatalf("call order = %v, want create before delete %v", got, want)
+	}
+}
+
+// A real move to a filer sink also goes through create-then-delete (the
+// filer-sink special-case that previously routed renames to UpdateEntry is
+// gone): UpdateEntry cannot move an entry across paths.
+// A sink without a native move falls back to create-then-delete for a rename.
+func TestReplicateRenameWithoutMoverUsesCreateThenDelete(t *testing.T) {
+	s := &recordingSink{
+		name:                "s3",
+		sinkToDirectory:     "/dest",
+		updateFoundExisting: true,
+	}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/source"},
+	}
+
+	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "renamed.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewParentPath: "/source/new",
+	})
+	if err != nil {
+		t.Fatalf("Replicate rename: %v", err)
+	}
+
+	if len(s.updateCalls) != 0 {
+		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
+	}
+	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
+		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
+	}
+	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
+		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
+	}
+	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
+		t.Fatalf("call order = %v, want create before delete %v", got, want)
+	}
+}
+
+// A sink with a native move (the filer) relocates a rename via MoveEntry — never
+// create-then-delete, so a failed copy can't delete the only valid destination.
+func TestReplicateRenameUsesMoveEntryWhenSupported(t *testing.T) {
+	s := &movingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/source"},
+	}
+
+	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
+		OldEntry:      &filer_pb.Entry{Name: "file.txt"},
+		NewEntry:      &filer_pb.Entry{Name: "renamed.txt"},
+		NewParentPath: "/source/new",
+	})
+	if err != nil {
+		t.Fatalf("Replicate rename: %v", err)
+	}
+
+	if len(s.moveCalls) != 1 || s.moveCalls[0].oldKey != "/dest/old/file.txt" || s.moveCalls[0].newKey != "/dest/new/renamed.txt" {
+		t.Fatalf("move calls = %+v, want one move /dest/old/file.txt => /dest/new/renamed.txt", s.moveCalls)
+	}
+	if len(s.createCalls) != 0 || len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
+		t.Fatalf("native move must not create/delete/update: creates=%v deletes=%v updates=%v", s.createCalls, s.deleteCalls, s.updateCalls)
+	}
+}
+
+// An in-place update (same parent + same name, so oldSinkKey == newSinkKey)
+// still routes to UpdateEntry rather than create-then-delete.
+func TestReplicateInPlaceUpdateUsesUpdateEntry(t *testing.T) {
+	s := &recordingSink{
+		name:                "filer",
+		sinkToDirectory:     "/dest",
+		updateFoundExisting: true,
+	}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/source"},
+	}
+
+	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 456,
+			},
+		},
+		NewParentPath: "/source/dir",
+	})
+	if err != nil {
+		t.Fatalf("Replicate in-place update: %v", err)
+	}
+
+	if len(s.updateCalls) != 1 {
+		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
+	}
+	if s.updateCalls[0].key != "/dest/dir/file.txt" {
+		t.Fatalf("update key = %q, want /dest/dir/file.txt", s.updateCalls[0].key)
+	}
+	if s.updateCalls[0].newParentPath != "/dest/dir" {
+		t.Fatalf("update newParentPath = %q, want /dest/dir", s.updateCalls[0].newParentPath)
+	}
+	if len(s.deleteCalls) != 0 || len(s.createCalls) != 0 {
+		t.Fatalf("unexpected delete/create calls: deletes=%+v creates=%+v", s.deleteCalls, s.createCalls)
+	}
+}
+
+// When the in-place update finds no existing entry to update, fall back to
+// delete-then-create at the same key.
+func TestReplicateInPlaceUpdateFallbackCreates(t *testing.T) {
+	s := &recordingSink{
+		name:                "filer",
+		sinkToDirectory:     "/dest",
+		updateFoundExisting: false,
+	}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/source"},
+	}
+
+	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 456,
+			},
+		},
+		NewParentPath: "/source/dir",
+	})
+	if err != nil {
+		t.Fatalf("Replicate in-place update fallback: %v", err)
+	}
+
+	if len(s.updateCalls) != 1 {
+		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
+	}
+	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/file.txt" {
+		t.Fatalf("delete calls = %+v, want same sink key", s.deleteCalls)
+	}
+	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/dir/file.txt" {
+		t.Fatalf("create calls = %+v, want same sink key", s.createCalls)
+	}
+	if got, want := s.ordered, []string{"update", "delete", "create"}; !equalStrings(got, want) {
+		t.Fatalf("call order = %v, want %v", got, want)
+	}
+}
+
+func equalStrings(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
+	tests := []struct {
+		name      string
+		candidate string
+		other     string
+		expected  bool
+	}{
+		{name: "equal", candidate: "/foo", other: "/foo", expected: true},
+		{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
+		{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
+		{name: "root", candidate: "/foo/bar", other: "/", expected: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
+				t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestReplicateRenameOutToSiblingPrefixBecomesDelete(t *testing.T) {
+	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
+	r := &Replicator{
+		sink:   s,
+		source: &source.FilerSource{Dir: "/foo"},
+	}
+
+	err := r.Replicate(context.Background(), "/foo/old/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewParentPath: "/foobar/new",
+	})
+	if err != nil {
+		t.Fatalf("Replicate rename out to sibling prefix: %v", err)
+	}
+
+	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
+		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
+	}
+	if len(s.createCalls) != 0 || len(s.updateCalls) != 0 {
+		t.Fatalf("unexpected create/update calls: creates=%+v updates=%+v", s.createCalls, s.updateCalls)
+	}
+}
+
+func TestReplicateRenameFromExcludedDirBecomesCreate(t *testing.T) {
+	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
+	r := &Replicator{
+		sink:        s,
+		source:      &source.FilerSource{Dir: "/foo"},
+		excludeDirs: []string{"/foo/excluded"},
+	}
+
+	err := r.Replicate(context.Background(), "/foo/excluded/file.txt", &filer_pb.EventNotification{
+		OldEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewEntry: &filer_pb.Entry{
+			Name: "file.txt",
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: 123,
+			},
+		},
+		NewParentPath: "/foo/live",
+	})
+	if err != nil {
+		t.Fatalf("Replicate rename from excluded dir: %v", err)
+	}
+
+	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/live/file.txt" {
+		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
+	}
+	if len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
+		t.Fatalf("unexpected delete/update calls: deletes=%+v updates=%+v", s.deleteCalls, s.updateCalls)
+	}
+}

weed/replication/sink/azuresink/azure_sink.go

@@ -0,0 +1,293 @@
+package azuresink
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage/azure"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type AzureSink struct {
+	client        *azblob.Client
+	accountName   string
+	endpoint      string
+	container     string
+	dir           string
+	filerSource   *source.FilerSource
+	isIncremental bool
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &AzureSink{})
+}
+
+func (g *AzureSink) GetName() string {
+	return "azure"
+}
+
+func (g *AzureSink) GetSinkToDirectory() string {
+	return g.dir
+}
+
+func (g *AzureSink) GetDestinationIdentity() string {
+	return g.accountName + "\x00" + g.endpoint + "\x00" + g.container + "\x00" + g.dir
+}
+
+func (g *AzureSink) IsIncremental() bool {
+	return g.isIncremental
+}
+
+func (g *AzureSink) Initialize(configuration util.Configuration, prefix string) error {
+	g.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	return g.initialize(
+		configuration.GetString(prefix+"account_name"),
+		configuration.GetString(prefix+"account_key"),
+		configuration.GetString(prefix+"client_id"),
+		configuration.GetString(prefix+"endpoint"),
+		configuration.GetString(prefix+"container"),
+		configuration.GetString(prefix+"directory"),
+	)
+}
+
+func (g *AzureSink) SetSourceFiler(s *source.FilerSource) {
+	g.filerSource = s
+}
+
+func (g *AzureSink) initialize(accountName, accountKey, clientID, endpoint, container, dir string) error {
+	g.accountName = accountName
+	g.endpoint = endpoint
+	g.container = container
+	g.dir = dir
+
+	client, err := azure.NewAzBlobClient(accountName, accountKey, clientID, endpoint, nil)
+	if err != nil {
+		return err
+	}
+
+	g.client = client
+
+	// Validate that the container exists early to catch configuration errors
+	containerClient := client.ServiceClient().NewContainerClient(container)
+	ctxValidate, cancelValidate := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelValidate()
+	_, err = containerClient.GetProperties(ctxValidate, nil)
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.ContainerNotFound) {
+			return fmt.Errorf("Azure container '%s' does not exist. Please create it first", container)
+		}
+		return fmt.Errorf("failed to validate Azure container '%s': %w", container, err)
+	}
+
+	return nil
+}
+
+func (g *AzureSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	key = cleanKey(key)
+
+	if isDirectory {
+		key = key + "/"
+	}
+
+	blobClient := g.client.ServiceClient().NewContainerClient(g.container).NewBlobClient(key)
+	ctxDelete, cancelDelete := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelDelete()
+	_, err := blobClient.Delete(ctxDelete, &blob.DeleteOptions{
+		DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
+	})
+	if err != nil {
+		// Make delete idempotent - don't return error if blob doesn't exist
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			return nil
+		}
+		return fmt.Errorf("azure delete %s/%s: %w", g.container, key, err)
+	}
+
+	return nil
+}
+
+func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+
+	key = cleanKey(key)
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	totalSize := filer.FileSize(entry)
+	chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
+
+	// Create append blob client
+	appendBlobClient := g.client.ServiceClient().NewContainerClient(g.container).NewAppendBlobClient(key)
+
+	// Try to create the blob first (without access conditions for initial creation)
+	ctxCreate, cancelCreate := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelCreate()
+	_, err := appendBlobClient.Create(ctxCreate, nil)
+
+	needsWrite := true
+	freshlyCreated := false
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobAlreadyExists) {
+			// Handle existing blob - check if overwrite is needed and perform it if necessary
+			var handleErr error
+			needsWrite, handleErr = g.handleExistingBlob(appendBlobClient, key, entry, totalSize)
+			if handleErr != nil {
+				return handleErr
+			}
+			// handleExistingBlob recreates the blob when needsWrite is true
+			freshlyCreated = needsWrite
+		} else {
+			return fmt.Errorf("azure create append blob %s/%s: %w", g.container, key, err)
+		}
+	} else {
+		freshlyCreated = true
+	}
+
+	// If we don't need to write (blob is up-to-date), return early
+	if !needsWrite {
+		return nil
+	}
+
+	// cleanupOnError deletes a freshly created blob when content write fails,
+	// preventing empty blobs from being left behind.
+	cleanupOnError := func(writeErr error) error {
+		if !freshlyCreated {
+			return writeErr
+		}
+		glog.Warningf("azure sink: cleaning up empty blob %s/%s after write failure: %v", g.container, key, writeErr)
+		ctxCleanup, cancelCleanup := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+		defer cancelCleanup()
+		if _, delErr := appendBlobClient.Delete(ctxCleanup, nil); delErr != nil {
+			if !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
+				glog.Warningf("azure sink: failed to clean up blob %s/%s: %v", g.container, key, delErr)
+			}
+		}
+		return writeErr
+	}
+
+	writeFunc := func(data []byte) error {
+		ctxWrite, cancelWrite := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+		defer cancelWrite()
+		_, writeErr := appendBlobClient.AppendBlock(ctxWrite, streaming.NopCloser(bytes.NewReader(data)), &appendblob.AppendBlockOptions{})
+		return writeErr
+	}
+
+	if len(entry.Content) > 0 {
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return cleanupOnError(fmt.Errorf("decrypt inline SSE content: %w", err))
+		}
+		if err := writeFunc(content); err != nil {
+			return cleanupOnError(err)
+		}
+		return nil
+	}
+
+	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
+		return cleanupOnError(err)
+	}
+
+	return nil
+}
+
+// handleExistingBlob determines whether an existing blob needs to be overwritten and performs the overwrite if necessary.
+// It returns:
+//   - needsWrite: true if the caller should write data to the blob, false if the blob is already up-to-date
+//   - error: any error encountered during the operation
+func (g *AzureSink) handleExistingBlob(appendBlobClient *appendblob.Client, key string, entry *filer_pb.Entry, totalSize uint64) (needsWrite bool, err error) {
+	// Get the blob's properties to decide whether to overwrite.
+	// Use a timeout to fail fast on network issues.
+	ctxProps, cancelProps := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelProps()
+	props, propErr := appendBlobClient.GetProperties(ctxProps, nil)
+
+	// Fail fast if we cannot fetch properties - we should not proceed to delete without knowing the blob state.
+	if propErr != nil {
+		return false, fmt.Errorf("azure get properties %s/%s: %w", g.container, key, propErr)
+	}
+
+	// Check if we can skip writing based on modification time and size.
+	if entry.Attributes != nil && entry.Attributes.Mtime > 0 && props.LastModified != nil && props.ContentLength != nil {
+		const clockSkewTolerance = int64(2) // seconds - allow small clock differences
+		remoteMtime := props.LastModified.Unix()
+		localMtime := entry.Attributes.Mtime
+		// Skip if remote is newer/same (within skew tolerance) and has the SAME size.
+		// This prevents skipping partial/corrupted files that may have a newer mtime.
+		if remoteMtime >= localMtime-clockSkewTolerance && *props.ContentLength == int64(totalSize) {
+			glog.V(2).Infof("skip overwriting %s/%s: remote is up-to-date (remote mtime: %d >= local mtime: %d, size: %d)",
+				g.container, key, remoteMtime, localMtime, *props.ContentLength)
+			return false, nil
+		}
+	}
+
+	// Blob is empty or outdated - we need to delete and recreate it.
+	// REQUIRE ETag for conditional delete to avoid race conditions and data loss.
+	if props.ETag == nil {
+		return false, fmt.Errorf("azure blob %s/%s: missing ETag; refusing to delete without conditional", g.container, key)
+	}
+
+	deleteOpts := &blob.DeleteOptions{
+		DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
+		AccessConditions: &blob.AccessConditions{
+			ModifiedAccessConditions: &blob.ModifiedAccessConditions{
+				IfMatch: props.ETag,
+			},
+		},
+	}
+
+	// Delete existing blob with conditional delete and timeout.
+	ctxDel, cancelDel := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelDel()
+	_, delErr := appendBlobClient.Delete(ctxDel, deleteOpts)
+
+	if delErr != nil {
+		// If the precondition fails, the blob was modified by another process after we checked it.
+		// Failing here is safe; replication will retry.
+		if bloberror.HasCode(delErr, bloberror.ConditionNotMet) {
+			return false, fmt.Errorf("azure blob %s/%s was modified concurrently, preventing overwrite: %w", g.container, key, delErr)
+		}
+		// Ignore BlobNotFound, as the goal is to delete it anyway.
+		if !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
+			return false, fmt.Errorf("azure delete existing blob %s/%s: %w", g.container, key, delErr)
+		}
+	}
+
+	// Recreate the blob with timeout.
+	ctxRecreate, cancelRecreate := context.WithTimeout(context.Background(), azure.DefaultAzureOpTimeout)
+	defer cancelRecreate()
+	_, createErr := appendBlobClient.Create(ctxRecreate, nil)
+
+	if createErr != nil {
+		// It's possible another process recreated it after our delete.
+		// Failing is safe, as a retry of the whole function will handle it.
+		return false, fmt.Errorf("azure recreate append blob %s/%s: %w", g.container, key, createErr)
+	}
+
+	return true, nil
+}
+
+func (g *AzureSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+	key = cleanKey(key)
+	return true, g.CreateEntry(key, newEntry, signatures)
+}
+
+func cleanKey(key string) string {
+	// Remove all leading slashes (TrimLeft handles multiple slashes, unlike TrimPrefix)
+	return strings.TrimLeft(key, "/")
+}

weed/replication/sink/azuresink/azure_sink_test.go

@@ -0,0 +1,476 @@
+package azuresink
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// MockConfiguration for testing
+type mockConfiguration struct {
+	values map[string]interface{}
+}
+
+func newMockConfiguration() *mockConfiguration {
+	return &mockConfiguration{
+		values: make(map[string]interface{}),
+	}
+}
+
+func (m *mockConfiguration) GetString(key string) string {
+	if v, ok := m.values[key]; ok {
+		return v.(string)
+	}
+	return ""
+}
+
+func (m *mockConfiguration) GetBool(key string) bool {
+	if v, ok := m.values[key]; ok {
+		return v.(bool)
+	}
+	return false
+}
+
+func (m *mockConfiguration) GetInt(key string) int {
+	if v, ok := m.values[key]; ok {
+		return v.(int)
+	}
+	return 0
+}
+
+func (m *mockConfiguration) GetInt64(key string) int64 {
+	if v, ok := m.values[key]; ok {
+		return v.(int64)
+	}
+	return 0
+}
+
+func (m *mockConfiguration) GetFloat64(key string) float64 {
+	if v, ok := m.values[key]; ok {
+		return v.(float64)
+	}
+	return 0.0
+}
+
+func (m *mockConfiguration) GetStringSlice(key string) []string {
+	if v, ok := m.values[key]; ok {
+		return v.([]string)
+	}
+	return nil
+}
+
+func (m *mockConfiguration) SetDefault(key string, value interface{}) {
+	if _, exists := m.values[key]; !exists {
+		m.values[key] = value
+	}
+}
+
+// Test the AzureSink interface implementation
+func TestAzureSinkInterface(t *testing.T) {
+	sink := &AzureSink{}
+
+	if sink.GetName() != "azure" {
+		t.Errorf("Expected name 'azure', got '%s'", sink.GetName())
+	}
+
+	// Test directory setting
+	sink.dir = "/test/dir"
+	if sink.GetSinkToDirectory() != "/test/dir" {
+		t.Errorf("Expected directory '/test/dir', got '%s'", sink.GetSinkToDirectory())
+	}
+
+	// Test incremental setting
+	sink.isIncremental = true
+	if !sink.IsIncremental() {
+		t.Error("Expected isIncremental to be true")
+	}
+}
+
+// Test Azure sink initialization
+func TestAzureSinkInitialization(t *testing.T) {
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure sink test: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	sink := &AzureSink{}
+
+	err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test")
+	if err != nil {
+		t.Fatalf("Failed to initialize Azure sink: %v", err)
+	}
+
+	if sink.container != testContainer {
+		t.Errorf("Expected container '%s', got '%s'", testContainer, sink.container)
+	}
+
+	if sink.dir != "/test" {
+		t.Errorf("Expected dir '/test', got '%s'", sink.dir)
+	}
+
+	if sink.client == nil {
+		t.Error("Expected client to be initialized")
+	}
+}
+
+// Test configuration-based initialization
+func TestAzureSinkInitializeFromConfig(t *testing.T) {
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure sink config test: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	config := newMockConfiguration()
+	config.values["azure.account_name"] = accountName
+	config.values["azure.account_key"] = accountKey
+	config.values["azure.container"] = testContainer
+	config.values["azure.directory"] = "/test"
+	config.values["azure.is_incremental"] = true
+
+	sink := &AzureSink{}
+	err := sink.Initialize(config, "azure.")
+	if err != nil {
+		t.Fatalf("Failed to initialize from config: %v", err)
+	}
+
+	if !sink.IsIncremental() {
+		t.Error("Expected incremental to be true")
+	}
+}
+
+// Test cleanKey function
+func TestCleanKey(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"/test/file.txt", "test/file.txt"},
+		{"test/file.txt", "test/file.txt"},
+		{"/", ""},
+		{"", ""},
+		{"/a/b/c", "a/b/c"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.input, func(t *testing.T) {
+			result := cleanKey(tt.input)
+			if result != tt.expected {
+				t.Errorf("cleanKey(%q) = %q, want %q", tt.input, result, tt.expected)
+			}
+		})
+	}
+}
+
+// Test entry operations (requires valid credentials)
+func TestAzureSinkEntryOperations(t *testing.T) {
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure sink entry test: credentials not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	sink := &AzureSink{}
+	err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test")
+	if err != nil {
+		t.Fatalf("Failed to initialize: %v", err)
+	}
+
+	// Test CreateEntry with directory (should be no-op)
+	t.Run("CreateDirectory", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			IsDirectory: true,
+		}
+		err := sink.CreateEntry("/test/dir", entry, nil)
+		if err != nil {
+			t.Errorf("CreateEntry for directory should not error: %v", err)
+		}
+	})
+
+	// Test CreateEntry with file
+	testKey := "/test-sink-file-" + time.Now().Format("20060102-150405") + ".txt"
+	t.Run("CreateFile", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			IsDirectory: false,
+			Content:     []byte("Test content for Azure sink"),
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: time.Now().Unix(),
+			},
+		}
+		err := sink.CreateEntry(testKey, entry, nil)
+		if err != nil {
+			t.Fatalf("Failed to create entry: %v", err)
+		}
+	})
+
+	// Test UpdateEntry
+	t.Run("UpdateEntry", func(t *testing.T) {
+		oldEntry := &filer_pb.Entry{
+			Content: []byte("Old content"),
+		}
+		newEntry := &filer_pb.Entry{
+			Content: []byte("New content for update test"),
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: time.Now().Unix(),
+			},
+		}
+		found, err := sink.UpdateEntry(testKey, oldEntry, "/test", newEntry, false, nil)
+		if err != nil {
+			t.Fatalf("Failed to update entry: %v", err)
+		}
+		if !found {
+			t.Error("Expected found to be true")
+		}
+	})
+
+	// Test DeleteEntry
+	t.Run("DeleteFile", func(t *testing.T) {
+		err := sink.DeleteEntry(testKey, false, false, nil)
+		if err != nil {
+			t.Fatalf("Failed to delete entry: %v", err)
+		}
+	})
+
+	// Test DeleteEntry with directory marker
+	testDirKey := "/test-dir-" + time.Now().Format("20060102-150405")
+	t.Run("DeleteDirectory", func(t *testing.T) {
+		// First create a directory marker
+		entry := &filer_pb.Entry{
+			IsDirectory: false,
+			Content:     []byte(""),
+		}
+		err := sink.CreateEntry(testDirKey+"/", entry, nil)
+		if err != nil {
+			t.Logf("Warning: Failed to create directory marker: %v", err)
+		}
+
+		// Then delete it
+		err = sink.DeleteEntry(testDirKey, true, false, nil)
+		if err != nil {
+			t.Logf("Warning: Failed to delete directory: %v", err)
+		}
+	})
+}
+
+// Test CreateEntry with precondition (IfUnmodifiedSince)
+func TestAzureSinkPrecondition(t *testing.T) {
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure sink precondition test: credentials not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	sink := &AzureSink{}
+	err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test")
+	if err != nil {
+		t.Fatalf("Failed to initialize: %v", err)
+	}
+
+	testKey := "/test-precondition-" + time.Now().Format("20060102-150405") + ".txt"
+
+	// Create initial entry
+	entry := &filer_pb.Entry{
+		Content: []byte("Initial content"),
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime: time.Now().Unix(),
+		},
+	}
+	err = sink.CreateEntry(testKey, entry, nil)
+	if err != nil {
+		t.Fatalf("Failed to create initial entry: %v", err)
+	}
+
+	// Try to create again with old mtime (should be skipped due to precondition)
+	oldEntry := &filer_pb.Entry{
+		Content: []byte("Should not overwrite"),
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime: time.Now().Add(-1 * time.Hour).Unix(), // Old timestamp
+		},
+	}
+	err = sink.CreateEntry(testKey, oldEntry, nil)
+	// Should either succeed (skip) or fail with precondition error
+	if err != nil {
+		t.Logf("Create with old mtime: %v (expected)", err)
+	}
+
+	// Clean up
+	sink.DeleteEntry(testKey, false, false, nil)
+}
+
+// Helper function to get blob content length with timeout
+func getBlobContentLength(t *testing.T, sink *AzureSink, key string) int64 {
+	t.Helper()
+	containerClient := sink.client.ServiceClient().NewContainerClient(sink.container)
+	blobClient := containerClient.NewAppendBlobClient(cleanKey(key))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+
+	props, err := blobClient.GetProperties(ctx, nil)
+	if err != nil {
+		t.Fatalf("Failed to get blob properties: %v", err)
+	}
+	if props.ContentLength == nil {
+		return 0
+	}
+	return *props.ContentLength
+}
+
+// Test that repeated creates don't result in zero-byte files (regression test for critical bug)
+func TestAzureSinkIdempotentCreate(t *testing.T) {
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure sink idempotent create test: credentials not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	sink := &AzureSink{}
+	err := sink.initialize(accountName, accountKey, "", "", testContainer, "/test")
+	if err != nil {
+		t.Fatalf("Failed to initialize: %v", err)
+	}
+
+	testKey := "/test-idempotent-" + time.Now().Format("20060102-150405") + ".txt"
+	testContent := []byte("This is test content that should never be empty!")
+
+	// Use fixed time reference for deterministic behavior
+	testTime := time.Now()
+
+	// Clean up at the end
+	defer sink.DeleteEntry(testKey, false, false, nil)
+
+	// Test 1: Create a file with content
+	t.Run("FirstCreate", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Content: testContent,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: testTime.Unix(),
+			},
+		}
+		err := sink.CreateEntry(testKey, entry, nil)
+		if err != nil {
+			t.Fatalf("Failed to create entry: %v", err)
+		}
+
+		// Verify the file has content (not zero bytes)
+		contentLength := getBlobContentLength(t, sink, testKey)
+		if contentLength == 0 {
+			t.Errorf("File has zero bytes after creation! Expected %d bytes", len(testContent))
+		} else if contentLength != int64(len(testContent)) {
+			t.Errorf("File size mismatch: expected %d, got %d", len(testContent), contentLength)
+		} else {
+			t.Logf("File created with correct size: %d bytes", contentLength)
+		}
+	})
+
+	// Test 2: Create the same file again (idempotent operation - simulates replication running multiple times)
+	// This is where the zero-byte bug occurred: blob existed, precondition failed, returned early without writing data
+	t.Run("IdempotentCreate", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Content: testContent,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: testTime.Add(1 * time.Second).Unix(), // Slightly newer mtime
+			},
+		}
+		err := sink.CreateEntry(testKey, entry, nil)
+		if err != nil {
+			t.Fatalf("Failed on idempotent create: %v", err)
+		}
+
+		// CRITICAL: Verify the file STILL has content (not zero bytes)
+		contentLength := getBlobContentLength(t, sink, testKey)
+		if contentLength == 0 {
+			t.Errorf("ZERO-BYTE BUG: File became empty after idempotent create! Expected %d bytes", len(testContent))
+		} else if contentLength < int64(len(testContent)) {
+			t.Errorf("File lost content: expected at least %d bytes, got %d", len(testContent), contentLength)
+		} else {
+			t.Logf("File still has content after idempotent create: %d bytes", contentLength)
+		}
+	})
+
+	// Test 3: Try creating with older mtime (should skip but not leave zero bytes)
+	t.Run("CreateWithOlderMtime", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Content: []byte("This content should be skipped"),
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: testTime.Add(-1 * time.Hour).Unix(), // Older timestamp
+			},
+		}
+		err := sink.CreateEntry(testKey, entry, nil)
+		// Should succeed by skipping (no error expected)
+		if err != nil {
+			t.Fatalf("Create with older mtime should be skipped and return no error, but got: %v", err)
+		}
+
+		// Verify file STILL has content
+		contentLength := getBlobContentLength(t, sink, testKey)
+		if contentLength == 0 {
+			t.Errorf("File became empty after create with older mtime!")
+		} else {
+			t.Logf("File preserved content despite older mtime: %d bytes", contentLength)
+		}
+	})
+}
+
+// Benchmark tests
+func BenchmarkCleanKey(b *testing.B) {
+	keys := []string{
+		"/simple/path.txt",
+		"no/leading/slash.txt",
+		"/",
+		"/complex/path/with/many/segments/file.txt",
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		cleanKey(keys[i%len(keys)])
+	}
+}
+
+// Test error handling with invalid credentials
+func TestAzureSinkInvalidCredentials(t *testing.T) {
+	sink := &AzureSink{}
+
+	err := sink.initialize("invalid-account", "aW52YWxpZGtleQ==", "", "", "test-container", "/test")
+	if err != nil {
+		t.Skip("Invalid credentials correctly rejected at initialization")
+	}
+
+	// If initialization succeeded, operations should fail
+	entry := &filer_pb.Entry{
+		Content: []byte("test"),
+	}
+	err = sink.CreateEntry("/test.txt", entry, nil)
+	if err == nil {
+		t.Log("Expected error with invalid credentials, but got none (might be cached)")
+	}
+}

weed/replication/sink/b2sink/b2_sink.go

@@ -0,0 +1,149 @@
+package B2Sink
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/kurin/blazer/b2"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type B2Sink struct {
+	client        *b2.Client
+	bucket        string
+	dir           string
+	filerSource   *source.FilerSource
+	isIncremental bool
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &B2Sink{})
+}
+
+func (g *B2Sink) GetName() string {
+	return "backblaze"
+}
+
+func (g *B2Sink) GetSinkToDirectory() string {
+	return g.dir
+}
+
+func (g *B2Sink) GetDestinationIdentity() string {
+	return g.bucket + "\x00" + g.dir
+}
+
+func (g *B2Sink) IsIncremental() bool {
+	return g.isIncremental
+}
+
+func (g *B2Sink) Initialize(configuration util.Configuration, prefix string) error {
+	g.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	return g.initialize(
+		configuration.GetString(prefix+"b2_account_id"),
+		configuration.GetString(prefix+"b2_master_application_key"),
+		configuration.GetString(prefix+"bucket"),
+		configuration.GetString(prefix+"directory"),
+	)
+}
+
+func (g *B2Sink) SetSourceFiler(s *source.FilerSource) {
+	g.filerSource = s
+}
+
+func (g *B2Sink) initialize(accountId, accountKey, bucket, dir string) error {
+	client, err := b2.NewClient(context.Background(), accountId, accountKey)
+	if err != nil {
+		return err
+	}
+
+	g.client = client
+	g.bucket = bucket
+	g.dir = dir
+
+	return nil
+}
+
+func (g *B2Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	key = cleanKey(key)
+
+	if isDirectory {
+		key = key + "/"
+	}
+
+	bucket, err := g.client.Bucket(context.Background(), g.bucket)
+	if err != nil {
+		return err
+	}
+
+	targetObject := bucket.Object(key)
+
+	err = targetObject.Delete(context.Background())
+	if err != nil {
+		// b2_download_file_by_name: 404: File with such name does not exist.
+		if strings.Contains(err.Error(), ": 404:") {
+			return nil
+		}
+	}
+	return err
+
+}
+
+func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+
+	key = cleanKey(key)
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	totalSize := filer.FileSize(entry)
+	chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
+
+	bucket, err := g.client.Bucket(context.Background(), g.bucket)
+	if err != nil {
+		return err
+	}
+
+	targetObject := bucket.Object(key)
+	writer := targetObject.NewWriter(context.Background())
+	defer writer.Close()
+
+	writeFunc := func(data []byte) error {
+		_, writeErr := writer.Write(data)
+		return writeErr
+	}
+
+	if len(entry.Content) > 0 {
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return fmt.Errorf("decrypt inline SSE content: %w", err)
+		}
+		return writeFunc(content)
+	}
+
+	if err := repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func (g *B2Sink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+	key = cleanKey(key)
+	return true, g.CreateEntry(key, newEntry, signatures)
+}
+
+func cleanKey(key string) string {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+	return key
+}

weed/replication/sink/filersink/README.txt

@@ -0,0 +1,12 @@
+How replication works
+======
+
+All metadata changes within current cluster would be notified to a message queue.
+
+If the meta data change is from other clusters, this metadata would change would not be notified to the message queue.
+
+So active<=>active replication is possible.
+
+
+All metadata changes would be published as metadata changes.
+So all mounts listening for metadata changes will get updated.

weed/replication/sink/filersink/fetch_write.go

@@ -0,0 +1,709 @@
+package filersink
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/schollz/progressbar/v3"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// missingSourceChunkGrace is how long a chunk the source cannot produce keeps
+// being retried before it counts as gone for good. Long enough to outlast a
+// volume server restart or a master failover, after which every volume that
+// still exists is registered again.
+var missingSourceChunkGrace = 30 * time.Minute
+
+// errSourceChunkMissing is a permanent replication failure: the source cluster
+// no longer holds the chunk's data, so no later attempt can fetch it.
+var errSourceChunkMissing = errors.New("source chunk missing")
+
+// isSourceChunkMissing reports whether the source answered that it does not have
+// the chunk — no location for its volume, or a 404 from the volume server that
+// does. Both shapes also cover a volume that is merely offline, which is why the
+// gate below waits them out instead of trusting a single answer.
+func isSourceChunkMissing(err error) bool {
+	return errors.Is(err, source.ErrVolumeNotFound) || errors.Is(err, util_http.ErrNotFound)
+}
+
+// missingSourceChunkGate decides when to stop retrying a chunk the source cannot
+// produce, so a lookup race or a restarting volume server is waited out while a
+// vacuumed one is written off instead of retried forever. The wait is kept per
+// volume on the sink: a volume that is gone took every file it held with it, and
+// waiting it out once per file would stall the sync for as long as it holds files.
+type missingSourceChunkGate struct {
+	sink     *FilerSink
+	volumeId string
+	gaveUp   bool
+}
+
+func (g *missingSourceChunkGate) isPermanent(err error) bool {
+	if !isSourceChunkMissing(err) {
+		return false
+	}
+	if time.Since(g.sink.sourceVolumeMissingSince(g.volumeId)) < missingSourceChunkGrace {
+		return false
+	}
+	g.gaveUp = true
+	return true
+}
+
+// wrap marks err permanent once the gate gave up, so the sink can tell a source
+// that lost the data from one that is only unreachable right now.
+func (g *missingSourceChunkGate) wrap(err error) error {
+	if err == nil || !g.gaveUp {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errSourceChunkMissing, err)
+}
+
+func (fs *FilerSink) replicateChunks(ctx context.Context, sourceChunks []*filer_pb.FileChunk, path string, sourceMtimeNs int64) (replicatedChunks []*filer_pb.FileChunk, err error) {
+	if len(sourceChunks) == 0 {
+		return
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	// a simple progress bar. Not ideal. Fix me.
+	var bar *progressbar.ProgressBar
+	if len(sourceChunks) > 1 {
+		name := filepath.Base(path)
+		bar = progressbar.NewOptions64(int64(len(sourceChunks)),
+			progressbar.OptionClearOnFinish(),
+			progressbar.OptionOnCompletion(func() {
+				fmt.Fprint(os.Stderr, "\n")
+			}),
+			progressbar.OptionFullWidth(),
+			progressbar.OptionSetDescription(name),
+		)
+	}
+
+	replicatedChunks = make([]*filer_pb.FileChunk, len(sourceChunks))
+	var errLock sync.Mutex
+	setError := func(e error) {
+		if e == nil {
+			return
+		}
+		errLock.Lock()
+		if err == nil {
+			err = e
+		}
+		errLock.Unlock()
+	}
+	hasError := func() bool {
+		errLock.Lock()
+		defer errLock.Unlock()
+		return err != nil
+	}
+
+	var wg sync.WaitGroup
+	for chunkIndex, sourceChunk := range sourceChunks {
+		if hasError() {
+			break
+		}
+
+		if sourceChunk.IsChunkManifest {
+			replicatedChunk, replicateErr := fs.replicateOneManifestChunk(ctx, sourceChunk, path, sourceMtimeNs)
+			if replicateErr != nil {
+				setError(replicateErr)
+				break
+			}
+			replicatedChunks[chunkIndex] = replicatedChunk
+			if bar != nil {
+				bar.Add(1)
+			}
+			continue
+		}
+
+		wg.Add(1)
+		index, source := chunkIndex, sourceChunk
+		fs.executor.Execute(func() {
+			defer wg.Done()
+			var replicatedChunk *filer_pb.FileChunk
+			retryErr := util.Retry("replicate chunks", func() error {
+				chunk, e := fs.replicateOneChunk(source, path, sourceMtimeNs)
+				if e != nil {
+					return e
+				}
+				replicatedChunk = chunk
+				return nil
+			})
+			if retryErr != nil {
+				setError(retryErr)
+				return
+			}
+
+			replicatedChunks[index] = replicatedChunk
+			if bar != nil {
+				bar.Add(1)
+			}
+		})
+	}
+	wg.Wait()
+
+	return
+}
+
+func (fs *FilerSink) replicateOneChunk(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
+
+	fileId, err := fs.fetchAndWrite(sourceChunk, path, sourceMtimeNs)
+	if err != nil {
+		return nil, fmt.Errorf("copy %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	return &filer_pb.FileChunk{
+		FileId:       fileId,
+		Offset:       sourceChunk.Offset,
+		Size:         sourceChunk.Size,
+		ModifiedTsNs: sourceChunk.ModifiedTsNs,
+		ETag:         sourceChunk.ETag,
+		SourceFileId: sourceChunk.GetFileIdString(),
+		CipherKey:    sourceChunk.CipherKey,
+		IsCompressed: sourceChunk.IsCompressed,
+		SseType:      sourceChunk.SseType,
+		SseMetadata:  sourceChunk.SseMetadata,
+	}, nil
+}
+
+func (fs *FilerSink) replicateOneManifestChunk(ctx context.Context, sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (*filer_pb.FileChunk, error) {
+	// LookupFileId can transiently fail during a write burst (volume not yet
+	// registered). Retry like the data-chunk path does; the gate stops on
+	// supersession or, when that cannot be checked, after a few attempts.
+	var resolvedChunks []*filer_pb.FileChunk
+	resolveName := fmt.Sprintf("resolve manifest %s", sourceChunk.GetFileIdString())
+	missingGate := fs.newMissingSourceChunkGate(sourceChunk.GetFileIdString())
+	err := util.RetryUntil(resolveName, func() error {
+		rc, e := filer.ResolveOneChunkManifest(ctx, fs.filerSource.LookupFileId, sourceChunk, nil)
+		if e != nil {
+			return e
+		}
+		resolvedChunks = rc
+		return nil
+	}, fs.manifestResolveRetryGate(path, sourceMtimeNs, sourceChunk.GetFileIdString(), missingGate))
+	if err != nil {
+		return nil, fmt.Errorf("resolve manifest %s: %w", sourceChunk.GetFileIdString(), missingGate.wrap(err))
+	}
+
+	replicatedResolvedChunks, err := fs.replicateChunks(ctx, resolvedChunks, path, sourceMtimeNs)
+	if err != nil {
+		return nil, fmt.Errorf("replicate manifest data chunks %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	manifestDataChunks := make([]*filer_pb.FileChunk, len(replicatedResolvedChunks))
+	for i, chunk := range replicatedResolvedChunks {
+		copied := *chunk
+		manifestDataChunks[i] = &copied
+	}
+	filer_pb.BeforeEntrySerialization(manifestDataChunks)
+	manifestData, err := proto.Marshal(&filer_pb.FileChunkManifest{
+		Chunks: manifestDataChunks,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("marshal manifest %s: %w", sourceChunk.GetFileIdString(), err)
+	}
+
+	manifestFileId, err := fs.uploadManifestChunk(path, sourceMtimeNs, sourceChunk.GetFileIdString(), manifestData)
+	if err != nil {
+		return nil, err
+	}
+
+	return &filer_pb.FileChunk{
+		FileId:          manifestFileId,
+		Offset:          sourceChunk.Offset,
+		Size:            sourceChunk.Size,
+		ModifiedTsNs:    sourceChunk.ModifiedTsNs,
+		ETag:            sourceChunk.ETag,
+		SourceFileId:    sourceChunk.GetFileIdString(),
+		IsChunkManifest: true,
+	}, nil
+}
+
+// maxUnverifiableResolveAttempts bounds manifest-resolve retries when
+// supersession cannot be checked for the target path.
+const maxUnverifiableResolveAttempts = 3
+
+// manifestResolveRetryGate decides whether a failing manifest resolve keeps
+// retrying: stop when the source superseded the replayed version (the caller
+// skips it as lossless), stop once the source has been unable to produce the
+// manifest for the whole grace period, stop immediately on non-transient errors
+// (e.g. corrupt manifest data) so the configured metadata error policy applies,
+// and stop after a few attempts when supersession cannot be checked at all (incremental
+// dated target keys don't map back to a source path) — propagating lets
+// filer.backup decide with the event's real source key instead of spinning
+// here forever.
+func (fs *FilerSink) manifestResolveRetryGate(path string, sourceMtimeNs int64, chunkName string, missingGate *missingSourceChunkGate) func(error) bool {
+	_, canCheckSupersession := fs.targetPathToSourcePath(path)
+	attempts := 0
+	return func(resolveErr error) (shouldContinue bool) {
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source manifest %s for %s: %v", chunkName, path, resolveErr)
+			return false
+		}
+		if missingGate.isPermanent(resolveErr) {
+			glog.Errorf("source has not had manifest %s for %s in %v, giving up on it: %v",
+				chunkName, path, missingSourceChunkGrace, resolveErr)
+			return false
+		}
+		if !isTransientResolveError(resolveErr) {
+			glog.V(0).Infof("resolve manifest %s for %s: non-transient error, propagating: %v", chunkName, path, resolveErr)
+			return false
+		}
+		attempts++
+		if !canCheckSupersession && attempts >= maxUnverifiableResolveAttempts {
+			glog.V(0).Infof("resolve manifest %s for %s: supersession unverifiable, propagating after %d attempts: %v", chunkName, path, attempts, resolveErr)
+			return false
+		}
+		glog.V(0).Infof("resolve manifest %s for %s: %v", chunkName, path, resolveErr)
+		return true
+	}
+}
+
+// isTransientResolveError reports whether a manifest resolve failure belongs to
+// the classes that clear on their own during a write burst — volume-lookup
+// races and network interruptions. Anything else (corrupt manifest data, bad
+// file ids) is permanent and must propagate rather than retry forever.
+func isTransientResolveError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if isRetryableNetworkError(err) {
+		return true
+	}
+	errStr := err.Error()
+	return strings.Contains(errStr, "LookupFileId") ||
+		(strings.Contains(errStr, "volume id") && strings.Contains(errStr, "not found"))
+}
+
+func (fs *FilerSink) uploadManifestChunk(path string, sourceMtimeNs int64, sourceFileId string, manifestData []byte) (fileId string, err error) {
+	uploader, err := fs.getUploader()
+	if err != nil {
+		glog.V(0).Infof("upload manifest data %v: %v", sourceFileId, err)
+		return "", fmt.Errorf("upload manifest data: %w", err)
+	}
+
+	retryName := fmt.Sprintf("replicate manifest chunk %s", sourceFileId)
+	err = util.RetryUntil(retryName, func() error {
+		uploadOption := &operation.UploadOption{
+			Filename:          "",
+			Cipher:            false,
+			IsInputCompressed: false,
+			MimeType:          "application/octet-stream",
+			PairMap:           nil,
+			RetryForever:      false,
+		}
+		if fs.writeChunkByFiler {
+			uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)
+		}
+		currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(
+			fs,
+			&filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: fs.replication,
+				Collection:  fs.collection,
+				TtlSec:      fs.ttlSec,
+				DataCenter:  fs.dataCenter,
+				DiskType:    fs.diskType,
+				Path:        path,
+			},
+			uploadOption,
+			bytes.NewReader(manifestData),
+		)
+		if uploadErr != nil {
+			return fmt.Errorf("upload manifest data: %w", uploadErr)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload manifest result: %v", uploadResult.Error)
+		}
+
+		fileId = currentFileId
+		return nil
+	}, func(uploadErr error) (shouldContinue bool) {
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source manifest %s for %s: %v", sourceFileId, path, uploadErr)
+			return false
+		}
+		glog.V(0).Infof("replicate manifest %s for %s: %v", sourceFileId, path, uploadErr)
+		return true
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return fileId, nil
+}
+
+func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string, sourceMtimeNs int64) (fileId string, err error) {
+	uploader, err := fs.getUploader()
+	if err != nil {
+		glog.V(0).Infof("upload source data %v: %v", sourceChunk.GetFileIdString(), err)
+		return "", fmt.Errorf("upload data: %w", err)
+	}
+
+	transferStatus := &ChunkTransferStatus{
+		ChunkTransferSnapshot: ChunkTransferSnapshot{
+			ChunkFileId: sourceChunk.GetFileIdString(),
+			Path:        path,
+			Status:      "downloading",
+		},
+	}
+	fs.activeTransfers.Store(sourceChunk.GetFileIdString(), transferStatus)
+	defer fs.activeTransfers.Delete(sourceChunk.GetFileIdString())
+
+	transientBackoff := time.Duration(0)
+	_, canCheckSupersession := fs.targetPathToSourcePath(path)
+	missingGate := fs.newMissingSourceChunkGate(sourceChunk.GetFileIdString())
+	var partialData []byte
+	var savedFilename string
+	var savedHeader http.Header
+	var savedSourceUrl string
+	retryName := fmt.Sprintf("replicate chunk %s", sourceChunk.GetFileIdString())
+	err = util.RetryUntil(retryName, func() error {
+		filename, header, resp, readErr := fs.filerSource.ReadPart(sourceChunk.GetFileIdString(), int64(len(partialData)))
+		if readErr != nil {
+			return fmt.Errorf("read part %s: %w", sourceChunk.GetFileIdString(), readErr)
+		}
+		defer util_http.CloseResponse(resp)
+
+		// Save metadata from first successful response
+		if len(partialData) == 0 {
+			savedFilename = filename
+			savedHeader = header
+			if resp.Request != nil && resp.Request.URL != nil {
+				savedSourceUrl = resp.Request.URL.String()
+			}
+		}
+
+		// Read the response body
+		data, readBodyErr := io.ReadAll(resp.Body)
+		if readBodyErr != nil {
+			// Keep whatever bytes we received before the error
+			partialData = append(partialData, data...)
+			return fmt.Errorf("read body: %w", readBodyErr)
+		}
+
+		// Combine with previously accumulated partial data
+		var fullData []byte
+		if len(partialData) > 0 {
+			fullData = append(partialData, data...)
+			glog.V(0).Infof("resumed reading %s, got %d + %d = %d bytes",
+				sourceChunk.GetFileIdString(), len(partialData), len(data), len(fullData))
+			partialData = nil
+		} else {
+			fullData = data
+		}
+
+		if err := validateReplicatedReadSize(sourceChunk, len(fullData)); err != nil {
+			return err
+		}
+		fs.sourceServed(sourceChunk.GetFileIdString())
+
+		transferStatus.mu.Lock()
+		transferStatus.BytesReceived = int64(len(fullData))
+		transferStatus.Status = "uploading"
+		transferStatus.mu.Unlock()
+
+		uploadOption := &operation.UploadOption{
+			Filename:          savedFilename,
+			Cipher:            false,
+			IsInputCompressed: "gzip" == savedHeader.Get("Content-Encoding"),
+			MimeType:          savedHeader.Get("Content-Type"),
+			PairMap:           nil,
+			RetryForever:      false,
+			SourceUrl:         savedSourceUrl,
+		}
+		if fs.writeChunkByFiler {
+			uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)
+		}
+		currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(
+			fs,
+			&filer_pb.AssignVolumeRequest{
+				Count:       1,
+				Replication: fs.replication,
+				Collection:  fs.collection,
+				TtlSec:      fs.ttlSec,
+				DataCenter:  fs.dataCenter,
+				DiskType:    fs.diskType,
+				Path:        path,
+			},
+			uploadOption,
+			util.NewBytesReader(fullData),
+		)
+		if uploadErr != nil {
+			return fmt.Errorf("upload data: %w", uploadErr)
+		}
+		if uploadResult.Error != "" {
+			return fmt.Errorf("upload result: %v", uploadResult.Error)
+		}
+
+		transientBackoff = 0
+		fileId = currentFileId
+		return nil
+	}, func(retryErr error) (shouldContinue bool) {
+		if errors.Is(retryErr, errChunkSizeMismatch) {
+			glog.V(0).Infof("permanent size mismatch replicating %s for %s: %v",
+				sourceChunk.GetFileIdString(), path, retryErr)
+			transferStatus.mu.Lock()
+			transferStatus.LastErr = retryErr.Error()
+			transferStatus.mu.Unlock()
+			return false
+		}
+		if fs.hasSourceNewerVersion(path, sourceMtimeNs) {
+			glog.V(1).Infof("skip retrying stale source %s for %s: %v", sourceChunk.GetFileIdString(), path, retryErr)
+			return false
+		}
+		transferStatus.mu.Lock()
+		transferStatus.LastErr = retryErr.Error()
+		transferStatus.mu.Unlock()
+		if isSourceChunkMissing(retryErr) && !canCheckSupersession {
+			glog.V(0).Infof("source does not have %s for %s, supersession unverifiable, propagating: %v",
+				sourceChunk.GetFileIdString(), path, retryErr)
+			return false
+		}
+		if missingGate.isPermanent(retryErr) {
+			glog.Errorf("source has not had %s for %s in %v, giving up on it: %v",
+				sourceChunk.GetFileIdString(), path, missingSourceChunkGrace, retryErr)
+			return false
+		}
+		if isRetryableNetworkError(retryErr) {
+			transientBackoff = nextTransientBackoff(transientBackoff)
+			transferStatus.mu.Lock()
+			transferStatus.BytesReceived = int64(len(partialData))
+			transferStatus.Status = fmt.Sprintf("waiting %v", transientBackoff)
+			transferStatus.mu.Unlock()
+			glog.V(0).Infof("connection interrupted while replicating %s for %s (%d bytes received so far), backing off %v: %v",
+				sourceChunk.GetFileIdString(), path, len(partialData), transientBackoff, retryErr)
+			time.Sleep(transientBackoff)
+			transferStatus.mu.Lock()
+			transferStatus.Status = "downloading"
+			transferStatus.mu.Unlock()
+		} else {
+			glog.V(0).Infof("replicate %s for %s: %v", sourceChunk.GetFileIdString(), path, retryErr)
+		}
+		return true
+	})
+	if err != nil {
+		return "", missingGate.wrap(err)
+	}
+
+	return fileId, nil
+}
+
+const maxTransientBackoff = 2 * time.Minute
+
+// nextTransientBackoff escalates 10s -> doubling -> 2m cap so an overloaded
+// destination can recover instead of being hammered by the RetryUntil loop.
+func nextTransientBackoff(current time.Duration) time.Duration {
+	if current < 10*time.Second {
+		return 10 * time.Second
+	}
+	current *= 2
+	if current > maxTransientBackoff {
+		current = maxTransientBackoff
+	}
+	return current
+}
+
+func isEofError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF)
+}
+
+// isRetryableNetworkError reports whether err is a transient network failure worth
+// a backoff-and-retry. A bare io.EOF counts here but not in util.IsTransientError:
+// mid-transfer it means the chunk read was truncated, not that a stream ended
+// cleanly.
+func isRetryableNetworkError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return isEofError(err) || util.IsTransientError(err)
+}
+
+// errChunkSizeMismatch is a permanent (non-retriable) replication failure.
+var errChunkSizeMismatch = errors.New("chunk size mismatch")
+
+func validateReplicatedReadSize(sourceChunk *filer_pb.FileChunk, readSize int) error {
+	if uint64(readSize) != sourceChunk.Size {
+		return fmt.Errorf("%w: read %s got %d bytes, source metadata says %d",
+			errChunkSizeMismatch, sourceChunk.GetFileIdString(),
+			readSize, sourceChunk.Size)
+	}
+	return nil
+}
+
+func (fs *FilerSink) newMissingSourceChunkGate(fileId string) *missingSourceChunkGate {
+	return &missingSourceChunkGate{sink: fs, volumeId: filer.VolumeId(fileId)}
+}
+
+// sourceVolumeMissingSince returns when the sink first found volumeId
+// unlocatable, recording now on the first sighting.
+func (fs *FilerSink) sourceVolumeMissingSince(volumeId string) time.Time {
+	since, _ := fs.missingVolumes.LoadOrStore(volumeId, time.Now())
+	return since.(time.Time)
+}
+
+// sourceServed records a chunk the source did serve: the probe
+// sourceStillServesChunks re-checks, and proof its volume is locatable again.
+func (fs *FilerSink) sourceServed(fileId string) {
+	fs.lastServedFileId.Store(&fileId)
+	fs.missingVolumes.Delete(filer.VolumeId(fileId))
+}
+
+// sourceStillServesChunks reports whether the source cluster can still produce the
+// last chunk it served. A volume with no locations reads the same whether it was
+// vacuumed away or every replica is down, so before an entry is written off the
+// sink re-reads a file id the source did serve: if that one cannot be produced
+// either the source is in an outage, and skipping would drop live files wholesale.
+// It is a read and not a lookup because a lookup only proves the master still has
+// the topology, not that a volume server answers.
+func (fs *FilerSink) sourceStillServesChunks() bool {
+	if fs.filerSource == nil {
+		return false
+	}
+	probe := fs.lastServedFileId.Load()
+	if probe == nil {
+		return false
+	}
+	_, _, resp, err := fs.filerSource.ReadPart(*probe, 0)
+	if err != nil {
+		glog.V(0).Infof("source cannot serve %s either, so it is not only the one chunk: %v", *probe, err)
+		return false
+	}
+	util_http.CloseResponse(resp)
+	return true
+}
+
+// hasSourceNewerVersion reports whether the source's current entry for targetPath
+// has moved past sourceMtimeNs — gone, or a strictly-newer mtime — meaning the
+// version being replayed is stale. The lookup runs regardless of sourceMtimeNs so
+// a deleted source is detected even when the replayed mtime is epoch/unset.
+func (fs *FilerSink) hasSourceNewerVersion(targetPath string, sourceMtimeNs int64) bool {
+	sourcePath, ok := fs.targetPathToSourcePath(targetPath)
+	if !ok {
+		return false
+	}
+	return SourceSupersedes(context.Background(), fs.filerSource, sourcePath, sourceMtimeNs)
+}
+
+// SourceSupersedes reports whether the live source's entry at sourcePath has
+// moved past the replayed version (mtimeNs) — deleted or strictly newer — so
+// skipping it is lossless. Exported for filer.backup's non-filer-sink decision.
+func SourceSupersedes(ctx context.Context, filerSource *source.FilerSource, sourcePath util.FullPath, mtimeNs int64) bool {
+	if filerSource == nil {
+		return false
+	}
+	sourceEntry, _, _, err := filer_pb.GetEntry(ctx, filerSource, sourcePath)
+	return sourceSupersedes(sourcePath, sourceEntry, err, mtimeNs)
+}
+
+// sourceSupersedes reports whether the replayed version (sourceMtimeNs) is stale:
+// true if the source entry is gone or strictly newer, false on a same/older version
+// or any non-"not found" lookup error (so a transient failure never skips a live
+// file). GetEntry signals missing as ErrNotFound — possibly wrapped or a plain gRPC
+// string, occasionally a nil entry — all treated as gone.
+func sourceSupersedes(sourcePath util.FullPath, sourceEntry *filer_pb.Entry, lookupErr error, sourceMtimeNs int64) bool {
+	if lookupErr != nil {
+		if errors.Is(lookupErr, filer_pb.ErrNotFound) || strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
+			glog.V(1).Infof("source entry %s no longer exists: %v", sourcePath, lookupErr)
+			return true
+		}
+		glog.V(1).Infof("lookup source entry %s: %v", sourcePath, lookupErr)
+		return false
+	}
+	if sourceEntry == nil {
+		glog.V(1).Infof("source entry %s no longer exists", sourcePath)
+		return true
+	}
+
+	// source still holds the entry; only a strictly-newer mtime proves staleness,
+	// and only when there is a valid replayed mtime to compare against.
+	if sourceMtimeNs <= 0 {
+		return false
+	}
+	return getEntryMtimeNs(sourceEntry) > sourceMtimeNs
+}
+
+func (fs *FilerSink) targetPathToSourcePath(targetPath string) (util.FullPath, bool) {
+	if fs.filerSource == nil {
+		return "", false
+	}
+
+	// Incremental sink keys carry a date prefix (sinkDir/YYYY-MM-DD/relPath) that
+	// can't be reversed to a source path; report unmappable rather than build a path
+	// under a nonexistent dated dir, which would read as ErrNotFound and skip a live entry.
+	if fs.isIncremental {
+		return "", false
+	}
+
+	normalizePath := func(p string) string {
+		p = strings.TrimSuffix(p, "/")
+		if p == "" {
+			return "/"
+		}
+		return p
+	}
+
+	sourceRoot := normalizePath(fs.filerSource.Dir)
+	targetRoot := normalizePath(fs.dir)
+	targetPath = normalizePath(targetPath)
+
+	var relative string
+	switch {
+	case targetRoot == "/":
+		relative = strings.TrimPrefix(targetPath, "/")
+	case targetPath == targetRoot:
+		relative = ""
+	case strings.HasPrefix(targetPath, targetRoot+"/"):
+		relative = targetPath[len(targetRoot)+1:]
+	default:
+		return "", false
+	}
+
+	if relative == "" {
+		return util.FullPath(sourceRoot), true
+	}
+	return util.FullPath(sourceRoot).Child(relative), true
+}
+
+var _ = filer_pb.FilerClient(&FilerSink{})
+
+func (fs *FilerSink) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	return pb.WithGrpcClient(context.Background(), streamingMode, fs.signature, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, false, fs.grpcDialOption)
+
+}
+
+func (fs *FilerSink) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+func (fs *FilerSink) GetDataCenter() string {
+	return fs.dataCenter
+}

weed/replication/sink/filersink/fetch_write_test.go

@@ -0,0 +1,675 @@
+package filersink
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func TestMain(m *testing.M) {
+	util_http.InitGlobalHttpClient()
+	os.Exit(m.Run())
+}
+
+func TestTargetPathToSourcePath(t *testing.T) {
+	tests := []struct {
+		name        string
+		targetRoot  string
+		sourceRoot  string
+		targetPath  string
+		incremental bool
+		wantPath    util.FullPath
+		wantOK      bool
+	}{
+		{
+			name:       "basic mapping",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/target/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			// incremental keys carry a date prefix that can't be reversed; unmappable
+			name:        "incremental sink is unmappable",
+			targetRoot:  "/target",
+			sourceRoot:  "/source",
+			targetPath:  "/target/2026-06-09/path/file.txt",
+			incremental: true,
+			wantPath:    "",
+			wantOK:      false,
+		},
+		{
+			name:       "trailing slash roots",
+			targetRoot: "/target/",
+			sourceRoot: "/source/",
+			targetPath: "/target/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			name:       "root target mapping",
+			targetRoot: "/",
+			sourceRoot: "/source",
+			targetPath: "/path/file.txt",
+			wantPath:   "/source/path/file.txt",
+			wantOK:     true,
+		},
+		{
+			name:       "target root itself",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/target",
+			wantPath:   "/source",
+			wantOK:     true,
+		},
+		{
+			name:       "outside target root",
+			targetRoot: "/target",
+			sourceRoot: "/source",
+			targetPath: "/other/path/file.txt",
+			wantPath:   "",
+			wantOK:     false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fs := &FilerSink{
+				dir:           tc.targetRoot,
+				isIncremental: tc.incremental,
+				filerSource: &source.FilerSource{
+					Dir: tc.sourceRoot,
+				},
+			}
+
+			gotPath, ok := fs.targetPathToSourcePath(tc.targetPath)
+			if ok != tc.wantOK {
+				t.Fatalf("ok mismatch: got %v, want %v", ok, tc.wantOK)
+			}
+			if gotPath != tc.wantPath {
+				t.Fatalf("path mismatch: got %q, want %q", gotPath, tc.wantPath)
+			}
+		})
+	}
+}
+
+// FilerSink must reject chunks whose received byte count disagrees with the
+// source filer metadata, instead of silently writing 0-byte needles with the
+// source size in the destination metadata.
+func TestValidateReplicatedChunkSize(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+
+	tests := []struct {
+		name         string
+		expectedSize uint64
+		readSize     int
+		wantErr      bool
+	}{
+		{
+			name:         "healthy",
+			expectedSize: 5171,
+			readSize:     5171,
+			wantErr:      false,
+		},
+		{
+			name:         "legitimately empty file",
+			expectedSize: 0,
+			readSize:     0,
+			wantErr:      false,
+		},
+		{
+			name:         "zero-byte read for non-empty source",
+			expectedSize: 5171,
+			readSize:     0,
+			wantErr:      true,
+		},
+		{
+			name:         "short read",
+			expectedSize: 5171,
+			readSize:     100,
+			wantErr:      true,
+		},
+		{
+			name:         "over-read (server returned more than metadata)",
+			expectedSize: 5171,
+			readSize:     8192,
+			wantErr:      true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			chunk := &filer_pb.FileChunk{FileId: fid, Size: tc.expectedSize}
+
+			gotErr := validateReplicatedReadSize(chunk, tc.readSize)
+
+			if tc.wantErr {
+				if gotErr == nil {
+					t.Fatalf("expected error, got nil (read=%d expected=%d)",
+						tc.readSize, tc.expectedSize)
+				}
+				if !errors.Is(gotErr, errChunkSizeMismatch) {
+					t.Fatalf("expected errChunkSizeMismatch, got %v", gotErr)
+				}
+				if !strings.Contains(gotErr.Error(), fid) {
+					t.Fatalf("error %q does not mention chunk id %q", gotErr, fid)
+				}
+				return
+			}
+			if gotErr != nil {
+				t.Fatalf("unexpected read-size error: %v", gotErr)
+			}
+		})
+	}
+}
+
+// End-to-end regression :
+// a source volume that responds 200 OK with Content-Length: 0
+// for a chunk that filer metadata claims is 5171 bytes must be rejected
+// by fetchAndWrite with a (non-retriable) size mismatch error,
+// instead of being silently propagated to the destination as a 0-byte needle.
+func TestFetchAndWriteRejectsZeroByteSource(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+	const expectedSize uint64 = 5171
+
+	// Shorten retry backoff so a fail-fast test that briefly enters the retry
+	// loop doesn't pay the production 1s+ wait. Scoped to this test so any
+	// future test in the package keeps the production constant.
+	prevRetryWaitTime := util.RetryWaitTime
+	util.RetryWaitTime = 100 * time.Millisecond
+	t.Cleanup(func() { util.RetryWaitTime = prevRetryWaitTime })
+
+	var hits atomic.Int32
+	sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.WriteHeader(http.StatusOK)
+		// Intentionally write no body — mimic the buggy volume response.
+	}))
+	defer sourceServer.Close()
+
+	serverAddr := strings.TrimPrefix(sourceServer.URL, "http://")
+
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(serverAddr, serverAddr, "/", true); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		address:     serverAddr,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	sourceChunk := &filer_pb.FileChunk{
+		FileId: fid,
+		Size:   expectedSize,
+	}
+
+	done := make(chan struct {
+		fileId string
+		err    error
+	}, 1)
+	go func() {
+		gotFileId, gotErr := fs.fetchAndWrite(sourceChunk, "/dst/index.bin", 0)
+		done <- struct {
+			fileId string
+			err    error
+		}{gotFileId, gotErr}
+	}()
+
+	select {
+	case result := <-done:
+		if result.err == nil {
+			t.Fatalf("expected size mismatch error, got nil (fileId=%q)", result.fileId)
+		}
+		if !errors.Is(result.err, errChunkSizeMismatch) {
+			t.Fatalf("expected errChunkSizeMismatch, got %v", result.err)
+		}
+		if !strings.Contains(result.err.Error(), "5171") {
+			t.Fatalf("error %q does not mention expected size 5171", result.err)
+		}
+		if !strings.Contains(result.err.Error(), fid) {
+			t.Fatalf("error %q does not mention chunk id %q", result.err, fid)
+		}
+		if h := hits.Load(); h != 1 {
+			t.Fatalf("expected exactly 1 source hit (fail-fast), got %d", h)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("fetchAndWrite did not return within 5s (retry loop not aborted on size mismatch); hits=%d", hits.Load())
+	}
+}
+
+type timeoutErr struct{}
+
+func (timeoutErr) Error() string   { return "synthetic timeout" }
+func (timeoutErr) Timeout() bool   { return true }
+func (timeoutErr) Temporary() bool { return true }
+
+// A transient network failure (interrupted read, idle-deadline timeout while
+// the destination reads the upload body, reset/broken pipe) must route through
+// the escalating backoff so an overloaded destination can recover instead of
+// being hammered. The volume server returns its idle timeout as a JSON error
+// string, so the text path matters as much as the net.Error interface.
+func TestIsRetryableNetworkError(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{"nil", nil, false},
+		{"eof", io.EOF, true},
+		{"unexpected eof", io.ErrUnexpectedEOF, true},
+		{"volume idle timeout json", fmt.Errorf("upload result: read tcp 10.0.0.1:8082->10.0.0.1:54848: i/o timeout"), true},
+		{"volume idle timeout capitalized", fmt.Errorf("Upload result: read tcp 10.0.0.1:8082->10.0.0.1:54848: I/O timeout"), true},
+		{"connection reset", fmt.Errorf("upload data: write tcp ...: connection reset by peer"), true},
+		{"connection reset capitalized", fmt.Errorf("Connection reset by peer"), true},
+		{"broken pipe", fmt.Errorf("broken pipe"), true},
+		{"broken pipe capitalized", fmt.Errorf("Broken pipe"), true},
+		{"net.Error timeout", fmt.Errorf("dial: %w", timeoutErr{}), true},
+		{"size mismatch is permanent", errChunkSizeMismatch, false},
+		{"unrelated error", errors.New("not found"), false},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isRetryableNetworkError(tc.err); got != tc.want {
+				t.Fatalf("isRetryableNetworkError(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+// Lock in that the errChunkSizeMismatch sentinel survives the wrap in
+// replicateOneChunk + pass-through in util.Retry, so filer_sink.go's
+// errors.Is check actually fires.
+func TestReplicateChunksPreservesSizeMismatchSentinel(t *testing.T) {
+	const fid = "74,047d16a94aa581"
+	const expectedSize uint64 = 5171
+
+	sourceServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer sourceServer.Close()
+
+	serverAddr := strings.TrimPrefix(sourceServer.URL, "http://")
+
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(serverAddr, serverAddr, "/", true); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		address:     serverAddr,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	sourceChunks := []*filer_pb.FileChunk{{FileId: fid, Size: expectedSize}}
+
+	_, err := fs.replicateChunks(nil, sourceChunks, "/dst/index.bin", 0)
+	if err == nil {
+		t.Fatal("expected error from replicateChunks, got nil")
+	}
+	if !errors.Is(err, errChunkSizeMismatch) {
+		t.Fatalf("error chain broken: errors.Is(err, errChunkSizeMismatch) = false; got %v", err)
+	}
+}
+
+// sourceSupersedes decides whether to skip a stale replayed event. The replayed
+// mtime is fixed; the table varies what the source lookup returned.
+func TestSourceSupersedes(t *testing.T) {
+	const eventNs int64 = 5_000_000_500 // the version being replayed (sec 5, ns 500)
+
+	withMtime := func(sec int64, ns int32) *filer_pb.Entry {
+		return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: sec, MtimeNs: ns}}
+	}
+
+	tests := []struct {
+		name      string
+		entry     *filer_pb.Entry
+		lookupErr error
+		want      bool
+	}{
+		// deleted on source: ErrNotFound in several shapes, all read as gone -> skip
+		{"not-found sentinel", nil, filer_pb.ErrNotFound, true},
+		{"not-found wrapped", nil, fmt.Errorf("lookup /x: %w", filer_pb.ErrNotFound), true},
+		{"not-found as string (gRPC)", nil, errors.New("rpc error: " + filer_pb.ErrNotFound.Error()), true},
+		{"nil entry, nil error", nil, nil, true},
+		// transient lookup failure must NOT skip a possibly-live file
+		{"network error", nil, errors.New("dial tcp: i/o timeout"), false},
+		// live entry: compare full-ns mtime against the replayed version
+		{"source strictly newer", withMtime(5, 600), nil, true},
+		{"source same version", withMtime(5, 500), nil, false},
+		{"source older (out-of-order replay)", withMtime(5, 400), nil, false},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := sourceSupersedes("/source/x/config", tc.entry, tc.lookupErr, eventNs)
+			if got != tc.want {
+				t.Fatalf("sourceSupersedes = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// An epoch/unset replayed mtime (0) must not block "gone" detection: a deleted
+// source still reports superseded so the event is skipped instead of wedging on
+// permanent retries. A live source stays not-superseded — no valid mtime to compare.
+func TestSourceSupersedesEpochMtime(t *testing.T) {
+	live := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 600}}
+	if !sourceSupersedes("/source/x", nil, filer_pb.ErrNotFound, 0) {
+		t.Fatal("epoch-mtime deleted source must be reported gone")
+	}
+	if sourceSupersedes("/source/x", live, nil, 0) {
+		t.Fatal("epoch-mtime live source must not be reported superseded")
+	}
+}
+
+// An incremental sink's dated target keys cannot be mapped back to a source
+// path, so supersession is unverifiable: the gate must stop after the bounded
+// attempts and propagate instead of spinning forever.
+func TestManifestResolveRetryGateUnverifiableSupersessionBounded(t *testing.T) {
+	fs := &FilerSink{isIncremental: true, dir: "/backup"}
+	gate := fs.manifestResolveRetryGate("/backup/2026-07-10/buckets/x/f.pt", 123, "3,01abc", fs.newMissingSourceChunkGate("3,01abc"))
+	resolveErr := errors.New("LookupFileId volume id 3: not found")
+	for i := 1; i < maxUnverifiableResolveAttempts; i++ {
+		if !gate(resolveErr) {
+			t.Fatalf("attempt %d: gate must keep retrying before the bound", i)
+		}
+	}
+	if gate(resolveErr) {
+		t.Error("gate must propagate once the bound is reached with supersession unverifiable")
+	}
+}
+
+// Non-transient resolve errors (corrupt manifest data, bad file ids) must
+// propagate immediately — before any attempt counting or supersession
+// mapping — so the configured metadata error policy applies, instead of
+// retrying until the source is superseded.
+func TestManifestResolveRetryGateNonTransientPropagates(t *testing.T) {
+	fs := &FilerSink{dir: "/backup"}
+	gate := fs.manifestResolveRetryGate("/backup/buckets/x/f.pt", 123, "3,01abc", fs.newMissingSourceChunkGate("3,01abc"))
+	permanentErrs := []error{
+		errors.New("fail to unmarshal manifest 3,01abc: proto: cannot parse invalid wire-format data"),
+		errors.New("invalid fileId abc"),
+	}
+	for _, err := range permanentErrs {
+		if gate(err) {
+			t.Errorf("non-transient error must propagate immediately: %v", err)
+		}
+	}
+	if !gate(errors.New("LookupFileId volume id 3: not found")) {
+		t.Error("transient lookup race must keep retrying on the first attempt")
+	}
+	if isTransientResolveError(nil) {
+		t.Error("isTransientResolveError(nil) must be false")
+	}
+}
+
+// sourceFilerServer answers as a source filer that still holds the entry,
+// unchanged, while its master locates the chunk's volume only for the volume ids
+// in resolvable, which it serves from volumeUrl. With none listed it is a cluster
+// that has vacuumed the volume away — or lost every replica of it.
+type sourceFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	mtime      int64
+	volumeUrl  string
+	resolvable []string
+}
+
+func (s *sourceFilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
+	locationsMap := make(map[string]*filer_pb.Locations)
+	for _, vid := range req.VolumeIds {
+		if slices.Contains(s.resolvable, vid) {
+			locationsMap[vid] = &filer_pb.Locations{
+				Locations: []*filer_pb.Location{{Url: s.volumeUrl}},
+			}
+		}
+	}
+	return &filer_pb.LookupVolumeResponse{LocationsMap: locationsMap}, nil
+}
+
+func (s *sourceFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	return &filer_pb.LookupDirectoryEntryResponse{Entry: &filer_pb.Entry{
+		Name:       req.Name,
+		Attributes: &filer_pb.FuseAttributes{Mtime: s.mtime},
+	}}, nil
+}
+
+// liveVolume serves a chunk read the way a healthy volume server would, so the
+// sink's probe finds the source still producing data.
+func liveVolume(t *testing.T) string {
+	t.Helper()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("chunk bytes"))
+	}))
+	t.Cleanup(server.Close)
+	return strings.TrimPrefix(server.URL, "http://")
+}
+
+func startSourceFiler(t *testing.T, mtime int64, volumeUrl string, resolvable ...string) *source.FilerSource {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	server := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(server, &sourceFilerServer{mtime: mtime, volumeUrl: volumeUrl, resolvable: resolvable})
+	go server.Serve(listener)
+	t.Cleanup(server.Stop)
+
+	address := listener.Addr().String()
+	filerSrc := &source.FilerSource{}
+	if err := filerSrc.DoInitialize(address, address, "/src", false); err != nil {
+		t.Fatalf("filerSource.DoInitialize: %v", err)
+	}
+	filerSrc.SetGrpcDialOption(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	return filerSrc
+}
+
+// A chunk the source cluster cannot produce must stop being retried once the
+// grace period is up. Before, the retry loop ran forever: the sync job never
+// completed, so it held its slot and pinned the offset watermark at the event
+// ahead of it, and filer.sync never checkpointed again.
+func TestFetchAndWriteStopsOnMissingSourceChunk(t *testing.T) {
+	prevRetryWaitTime, prevGrace := util.RetryWaitTime, missingSourceChunkGrace
+	util.RetryWaitTime = 100 * time.Millisecond
+	missingSourceChunkGrace = 500 * time.Millisecond
+	t.Cleanup(func() {
+		util.RetryWaitTime, missingSourceChunkGrace = prevRetryWaitTime, prevGrace
+	})
+
+	filerSrc := startSourceFiler(t, 5, "")
+
+	fs := &FilerSink{
+		filerSource: filerSrc,
+		dir:         "/dst",
+		executor:    util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	done := make(chan error, 1)
+	go func() {
+		_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10}, "/dst/x.bin", 5*int64(time.Second))
+		done <- fetchErr
+	}()
+
+	select {
+	case fetchErr := <-done:
+		if !errors.Is(fetchErr, errSourceChunkMissing) {
+			t.Fatalf("expected errSourceChunkMissing, got %v", fetchErr)
+		}
+		if !errors.Is(fetchErr, source.ErrVolumeNotFound) {
+			t.Fatalf("expected the underlying lookup failure to survive, got %v", fetchErr)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("fetchAndWrite is still retrying a chunk the source cannot produce")
+	}
+}
+
+// The gate waits out the shapes a restarting volume server produces, and only
+// gives up once the source has been saying "gone" for the whole grace period.
+// The wait belongs to the volume: a vacuumed one takes every file it held with
+// it, and waiting it out per file would stall the sync for as long as it holds
+// files.
+func TestMissingSourceChunkGate(t *testing.T) {
+	volumeGone := fmt.Errorf("read part 5617,01abc: %w", source.ErrVolumeNotFound)
+	needleGone := fmt.Errorf("read part 5617,01abc: 404 Not Found: %w", util_http.ErrNotFound)
+
+	fs := &FilerSink{}
+	gate := fs.newMissingSourceChunkGate("5617,01abc")
+	for _, err := range []error{volumeGone, needleGone} {
+		if gate.isPermanent(err) {
+			t.Fatalf("must keep retrying inside the grace period: %v", err)
+		}
+	}
+	if got := gate.wrap(volumeGone); errors.Is(got, errSourceChunkMissing) {
+		t.Fatalf("must not mark permanent while still retrying: %v", got)
+	}
+
+	// an unrelated failure is not the source answering "gone": it must not start a wait
+	other := fs.newMissingSourceChunkGate("5618,01abc")
+	other.isPermanent(errors.New("connection reset by peer"))
+	if _, found := fs.missingVolumes.Load("5618"); found {
+		t.Fatal("a non-missing error must not start the volume's wait")
+	}
+
+	// a second file in the same volume inherits that wait instead of restarting it
+	fs.missingVolumes.Store("5617", time.Now().Add(-2*missingSourceChunkGrace))
+	second := fs.newMissingSourceChunkGate("5617,02def")
+	if !second.isPermanent(volumeGone) {
+		t.Fatal("a later file must inherit the wait its volume already served")
+	}
+	wrapped := second.wrap(volumeGone)
+	if !errors.Is(wrapped, errSourceChunkMissing) || !errors.Is(wrapped, source.ErrVolumeNotFound) {
+		t.Fatalf("wrapped error lost a sentinel: %v", wrapped)
+	}
+	if second.wrap(nil) != nil {
+		t.Fatal("a successful retry must not be turned into an error")
+	}
+
+	// the volume answering again clears the wait it had served
+	fs.sourceServed("5617,09fff")
+	if _, found := fs.missingVolumes.Load("5617"); found {
+		t.Fatal("a served chunk must clear its volume's wait")
+	}
+}
+
+// Once the source has lost a chunk for good, holding the sync offset for its
+// entry only stops every later event from ever being checkpointed: the bytes are
+// not coming back. Skip it loudly instead — but only while the source is
+// demonstrably still serving other chunks, since a cluster that cannot locate
+// anything is having an outage and skipping would drop live files wholesale.
+func TestOnReplicateChunkErrorMissingSourceChunk(t *testing.T) {
+	const probe = "5616,01abc"
+	liveEntry := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5}}
+	missing := fmt.Errorf("copy 5617,02def: %w: %w", errSourceChunkMissing, source.ErrVolumeNotFound)
+
+	t.Run("source still serving", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, liveVolume(t), "5616"), dir: "/dst"}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); err != nil {
+			t.Fatalf("expected the entry to be skipped, got %v", err)
+		}
+	})
+
+	t.Run("source locating nothing", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, ""), dir: "/dst"}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("expected the error to be held for a retry, got %v", err)
+		}
+	})
+
+	t.Run("probe locates but cannot be read", func(t *testing.T) {
+		gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			http.Error(w, "Not Found", http.StatusNotFound)
+		}))
+		defer gone.Close()
+		fs := &FilerSink{
+			filerSource: startSourceFiler(t, 5, strings.TrimPrefix(gone.URL, "http://"), "5616"),
+			dir:         "/dst",
+		}
+		served := probe
+		fs.lastServedFileId.Store(&served)
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("a volume the master lists but no server answers must not license a skip, got %v", err)
+		}
+	})
+
+	t.Run("nothing served yet", func(t *testing.T) {
+		fs := &FilerSink{filerSource: startSourceFiler(t, 5, liveVolume(t), "5616"), dir: "/dst"}
+
+		if err := fs.onReplicateChunkError("/dst/x.bin", liveEntry, missing); !errors.Is(err, errSourceChunkMissing) {
+			t.Fatalf("expected the error to be held with no probe to check, got %v", err)
+		}
+	})
+}
+
+// An incremental sink's dated target keys cannot be mapped back to a source path,
+// so nothing here can tell a vacuumed chunk from a superseded one. Waiting out the
+// grace period would stall every such entry for half an hour; propagate instead
+// and let filer.backup decide with the event's real source key.
+func TestFetchAndWriteMissingSourceChunkUnverifiableSupersession(t *testing.T) {
+	prevRetryWaitTime := util.RetryWaitTime
+	util.RetryWaitTime = 100 * time.Millisecond
+	t.Cleanup(func() { util.RetryWaitTime = prevRetryWaitTime })
+
+	fs := &FilerSink{
+		filerSource:   startSourceFiler(t, 5, ""),
+		dir:           "/backup",
+		isIncremental: true,
+		executor:      util.NewLimitedConcurrentExecutor(1),
+	}
+	fs.SetUploader(operation.NewUploaderWithHttpClient(http.DefaultClient))
+
+	done := make(chan error, 1)
+	go func() {
+		_, fetchErr := fs.fetchAndWrite(&filer_pb.FileChunk{FileId: "5617,01abc", Size: 10},
+			"/backup/2026-07-10/x.bin", 5*int64(time.Second))
+		done <- fetchErr
+	}()
+
+	select {
+	case fetchErr := <-done:
+		if !errors.Is(fetchErr, source.ErrVolumeNotFound) {
+			t.Fatalf("expected the lookup failure to propagate, got %v", fetchErr)
+		}
+		if errors.Is(fetchErr, errSourceChunkMissing) {
+			t.Fatalf("must not write the chunk off without checking supersession: %v", fetchErr)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("fetchAndWrite waited out the grace period with supersession unverifiable")
+	}
+}

weed/replication/sink/filersink/filer_sink.go

@@ -0,0 +1,668 @@
+package filersink
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ChunkTransferSnapshot is a lock-free, copyable view of a chunk transfer's
+// progress, returned by ActiveTransfers.
+type ChunkTransferSnapshot struct {
+	ChunkFileId   string
+	Path          string
+	BytesReceived int64
+	Status        string // "downloading", "uploading", or "waiting 10s" etc.
+	LastErr       string
+}
+
+// ChunkTransferStatus tracks the progress of a single chunk being replicated.
+// Fields are guarded by mu: ChunkFileId and Path are immutable after creation,
+// while BytesReceived, Status, and LastErr are updated by fetchAndWrite and
+// read by ActiveTransfers.
+type ChunkTransferStatus struct {
+	mu sync.RWMutex
+	ChunkTransferSnapshot
+}
+
+type FilerSink struct {
+	filerSource       *source.FilerSource
+	grpcAddress       string
+	dir               string
+	replication       string
+	collection        string
+	ttlSec            int32
+	diskType          string
+	dataCenter        string
+	grpcDialOption    grpc.DialOption
+	address           string
+	writeChunkByFiler bool
+	isIncremental     bool
+	executor          *util.LimitedConcurrentExecutor
+	signature         int32
+	activeTransfers   sync.Map // chunkFileId -> *ChunkTransferStatus
+	uploader          *operation.Uploader
+	// lastServedFileId is the most recent chunk the source did serve, the probe
+	// sourceStillServesChunks re-checks before writing an entry off.
+	lastServedFileId atomic.Pointer[string]
+	missingVolumes   sync.Map // source volume id -> time.Time first found unlocatable
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &FilerSink{})
+}
+
+func (fs *FilerSink) GetName() string {
+	return "filer"
+}
+
+func (fs *FilerSink) GetSinkToDirectory() string {
+	return fs.dir
+}
+
+func (fs *FilerSink) GetDestinationIdentity() string {
+	return fs.grpcAddress + "\x00" + fs.dir
+}
+
+func (fs *FilerSink) IsIncremental() bool {
+	return fs.isIncremental
+}
+
+func (fs *FilerSink) Initialize(configuration util.Configuration, prefix string) error {
+	fs.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	fs.dataCenter = configuration.GetString(prefix + "dataCenter")
+	fs.signature = util.RandomInt32()
+	return fs.DoInitialize(
+		"",
+		configuration.GetString(prefix+"grpcAddress"),
+		configuration.GetString(prefix+"directory"),
+		configuration.GetString(prefix+"replication"),
+		configuration.GetString(prefix+"collection"),
+		configuration.GetInt(prefix+"ttlSec"),
+		configuration.GetString(prefix+"disk"),
+		security.LoadClientTLS(util.GetViper(), "grpc.client"),
+		false)
+}
+
+func (fs *FilerSink) SetSourceFiler(s *source.FilerSource) {
+	fs.filerSource = s
+}
+
+// SetUploader sets a custom uploader for this sink, used when the target
+// cluster requires different TLS certificates than the global config.
+// Must be called during initialization, before any replication goroutines
+// start, since it writes fs.uploader without synchronization.
+func (fs *FilerSink) SetUploader(uploader *operation.Uploader) {
+	fs.uploader = uploader
+}
+
+func (fs *FilerSink) getUploader() (*operation.Uploader, error) {
+	if fs.uploader != nil {
+		return fs.uploader, nil
+	}
+	return operation.NewUploader()
+}
+
+func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
+	replication string, collection string, ttlSec int, diskType string, grpcDialOption grpc.DialOption, writeChunkByFiler bool) (err error) {
+	fs.address = address
+	if fs.address == "" {
+		fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
+	}
+	fs.grpcAddress = grpcAddress
+	fs.dir = dir
+	fs.replication = replication
+	fs.collection = collection
+	fs.ttlSec = int32(ttlSec)
+	fs.diskType = diskType
+	fs.grpcDialOption = grpcDialOption
+	fs.writeChunkByFiler = writeChunkByFiler
+	fs.executor = util.NewLimitedConcurrentExecutor(32)
+	return nil
+}
+
+// SetChunkConcurrency replaces the chunk replication executor with one using the
+// given concurrency limit. Must be called during initialization, before any
+// replication goroutines start, since it replaces fs.executor without
+// synchronization.
+func (fs *FilerSink) SetChunkConcurrency(concurrency int) {
+	if concurrency > 0 {
+		fs.executor = util.NewLimitedConcurrentExecutor(concurrency)
+	}
+}
+
+// ActiveTransfers returns an immutable snapshot of all in-progress chunk transfers.
+func (fs *FilerSink) ActiveTransfers() []ChunkTransferSnapshot {
+	var transfers []ChunkTransferSnapshot
+	fs.activeTransfers.Range(func(key, value any) bool {
+		t := value.(*ChunkTransferStatus)
+		t.mu.RLock()
+		transfers = append(transfers, t.ChunkTransferSnapshot)
+		t.mu.RUnlock()
+		return true
+	})
+	return transfers
+}
+
+func (fs *FilerSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	dir, name := util.FullPath(key).DirAndName()
+
+	glog.V(4).Infof("delete entry: %v", key)
+	err := filer_pb.Remove(context.Background(), fs, dir, name, deleteIncludeChunks, true, true, true, signatures)
+	if err != nil {
+		glog.V(0).Infof("delete entry %s: %v", key, err)
+		return fmt.Errorf("delete entry %s: %v", key, err)
+	}
+	return nil
+}
+
+var _ sink.EntryMover = (*FilerSink)(nil)
+
+// MoveEntry relocates oldKey to newKey on the target filer via AtomicRenameEntry:
+// a metadata-only move that relocates a whole subtree in one transaction, so a
+// directory rename never leaves descendants missing and chunks are neither
+// re-copied nor leaked.
+//
+// When the move fails because the old path is genuinely gone on the sink — a
+// descendant the parent rename already relocated, or one never replicated —
+// there is nothing to move, so it creates the new path instead (CreateEntry
+// short-circuits when the entry is already there, and never deletes). Existence
+// is re-checked with a direct lookup rather than inferred from the rename error,
+// so a rolled-back move that left the old entry intact propagates for retry
+// instead of being mistaken for "gone".
+func (fs *FilerSink) MoveEntry(oldKey, newKey string, newEntry *filer_pb.Entry, signatures []int32) error {
+	oldDir, oldName := util.FullPath(oldKey).DirAndName()
+	newDir, newName := util.FullPath(newKey).DirAndName()
+
+	err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, err := client.AtomicRenameEntry(context.Background(), &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: oldDir,
+			OldName:      oldName,
+			NewDirectory: newDir,
+			NewName:      newName,
+			Signatures:   signatures,
+		})
+		return err
+	})
+	if err == nil {
+		return nil
+	}
+	if missing, lookupErr := fs.entryMissing(oldKey); lookupErr == nil && missing {
+		glog.V(2).Infof("move %s => %s: old path gone, creating %s", oldKey, newKey, newKey)
+		return fs.CreateEntry(newKey, newEntry, signatures)
+	}
+	return fmt.Errorf("move %s => %s: %w", oldKey, newKey, err)
+}
+
+// entryMissing reports whether key has no entry on the target filer. A lookup
+// not-found (sentinel or the gRPC string form) means missing; any other lookup
+// error is returned so the caller does not treat an unknown state as missing.
+func (fs *FilerSink) entryMissing(key string) (bool, error) {
+	dir, name := util.FullPath(key).DirAndName()
+	missing := false
+	err := fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		})
+		if lookupErr == nil {
+			return nil
+		}
+		// The string check is a deliberate compatibility fallback: a cross-cluster
+		// gRPC error often arrives as a plain status string that no longer wraps the
+		// filer_pb.ErrNotFound sentinel, so errors.Is alone would miss it.
+		if errors.Is(lookupErr, filer_pb.ErrNotFound) || strings.Contains(lookupErr.Error(), filer_pb.ErrNotFound.Error()) {
+			missing = true
+			return nil
+		}
+		return lookupErr
+	})
+	return missing, err
+}
+
+func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+
+	return fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		dir, name := util.FullPath(key).DirAndName()
+
+		// look up existing entry
+		lookupRequest := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+		// glog.V(1).Infof("lookup: %v", lookupRequest)
+		if resp, err := filer_pb.LookupEntry(context.Background(), client, lookupRequest); err == nil {
+			if destinationSatisfiesSource(resp.Entry, entry) {
+				glog.V(3).Infof("skip overwriting %s", key)
+				return nil
+			}
+			glog.Warningf("re-replicating %s: destination not current (dst %d chunks, src %d chunks)",
+				key, len(resp.Entry.GetChunks()), len(entry.GetChunks()))
+		}
+
+		replicatedChunks, err := fs.replicateChunks(context.Background(), entry.GetChunks(), key, getEntryMtimeNs(entry))
+
+		if err != nil {
+			// don't swallow: committing mismatched bytes would propagate corruption
+			if errors.Is(err, errChunkSizeMismatch) {
+				return fs.onCorruptChunk(key, entry, err)
+			}
+			return fs.onReplicateChunkError(key, entry, err)
+		}
+
+		// glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks)
+
+		request := &filer_pb.CreateEntryRequest{
+			Directory: dir,
+			Entry: &filer_pb.Entry{
+				Name:        name,
+				IsDirectory: entry.IsDirectory,
+				Attributes:  entry.Attributes,
+				Extended:    entry.Extended,
+				Chunks:      replicatedChunks,
+				Content:     entry.Content,
+				RemoteEntry: entry.RemoteEntry,
+			},
+			IsFromOtherCluster: true,
+			Signatures:         signatures,
+		}
+
+		glog.V(3).Infof("create: %v", request)
+		if err := filer_pb.CreateEntry(context.Background(), client, request); err != nil {
+			glog.V(0).Infof("create entry %s: %v", key, err)
+			return fmt.Errorf("create entry %s: %v", key, err)
+		}
+
+		return nil
+	})
+}
+
+func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+
+	dir, name := util.FullPath(key).DirAndName()
+
+	// read existing entry
+	var existingEntry *filer_pb.Entry
+	err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		}
+
+		glog.V(4).Infof("lookup entry: %v", request)
+		resp, err := filer_pb.LookupEntry(context.Background(), client, request)
+		if err != nil {
+			glog.V(0).Infof("lookup %s: %v", key, err)
+			return err
+		}
+
+		existingEntry = resp.Entry
+
+		return nil
+	})
+
+	if err != nil {
+		return false, fmt.Errorf("lookup %s: %v", key, err)
+	}
+
+	glog.V(4).Infof("oldEntry %+v, newEntry %+v, existingEntry: %+v", oldEntry, newEntry, existingEntry)
+
+	// the supersession checks below must look up where the incoming entry lives
+	// now, which for a rename is newParentPath/newEntry.Name, not the old key.
+	targetKey := updatedEntryKey(key, newParentPath, newEntry)
+
+	switch chooseUpdateAction(existingEntry, oldEntry, newEntry) {
+	case updateSkip:
+		// a newer, complete version already landed (out-of-order); leave it
+		glog.V(2).Infof("late updates %s", key)
+		return true, nil
+	case updateRepair:
+		glog.Warningf("repair %s: destination diverged from source (dst %d chunks, src %d chunks); re-replicating full source content",
+			key, len(existingEntry.GetChunks()), len(newEntry.GetChunks()))
+		replicatedChunks, err := fs.replicateChunks(context.Background(), newEntry.GetChunks(), targetKey, getEntryMtimeNs(newEntry))
+		if err != nil {
+			if errors.Is(err, errChunkSizeMismatch) {
+				return true, fs.onCorruptChunk(targetKey, newEntry, err)
+			}
+			return true, fs.onReplicateChunkError(targetKey, newEntry, err)
+		}
+		existingEntry.Chunks = replicatedChunks
+		existingEntry.Attributes = newEntry.Attributes
+		existingEntry.Extended = newEntry.Extended
+		existingEntry.HardLinkId = newEntry.HardLinkId
+		existingEntry.HardLinkCounter = newEntry.HardLinkCounter
+		existingEntry.Content = newEntry.Content
+		existingEntry.RemoteEntry = newEntry.RemoteEntry
+	default:
+		// source-side chunks resolve via source filer; sink volume IDs may collide.
+		deletedChunks, newChunks, err := compareChunks(context.Background(), filer.LookupFn(fs.filerSource), oldEntry, newEntry)
+		if err != nil {
+			return true, fmt.Errorf("replicate %s compare chunks error: %v", key, err)
+		}
+
+		// delete the chunks that are deleted from the source
+		if deleteIncludeChunks {
+			// remove the deleted chunks. Actual data deletion happens in filer UpdateEntry FindUnusedFileChunks
+			existingEntry.Chunks = filer.DoMinusChunksBySourceFileId(existingEntry.GetChunks(), deletedChunks)
+		}
+
+		// replicate the chunks that are new in the source
+		replicatedChunks, err := fs.replicateChunks(context.Background(), newChunks, targetKey, getEntryMtimeNs(newEntry))
+		if err != nil {
+			if errors.Is(err, errChunkSizeMismatch) {
+				return true, fs.onCorruptChunk(targetKey, newEntry, err)
+			}
+			return true, fs.onReplicateChunkError(targetKey, newEntry, err)
+		}
+		existingEntry.Chunks = append(existingEntry.GetChunks(), replicatedChunks...)
+		existingEntry.Attributes = newEntry.Attributes
+		existingEntry.Extended = newEntry.Extended
+		existingEntry.HardLinkId = newEntry.HardLinkId
+		existingEntry.HardLinkCounter = newEntry.HardLinkCounter
+		existingEntry.Content = newEntry.Content
+		existingEntry.RemoteEntry = newEntry.RemoteEntry
+	}
+
+	// save updated meta data
+	return true, fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.UpdateEntryRequest{
+			Directory:          newParentPath,
+			Entry:              existingEntry,
+			IsFromOtherCluster: true,
+			Signatures:         signatures,
+		}
+
+		if _, err := client.UpdateEntry(context.Background(), request); err != nil {
+			return fmt.Errorf("update existingEntry %s: %v", key, err)
+		}
+
+		return nil
+	})
+
+}
+func compareChunks(ctx context.Context, lookupFileIdFn wdclient.LookupFileIdFunctionType, oldEntry, newEntry *filer_pb.Entry) (deletedChunks, newChunks []*filer_pb.FileChunk, err error) {
+	aData, aMeta, aErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, oldEntry.GetChunks(), 0, math.MaxInt64, nil)
+	if aErr != nil {
+		return nil, nil, aErr
+	}
+	bData, bMeta, bErr := filer.ResolveChunkManifest(ctx, lookupFileIdFn, newEntry.GetChunks(), 0, math.MaxInt64, nil)
+	if bErr != nil {
+		return nil, nil, bErr
+	}
+
+	deletedChunks = append(deletedChunks, filer.DoMinusChunks(aData, bData)...)
+	deletedChunks = append(deletedChunks, filer.DoMinusChunks(aMeta, bMeta)...)
+
+	newChunks = append(newChunks, filer.DoMinusChunks(bData, aData)...)
+	newChunks = append(newChunks, filer.DoMinusChunks(bMeta, aMeta)...)
+
+	return
+}
+
+// getEntryMtimeNs returns the mtime at full nanosecond precision so versions
+// rewritten within the same second can be ordered. int64 ns is safe until ~2262.
+func getEntryMtimeNs(entry *filer_pb.Entry) int64 {
+	if entry == nil || entry.Attributes == nil {
+		return 0
+	}
+	return entry.Attributes.Mtime*int64(1e9) + int64(entry.Attributes.MtimeNs)
+}
+
+// EntryMtimeNs exposes getEntryMtimeNs for filer.backup's supersession check.
+func EntryMtimeNs(entry *filer_pb.Entry) int64 {
+	return getEntryMtimeNs(entry)
+}
+
+// onCorruptChunk handles a permanent chunk size-mismatch (fetched source bytes
+// disagree with source metadata). If the source already holds a newer version the
+// event is a stale replay whose chunk was overwritten and GC'd — skip it (lossless:
+// meta events are full snapshots, so a later event re-carries the current chunks).
+// Otherwise the live version is corrupt, so return the error and halt loudly.
+func (fs *FilerSink) onCorruptChunk(key string, entry *filer_pb.Entry, err error) error {
+	if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
+		glog.Warningf("skip stale entry %s with superseded corrupt chunk: %v", key, err)
+		return nil
+	}
+	glog.Errorf("refuse to replicate entry with corrupt chunk %s: %v", key, err)
+	return err
+}
+
+// onReplicateChunkError handles a non-size-mismatch replicateChunks failure.
+// Skip (return nil) when the live source has moved past this version — deleted
+// or strictly-newer mtime — so a later event carries the current content and the
+// skip is lossless, or when the source has lost the chunk for good: it can never
+// serve those bytes to anyone, and holding the offset for it only stops every
+// later event from ever being checkpointed. Otherwise propagate so the offset
+// stays put and the event is retried; returning nil would silently drop the file.
+func (fs *FilerSink) onReplicateChunkError(key string, entry *filer_pb.Entry, err error) error {
+	if fs.hasSourceNewerVersion(key, getEntryMtimeNs(entry)) {
+		glog.Warningf("skip stale entry %s, source superseded during replicate: %v", key, err)
+		return nil
+	}
+	if errors.Is(err, errSourceChunkMissing) && fs.sourceStillServesChunks() {
+		glog.Errorf("skip %s: the source cluster cannot produce its data, so it stays unreplicated: %v", key, err)
+		return nil
+	}
+	return fmt.Errorf("replicate entry chunks %s: %w", key, err)
+}
+
+// updateAction decides how UpdateEntry reconciles an incoming source version with
+// the destination's current entry.
+type updateAction int
+
+const (
+	// updateNormal applies the source diff: destination at or behind the source.
+	updateNormal updateAction = iota
+	// updateSkip leaves the destination untouched: a newer complete version already landed.
+	updateSkip
+	// updateRepair re-replicates full source over a destination strictly shorter than
+	// the source despite a newer mtime — a truncated copy from a failed replication.
+	updateRepair
+)
+
+// updatedEntryKey returns the sink path the incoming entry lands at: for a rename
+// newParentPath/newEntry.Name, else key's parent + name. Supersession must use this,
+// not the pre-rename key, which would look deleted on the source and skip a live event.
+func updatedEntryKey(key, newParentPath string, newEntry *filer_pb.Entry) string {
+	targetDir := newParentPath
+	if targetDir == "" {
+		targetDir, _ = util.FullPath(key).DirAndName()
+	}
+	return string(util.NewFullPath(targetDir, newEntry.Name))
+}
+
+func chooseUpdateAction(existing, oldEntry, incoming *filer_pb.Entry) updateAction {
+	exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
+	if exNs > inNs {
+		// Destination is strictly newer: incoming is a stale/out-of-order replay of
+		// an older source version, so never roll the destination back to it — even
+		// when the newer destination is smaller (a legitimate truncate). A newer copy
+		// that is itself incomplete is reconciled by -initialSnapshot, not by an
+		// older event.
+		return updateSkip
+	}
+	// Destination at or behind incoming. updateNormal applies the incremental
+	// oldEntry->incoming diff, valid only when the destination already holds
+	// oldEntry's chunks. A cheap, allocation-free positional identity match proves
+	// that for the common (in-sync) path; only when it fails — divergence, chunk
+	// reordering, or an out-of-band destination — do we run the precise O(n log n)
+	// checks.
+	if existing != nil && oldEntry != nil && !destinationMatchesReference(existing, oldEntry) {
+		// Truncated/gapped: the destination does not cover every byte range oldEntry
+		// holds. Range containment (not a scalar byte count, attr FileSize, or ETag)
+		// catches it for any destination, and is safe at equal mtime since a complete
+		// destination always covers oldEntry.
+		if !coversReference(existing, oldEntry) {
+			return updateRepair
+		}
+		// Stale superseded chunk at full coverage: a chunk this backup wrote whose
+		// source oldEntry no longer references. Restricted to backup-written chunks
+		// (out-of-band rsync/direct chunks are ignored) AND to a strictly-older
+		// destination — at equal mtime same-second versions cannot be ordered, so
+		// defer rather than risk a rollback (reliable ordering is a separate fix).
+		if exNs < inNs && hasStaleBackupChunk(existing, oldEntry) {
+			return updateRepair
+		}
+	}
+	return updateNormal
+}
+
+// destinationSatisfiesSource reports whether an existing destination entry already
+// satisfies the incoming source entry, so CreateEntry can skip the write. Coverage
+// is checked first: a chunk-truncated destination (covering fewer bytes than the
+// source) must be re-replicated even when its attr.Md5-backed ETag still matches
+// the source, so ETag equality must never bypass the coverage check. A destination
+// that is at least as complete is skippable when it has identical content (ETag) or
+// is a newer-or-equal version.
+func destinationSatisfiesSource(existing, incoming *filer_pb.Entry) bool {
+	if existing == nil {
+		return false
+	}
+	exNs, inNs := getEntryMtimeNs(existing), getEntryMtimeNs(incoming)
+	if exNs > inNs {
+		return true // destination strictly newer: incoming is older → keep destination
+	}
+	if !destinationMatchesReference(existing, incoming) {
+		if !coversReference(existing, incoming) {
+			return false // missing a byte range the source covers (truncated/gapped)
+		}
+		if exNs < inNs && hasStaleBackupChunk(existing, incoming) {
+			return false // a provably-older destination holds a superseded backup chunk
+		}
+	}
+	if filer.ETag(existing) == filer.ETag(incoming) {
+		return true // identical content
+	}
+	return exNs >= inNs // equal mtime → keep (deferred); strictly older → re-replicate
+}
+
+// destinationMatchesReference is a cheap, allocation-free sufficient check that the
+// destination already holds exactly the reference's chunks: equal count and, in
+// order, each destination chunk was replicated from the corresponding reference
+// chunk (SourceFileId == reference FileId) at the same byte range (Offset/Size).
+// True ⇒ full coverage and no stale chunk, so updateNormal/skip is safe and the
+// O(n log n) range and identity checks can be skipped — the common in-sync path.
+// False is never a false positive: a reordered, diverged, or out-of-band
+// (rsync/direct) destination just falls back to the precise checks.
+func destinationMatchesReference(existing, reference *filer_pb.Entry) bool {
+	ex, ref := existing.GetChunks(), reference.GetChunks()
+	if len(ex) != len(ref) {
+		return false
+	}
+	for i := range ref {
+		if ex[i].SourceFileId == "" || ex[i].SourceFileId != ref[i].GetFileIdString() {
+			return false
+		}
+		// Require the same byte range too. replicateOneChunk copies the source
+		// chunk's Offset/Size verbatim, so SourceFileId identity already implies an
+		// identical range today; verifying it here keeps this fast path sound even
+		// if replication ever re-chunks (split/coalesce), instead of silently
+		// depending on that invariant from another file.
+		if ex[i].Offset != ref[i].Offset || ex[i].Size != ref[i].Size {
+			return false
+		}
+	}
+	return true
+}
+
+// hasStaleBackupChunk reports whether existing holds a chunk that this backup
+// wrote (SourceFileId set) whose source chunk `reference` no longer references.
+// That means an overwrite/deletion event was missed: the destination still carries
+// the superseded source chunk's bytes even though total coverage looks full, so the
+// incremental diff cannot fix it. Chunks without SourceFileId (seeded out-of-band,
+// e.g. rsync or a direct write) are ignored, so such destinations keep their normal
+// path and are never re-copied on this signal.
+func hasStaleBackupChunk(existing, reference *filer_pb.Entry) bool {
+	if existing == nil || reference == nil {
+		return false
+	}
+	refFids := make(map[string]bool, len(reference.GetChunks()))
+	for _, c := range reference.GetChunks() {
+		refFids[c.GetFileIdString()] = true
+	}
+	for _, c := range existing.GetChunks() {
+		if c.SourceFileId == "" {
+			continue // not backup-written → do not judge staleness
+		}
+		if !refFids[c.SourceFileId] {
+			return true
+		}
+	}
+	return false
+}
+
+// mergedIntervals returns the entry's chunk byte ranges, sorted and merged
+// (overlapping or touching ranges combined). Chunk manifests are counted by their
+// own offset/size span, which already covers the range they wrap, so no manifest
+// resolution is needed.
+func mergedIntervals(entry *filer_pb.Entry) [][2]int64 {
+	if entry == nil {
+		return nil
+	}
+	chunks := entry.GetChunks()
+	if len(chunks) == 0 {
+		return nil
+	}
+	ivs := make([][2]int64, 0, len(chunks))
+	for _, c := range chunks {
+		ivs = append(ivs, [2]int64{int64(c.Offset), int64(c.Offset) + int64(c.Size)})
+	}
+	sort.Slice(ivs, func(i, j int) bool { return ivs[i][0] < ivs[j][0] })
+	merged := make([][2]int64, 0, len(ivs))
+	for _, iv := range ivs {
+		if n := len(merged); n > 0 && iv[0] <= merged[n-1][1] {
+			if iv[1] > merged[n-1][1] {
+				merged[n-1][1] = iv[1]
+			}
+			continue
+		}
+		merged = append(merged, iv)
+	}
+	return merged
+}
+
+// coversReference reports whether existing's chunks cover every byte range that
+// reference's chunks cover. Equal total coverage is not sufficient proof — extra
+// chunks at other offsets can hide a missing range — so containment is verified
+// range by range against the merged interval sets.
+func coversReference(existing, reference *filer_pb.Entry) bool {
+	ex := mergedIntervals(existing)
+	i := 0
+	for _, r := range mergedIntervals(reference) {
+		lo, hi := r[0], r[1]
+		for lo < hi {
+			for i < len(ex) && ex[i][1] <= lo {
+				i++
+			}
+			if i >= len(ex) || ex[i][0] > lo {
+				return false // a byte in reference is not covered by existing
+			}
+			lo = ex[i][1]
+		}
+	}
+	return true
+}

weed/replication/sink/filersink/filer_sink_test.go

@@ -0,0 +1,342 @@
+package filersink
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// entry builds a file entry with the given mtime and size for the update-action
+// decision, which only looks at those two attributes.
+func entry(mtime int64, size uint64) *filer_pb.Entry {
+	return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: size}}
+}
+
+// entryNs adds a sub-second component to check same-second ordering.
+func entryNs(mtime int64, ns int32, size uint64) *filer_pb.Entry {
+	return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, MtimeNs: ns, FileSize: size}}
+}
+
+// entryChunks builds an entry with nChunks contiguous 1-byte chunks at a given
+// mtime, so coveredBytes == nChunks. Different counts give different coverage,
+// which is what the truncation-repair routing keys on.
+func entryChunks(mtime int64, nChunks int) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: uint64(nChunks)}}
+	for i := range nChunks {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{
+			FileId: "3," + string(rune('a'+i)), Offset: int64(i), Size: 1,
+		})
+	}
+	return e
+}
+
+// entryMd5 is like entryChunks but also sets attr.Md5, so filer.ETag returns the
+// stored Md5 (the S3 PutObject case) instead of a chunk-derived ETag. Used to show
+// that coverage — not the Md5-backed ETag — drives the truncation decision.
+func entryMd5(mtime int64, nChunks int, md5 []byte) *filer_pb.Entry {
+	e := entryChunks(mtime, nChunks)
+	e.Attributes.Md5 = md5
+	return e
+}
+
+// srcEntry builds a source-side entry at mtime whose chunks carry the given
+// FileIds — the identities a backup destination's SourceFileId is matched against.
+func srcEntry(mtime int64, fids ...string) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for i, f := range fids {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{FileId: f, Offset: int64(i), Size: 1})
+	}
+	return e
+}
+
+// dstEntry builds a destination entry at mtime. An empty src marks a chunk
+// written out-of-band (no SourceFileId, e.g. rsync/direct); otherwise it is a
+// backup chunk copied from that source FileId.
+func dstEntry(mtime int64, srcFids ...string) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for i, s := range srcFids {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{FileId: "dst", SourceFileId: s, Offset: int64(i), Size: 1})
+	}
+	return e
+}
+
+// ivEntry builds an entry at mtime with chunks at the given [offset,end) byte
+// ranges — for coverage/containment tests where exact ranges matter.
+func ivEntry(mtime int64, ivs ...[2]int64) *filer_pb.Entry {
+	e := &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: mtime}}
+	for _, iv := range ivs {
+		e.Chunks = append(e.Chunks, &filer_pb.FileChunk{Offset: iv[0], Size: uint64(iv[1] - iv[0])})
+	}
+	return e
+}
+
+// getEntryMtimeNs must order versions written within the same second, which
+// plain second-grained mtime cannot. It also must be nil-safe.
+func TestGetEntryMtimeNs(t *testing.T) {
+	if got := getEntryMtimeNs(nil); got != 0 {
+		t.Fatalf("nil entry: got %d, want 0", got)
+	}
+	if got := getEntryMtimeNs(&filer_pb.Entry{}); got != 0 {
+		t.Fatalf("nil attributes: got %d, want 0", got)
+	}
+
+	mk := func(sec int64, ns int32) *filer_pb.Entry {
+		return &filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: sec, MtimeNs: ns}}
+	}
+	if got, want := getEntryMtimeNs(mk(5, 200)), int64(5_000_000_200); got != want {
+		t.Fatalf("full-ns: got %d, want %d", got, want)
+	}
+	// same second, later sub-second component must compare strictly greater
+	if !(getEntryMtimeNs(mk(5, 200)) > getEntryMtimeNs(mk(5, 100))) {
+		t.Fatalf("same-second ordering failed: %d not > %d",
+			getEntryMtimeNs(mk(5, 200)), getEntryMtimeNs(mk(5, 100)))
+	}
+}
+
+// onCorruptChunk must surface the error, not skip, when it can't confirm
+// supersession (no source filer here) — otherwise it could drop a live file.
+func TestOnCorruptChunkRefusesWhenSupersessionUnconfirmed(t *testing.T) {
+	fs := &FilerSink{} // filerSource nil => hasSourceNewerVersion is always false
+	sentinel := errors.New("corrupt chunk: read 0 bytes, source says 107")
+
+	got := fs.onCorruptChunk("/buckets/x/config",
+		&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 200}},
+		sentinel)
+
+	if !errors.Is(got, sentinel) {
+		t.Fatalf("expected the error to be returned (refuse), got %v", got)
+	}
+}
+
+// onReplicateChunkError must NOT swallow a transient/partial replicate failure
+// (a volume-id lookup race or unretried manifest-chunk resolve during a write
+// burst) when it cannot confirm the source has moved on. Here filerSource is
+// nil, so hasSourceNewerVersion is always false — the "supersession unconfirmed"
+// case. Returning nil would let the metadata subscription advance its offset
+// past an event whose file was never written, silently dropping it (the
+// filer.backup partial-landing bug). It must propagate so the offset stays put
+// and the event is reprocessed. Genuine source-404 (gone/superseded) skipping is
+// covered by TestSourceSupersedes, which gates the return-nil branch.
+func TestOnReplicateChunkErrorPropagatesWhenSupersessionUnconfirmed(t *testing.T) {
+	fs := &FilerSink{} // filerSource nil => hasSourceNewerVersion always false
+	sentinel := errors.New("replicate manifest data chunks 7,02def: volume id 7 not found")
+
+	got := fs.onReplicateChunkError("/buckets/x/model.pt",
+		&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 5, MtimeNs: 200}},
+		sentinel)
+
+	if !errors.Is(got, sentinel) {
+		t.Fatalf("expected the transient replicate error to be propagated (not swallowed), got %v", got)
+	}
+}
+
+// chooseUpdateAction decides skip vs repair vs normal. A destination left
+// truncated by an earlier failed replication can carry a newer mtime (the
+// source preserved an old mtime while the partial copy was written recently),
+// so a pure mtime guard would strand the corruption. A shorter destination with
+// a newer mtime must be repaired, not skipped.
+func TestChooseUpdateAction(t *testing.T) {
+	tests := []struct {
+		name     string
+		existing *filer_pb.Entry
+		oldEntry *filer_pb.Entry
+		incoming *filer_pb.Entry
+		want     updateAction
+	}{
+		{"nil existing entry", nil, entry(100, 100), entry(200, 100), updateNormal},
+		{"destination older, catching up", entry(100, 50), entry(100, 50), entry(200, 100), updateNormal},
+		{"same mtime", entry(200, 50), entry(200, 50), entry(200, 100), updateNormal},
+		{"destination newer and complete", entry(300, 100), entry(100, 100), entry(200, 100), updateSkip},
+		{"destination newer and larger", entry(300, 200), entry(100, 100), entry(200, 100), updateSkip},
+		// out-of-order: a newer (smaller) destination must NOT be rolled back by an
+		// older, larger replayed event — even though it does not cover oldEntry.
+		{"newer smaller dest, older larger event -> skip", ivEntry(300, [2]int64{0, 5}), ivEntry(100, [2]int64{0, 10}), ivEntry(100, [2]int64{0, 10}), updateSkip},
+		// same second: sub-second ordering must still pick the winner
+		{"same second, destination newer and complete", entryNs(5, 200, 100), entryNs(5, 100, 100), entryNs(5, 100, 100), updateSkip},
+		{"same second, destination older", entryNs(5, 100, 100), entryNs(5, 100, 100), entryNs(5, 200, 100), updateNormal},
+		// chunk-incompleteness (the truncation bug): destination is at/behind the
+		// incoming mtime but covers fewer bytes than oldEntry, so the incremental
+		// diff cannot heal it — must repair in full, not append. entryChunks(n) has
+		// n contiguous 1-byte chunks → coverage n.
+		{"behind, covers less than oldEntry -> repair", entryChunks(200, 2), entryChunks(200, 4), entryChunks(200, 4), updateRepair},
+		{"behind, matches oldEntry coverage -> normal", entryChunks(200, 3), entryChunks(200, 3), entryChunks(200, 5), updateNormal},
+		{"behind, exceeds oldEntry coverage -> normal", entryChunks(200, 5), entryChunks(200, 3), entryChunks(200, 5), updateNormal},
+		// range containment, not scalar coverage: equal total bytes but a missing
+		// range (the destination has an extra range at another offset) → repair.
+		{"behind, equal total but range gap -> repair", ivEntry(200, [2]int64{0, 10}, [2]int64{30, 40}), ivEntry(200, [2]int64{0, 20}), ivEntry(200, [2]int64{0, 20}), updateRepair},
+		// same-range stale: full coverage but a backup-written chunk whose source
+		// oldEntry no longer references → repair only when the destination is
+		// provably older. Equal-mtime is deferred (cannot order same-second
+		// versions). Out-of-band (rsync) chunks are never stale-checked.
+		{"older, stale backup chunk -> repair", dstEntry(100, "A", "X"), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateRepair},
+		{"equal-mtime stale -> deferred (normal)", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+		{"older, rsync chunks -> normal", dstEntry(100, "", ""), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+		// in-sync fast path: destination holds exactly oldEntry's chunks → normal
+		{"in-sync (fast path) -> normal", dstEntry(100, "A", "B"), srcEntry(100, "A", "B"), srcEntry(200, "A", "B"), updateNormal},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := chooseUpdateAction(tc.existing, tc.oldEntry, tc.incoming); got != tc.want {
+				t.Fatalf("chooseUpdateAction = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// coversReference must verify range containment, not just equal total bytes: a
+// destination with the same byte count but a missing range (extra chunk elsewhere)
+// does not cover the reference.
+func TestCoversReference(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"empty reference", ivEntry(0), ivEntry(0), true},
+		{"reference no chunks", ivEntry(0, [2]int64{0, 10}), ivEntry(0), true},
+		{"exact", ivEntry(0, [2]int64{0, 10}), ivEntry(0, [2]int64{0, 10}), true},
+		{"superset", ivEntry(0, [2]int64{0, 20}), ivEntry(0, [2]int64{0, 10}), true},
+		{"truncated tail", ivEntry(0, [2]int64{0, 5}), ivEntry(0, [2]int64{0, 10}), false},
+		{"equal total, range gap", ivEntry(0, [2]int64{0, 10}, [2]int64{30, 40}), ivEntry(0, [2]int64{0, 20}), false},
+		{"contiguous parts cover", ivEntry(0, [2]int64{0, 10}, [2]int64{10, 20}), ivEntry(0, [2]int64{0, 20}), true},
+		{"overlapping parts cover", ivEntry(0, [2]int64{0, 12}, [2]int64{8, 20}), ivEntry(0, [2]int64{0, 20}), true},
+		{"existing empty, reference has bytes", ivEntry(0), ivEntry(0, [2]int64{0, 1}), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := coversReference(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("coversReference = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// destinationSatisfiesSource gates CreateEntry's skip. Coverage/identity must be
+// checked before ETag: a truncated or stale destination whose attr.Md5 still
+// matches the source (so filer.ETag is equal) must NOT be skipped, or
+// -initialSnapshot would leave it permanently diverged.
+func TestDestinationSatisfiesSource(t *testing.T) {
+	md5A := []byte("0123456789abcdef")
+	md5B := []byte("fedcba9876543210")
+	cases := []struct {
+		name               string
+		existing, incoming *filer_pb.Entry
+		want               bool
+	}{
+		{"nil existing", nil, entryChunks(200, 4), false},
+		{"identical complete -> skip", entryChunks(200, 4), entryChunks(200, 4), true},
+		{"truncated (no md5) -> replicate", entryChunks(200, 2), entryChunks(200, 4), false},
+		// regression: same attr.Md5 (equal filer.ETag) but fewer chunks must repair
+		{"md5-backed truncated -> replicate", entryMd5(200, 2, md5A), entryMd5(200, 4, md5A), false},
+		{"md5 equal and complete -> skip", entryMd5(200, 4, md5A), entryMd5(200, 4, md5A), true},
+		// complete but different content: keep newer, replace older
+		{"complete, dest newer -> skip", entryMd5(300, 4, md5A), entryMd5(200, 4, md5B), true},
+		{"complete, dest older -> replicate", entryMd5(200, 4, md5A), entryMd5(300, 4, md5B), false},
+		// same-range stale backup chunk: replicate only when provably older; an
+		// equal-mtime stale dest is deferred (skip), and rsync chunks are not
+		// stale-checked.
+		{"older, stale backup chunk -> replicate", dstEntry(100, "A", "X"), srcEntry(200, "A", "B"), false},
+		{"equal-mtime stale -> skip (deferred)", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), true},
+		{"rsync complete -> skip", dstEntry(200, "", ""), srcEntry(200, "A", "B"), true},
+		{"in-sync (fast path) -> skip", dstEntry(200, "A", "B"), srcEntry(200, "A", "B"), true},
+		// out-of-order: newer (smaller) destination must not be rolled back by an
+		// older, larger source replay.
+		{"newer smaller dest, older larger source -> skip", ivEntry(300, [2]int64{0, 5}), ivEntry(100, [2]int64{0, 10}), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := destinationSatisfiesSource(tc.existing, tc.incoming); got != tc.want {
+				t.Fatalf("destinationSatisfiesSource = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// destinationMatchesReference is the allocation-free fast path: true only when the
+// destination holds exactly the reference's chunks in order (by SourceFileId). It
+// must be conservative — never a false positive — so reorders, count mismatches,
+// and out-of-band (no SourceFileId) chunks all return false.
+func TestDestinationMatchesReference(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"both empty", dstEntry(0), srcEntry(0), true},
+		{"exact in order", dstEntry(0, "A", "B"), srcEntry(0, "A", "B"), true},
+		{"count mismatch", dstEntry(0, "A"), srcEntry(0, "A", "B"), false},
+		{"reordered", dstEntry(0, "B", "A"), srcEntry(0, "A", "B"), false},
+		{"stale identity", dstEntry(0, "A", "X"), srcEntry(0, "A", "B"), false},
+		{"out-of-band chunk", dstEntry(0, "A", ""), srcEntry(0, "A", "B"), false},
+		// Same identity but a different byte range must not fast-path as a match,
+		// so the precise checks still run if replication ever re-chunks.
+		{"shifted offset, same identity",
+			&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 0}, Chunks: []*filer_pb.FileChunk{
+				{FileId: "dst", SourceFileId: "A", Offset: 0, Size: 1},
+				{FileId: "dst", SourceFileId: "B", Offset: 5, Size: 1},
+			}}, srcEntry(0, "A", "B"), false},
+		{"larger size, same identity",
+			&filer_pb.Entry{Attributes: &filer_pb.FuseAttributes{Mtime: 0}, Chunks: []*filer_pb.FileChunk{
+				{FileId: "dst", SourceFileId: "A", Offset: 0, Size: 2},
+			}}, srcEntry(0, "A"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := destinationMatchesReference(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("destinationMatchesReference = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// hasStaleBackupChunk flags a destination that still holds a backup-written chunk
+// (SourceFileId set) the source reference no longer lists — a missed
+// overwrite/deletion. Out-of-band chunks (no SourceFileId, e.g. rsync) are ignored.
+func TestHasStaleBackupChunk(t *testing.T) {
+	cases := []struct {
+		name                string
+		existing, reference *filer_pb.Entry
+		want                bool
+	}{
+		{"nil existing", nil, srcEntry(200, "A"), false},
+		{"in sync", dstEntry(200, "A", "B"), srcEntry(200, "A", "B"), false},
+		{"stale backup chunk", dstEntry(200, "A", "X"), srcEntry(200, "A", "B"), true},
+		{"rsync chunks ignored", dstEntry(200, "", ""), srcEntry(200, "A", "B"), false},
+		{"synced backup + rsync", dstEntry(200, "A", ""), srcEntry(200, "A", "B"), false},
+		{"stale backup among rsync", dstEntry(200, "X", ""), srcEntry(200, "A", "B"), true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := hasStaleBackupChunk(tc.existing, tc.reference); got != tc.want {
+				t.Fatalf("hasStaleBackupChunk = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
+
+// updatedEntryKey must resolve the incoming entry's new path. For a rename the
+// supersession check has to target newParentPath/newEntry.Name, not the old key,
+// or the renamed-away old path looks deleted on the source and skips a live event.
+func TestUpdatedEntryKey(t *testing.T) {
+	named := func(name string) *filer_pb.Entry { return &filer_pb.Entry{Name: name} }
+	tests := []struct {
+		name          string
+		key           string
+		newParentPath string
+		entry         *filer_pb.Entry
+		want          string
+	}{
+		{"content update, no new parent", "/dst/a.txt", "", named("a.txt"), "/dst/a.txt"},
+		{"rename same dir", "/dst/old.txt", "/dst", named("new.txt"), "/dst/new.txt"},
+		{"rename to subdir", "/dst/old.txt", "/dst/sub", named("new.txt"), "/dst/sub/new.txt"},
+		{"root parent", "/old.txt", "/", named("new.txt"), "/new.txt"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := updatedEntryKey(tc.key, tc.newParentPath, tc.entry); got != tc.want {
+				t.Fatalf("updatedEntryKey = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}

weed/replication/sink/gcssink/gcs_sink.go

@@ -0,0 +1,158 @@
+package gcssink
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/option"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type GcsSink struct {
+	client        *storage.Client
+	bucket        string
+	dir           string
+	filerSource   *source.FilerSource
+	isIncremental bool
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &GcsSink{})
+}
+
+func (g *GcsSink) GetName() string {
+	return "google_cloud_storage"
+}
+
+func (g *GcsSink) GetSinkToDirectory() string {
+	return g.dir
+}
+
+func (g *GcsSink) GetDestinationIdentity() string {
+	return g.bucket + "\x00" + g.dir
+}
+
+func (g *GcsSink) IsIncremental() bool {
+	return g.isIncremental
+}
+
+func (g *GcsSink) Initialize(configuration util.Configuration, prefix string) error {
+	g.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	return g.initialize(
+		configuration.GetString(prefix+"google_application_credentials"),
+		configuration.GetString(prefix+"bucket"),
+		configuration.GetString(prefix+"directory"),
+	)
+}
+
+func (g *GcsSink) SetSourceFiler(s *source.FilerSource) {
+	g.filerSource = s
+}
+
+func (g *GcsSink) initialize(google_application_credentials, bucketName, dir string) error {
+	g.bucket = bucketName
+	g.dir = dir
+
+	// Creates a client.
+	var clientOpts []option.ClientOption
+	if google_application_credentials != "" {
+		var data []byte
+		var err error
+		if strings.HasPrefix(google_application_credentials, "{") {
+			data = []byte(google_application_credentials)
+		} else {
+			googleCredentialsPath := util.ResolvePath(google_application_credentials)
+			data, err = os.ReadFile(googleCredentialsPath)
+			if err != nil {
+				return fmt.Errorf("failed to read credentials file %s: %v", googleCredentialsPath, err)
+			}
+		}
+		creds, err := google.CredentialsFromJSON(context.Background(), data, storage.ScopeFullControl)
+		if err != nil {
+			return fmt.Errorf("failed to parse credentials: %v", err)
+		}
+		httpClient := oauth2.NewClient(context.Background(), creds.TokenSource)
+		clientOpts = append(clientOpts, option.WithHTTPClient(httpClient), option.WithoutAuthentication())
+	}
+	client, err := storage.NewClient(context.Background(), clientOpts...)
+	if err != nil {
+		return fmt.Errorf("failed to create client with credentials \"%s\" env \"%s\": %v",
+			google_application_credentials, os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"), err)
+	}
+
+	g.client = client
+
+	return nil
+}
+
+func (g *GcsSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	if isDirectory {
+		key = key + "/"
+	}
+
+	if err := g.client.Bucket(g.bucket).Object(key).Delete(context.Background()); err != nil {
+		return fmt.Errorf("gcs delete %s/%s: %v", g.bucket, key, err)
+	}
+
+	return nil
+
+}
+
+func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	totalSize := filer.FileSize(entry)
+	chunkViews := filer.ViewFromChunks(context.Background(), g.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	wc := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
+
+	writeFunc := func(data []byte) error {
+		_, writeErr := wc.Write(data)
+		return writeErr
+	}
+
+	var writeErr error
+	if len(entry.Content) > 0 {
+		content, decErr := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if decErr != nil {
+			writeErr = fmt.Errorf("decrypt inline SSE content: %w", decErr)
+		} else {
+			writeErr = writeFunc(content)
+		}
+	} else {
+		writeErr = repl_util.CopyFromChunkViews(chunkViews, g.filerSource, writeFunc, entry)
+	}
+
+	if writeErr != nil {
+		// Cancel the context to abort the GCS upload without touching
+		// any existing object at this key.
+		cancel()
+		wc.Close()
+		return writeErr
+	}
+
+	return wc.Close()
+
+}
+
+func (g *GcsSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+	return true, g.CreateEntry(key, newEntry, signatures)
+}

weed/replication/sink/localsink/local_incremental_sink.go

@@ -0,0 +1,17 @@
+package localsink
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+)
+
+type LocalIncSink struct {
+	LocalSink
+}
+
+func (localincsink *LocalIncSink) GetName() string {
+	return "local_incremental"
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &LocalIncSink{})
+}

weed/replication/sink/localsink/local_sink.go

@@ -0,0 +1,169 @@
+package localsink
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type LocalSink struct {
+	Dir           string
+	filerSource   *source.FilerSource
+	isIncremental bool
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &LocalSink{})
+}
+
+func (localsink *LocalSink) SetSourceFiler(s *source.FilerSource) {
+	localsink.filerSource = s
+}
+
+func (localsink *LocalSink) GetName() string {
+	return "local"
+}
+
+func (localsink *LocalSink) isMultiPartEntry(key string) bool {
+	return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/")
+}
+
+func (localsink *LocalSink) initialize(dir string, isIncremental bool) error {
+	localsink.Dir = dir
+	localsink.isIncremental = isIncremental
+	return nil
+}
+
+func (localsink *LocalSink) Initialize(configuration util.Configuration, prefix string) error {
+	dir := configuration.GetString(prefix + "directory")
+	isIncremental := configuration.GetBool(prefix + "is_incremental")
+	glog.V(4).Infof("sink.local.directory: %v", dir)
+	return localsink.initialize(dir, isIncremental)
+}
+
+func (localsink *LocalSink) GetSinkToDirectory() string {
+	return localsink.Dir
+}
+
+func (localsink *LocalSink) GetDestinationIdentity() string {
+	return localsink.Dir
+}
+
+func (localsink *LocalSink) IsIncremental() bool {
+	return localsink.isIncremental
+}
+
+func (localsink *LocalSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+	key = sanitizeFsKey(key)
+	if localsink.isMultiPartEntry(key) {
+		return nil
+	}
+	glog.V(4).Infof("Delete Entry key: %s", key)
+	if err := os.Remove(key); err != nil {
+		glog.V(0).Infof("remove entry key %s: %s", key, err)
+	}
+	return nil
+}
+
+func (localsink *LocalSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
+	key = sanitizeFsKey(key)
+	if entry.IsDirectory || localsink.isMultiPartEntry(key) {
+		return nil
+	}
+	glog.V(4).Infof("Create Entry key: %s", key)
+
+	totalSize := filer.FileSize(entry)
+	chunkViews := filer.ViewFromChunks(context.Background(), localsink.filerSource.LookupFileId, entry.GetChunks(), 0, int64(totalSize))
+
+	dir := filepath.Dir(key)
+
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		glog.V(4).Infof("Create Directory key: %s", dir)
+		if err = os.MkdirAll(dir, 0755); err != nil {
+			return err
+		}
+	}
+
+	if entry.IsDirectory {
+		return os.Mkdir(key, os.FileMode(entry.Attributes.FileMode))
+	}
+
+	mode := os.FileMode(entry.Attributes.FileMode)
+	shortFileName := util.ToShortFileName(key)
+
+	// Write to a temp file in the same directory, then atomically rename
+	// on success. This prevents leaving a truncated/empty file if
+	// decryption or chunk copy fails.
+	tmpFile, err := os.CreateTemp(dir, ".seaweedfs-tmp-*")
+	if err != nil {
+		return err
+	}
+	tmpName := tmpFile.Name()
+	defer func() {
+		// Clean up temp file on any error (rename removes it on success)
+		if tmpFile != nil {
+			tmpFile.Close()
+			os.Remove(tmpName)
+		}
+	}()
+
+	if err := tmpFile.Chmod(mode); err != nil {
+		return err
+	}
+
+	writeFunc := func(data []byte) error {
+		_, writeErr := tmpFile.Write(data)
+		return writeErr
+	}
+
+	if len(entry.Content) > 0 {
+		content, err := repl_util.MaybeDecryptContent(entry.Content, entry)
+		if err != nil {
+			return fmt.Errorf("decrypt inline SSE content: %w", err)
+		}
+		if err := writeFunc(content); err != nil {
+			return err
+		}
+	} else {
+		if err := repl_util.CopyFromChunkViews(chunkViews, localsink.filerSource, writeFunc, entry); err != nil {
+			return err
+		}
+	}
+
+	// Close before rename so the data is flushed
+	if err := tmpFile.Close(); err != nil {
+		return err
+	}
+	tmpFile = nil // prevent deferred cleanup
+
+	// Atomic rename into final destination
+	if err := os.Rename(tmpName, shortFileName); err != nil {
+		os.Remove(tmpName)
+		return err
+	}
+
+	return nil
+}
+
+func (localsink *LocalSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+	key = sanitizeFsKey(key)
+	if localsink.isMultiPartEntry(key) {
+		return true, nil
+	}
+	glog.V(4).Infof("Update Entry key: %s", key)
+	// do delete and create
+	foundExistingEntry = util.FileExists(key)
+	err = localsink.CreateEntry(key, newEntry, signatures)
+	return
+}

weed/replication/sink/localsink/local_sink_std.go

@@ -0,0 +1,9 @@
+//go:build !windows
+
+package localsink
+
+// sanitizeFsKey is a no-op on non-Windows filesystems. The Windows build
+// strips characters that are illegal in NTFS paths.
+func sanitizeFsKey(key string) string {
+	return key
+}

weed/replication/sink/localsink/local_sink_std_test.go

@@ -0,0 +1,12 @@
+//go:build !windows
+
+package localsink
+
+import "testing"
+
+func TestSanitizeFsKeyKeepsColons(t *testing.T) {
+	key := "/backup/a:b/c:d.txt"
+	if got := sanitizeFsKey(key); got != key {
+		t.Errorf("sanitizeFsKey(%q) = %q, want identity", key, got)
+	}
+}

weed/replication/sink/localsink/local_sink_test.go

@@ -0,0 +1,74 @@
+package localsink
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+)
+
+// TestCreateEntry_OverwriteReadOnlyFile reproduces a bug where
+// filer.backup receives a create event (0-byte, mode 0400) followed by
+// an update event (with chunks) for the same file. The update event
+// calls CreateEntry again, which fails with "permission denied" because
+// OpenFile with O_RDWR cannot open the 0400 file created by the first
+// event.
+func TestCreateEntry_OverwriteReadOnlyFile(t *testing.T) {
+	tmpDir := t.TempDir()
+	sink := &LocalSink{}
+	sink.initialize(tmpDir, false)
+	sink.SetSourceFiler(&source.FilerSource{})
+
+	key := filepath.Join(tmpDir, "objects", "5c", "9fb207")
+
+	// Create event: 0-byte file with mode 0400 (metadata only, no chunks)
+	createEntry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{
+			FileMode: uint32(0400),
+		},
+	}
+	if err := os.MkdirAll(filepath.Dir(key), 0755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+	if err := sink.CreateEntry(key, createEntry, nil); err != nil {
+		t.Fatalf("CreateEntry (create event) failed: %v", err)
+	}
+
+	// Verify: 0-byte, 0400 file exists
+	fi, err := os.Stat(key)
+	if err != nil {
+		t.Fatalf("stat after create event: %v", err)
+	}
+	if fi.Size() != 0 {
+		t.Errorf("expected 0 bytes after create event, got %d", fi.Size())
+	}
+	if fi.Mode().Perm() != 0400 {
+		t.Fatalf("expected 0400 after create event, got %o", fi.Mode().Perm())
+	}
+
+	// Update event: same file, now with content (simulated by entry.Content)
+	updateEntry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{
+			FileMode: uint32(0400),
+			FileSize: 552,
+		},
+		Content: []byte("git object data placeholder"),
+	}
+	if err := sink.CreateEntry(key, updateEntry, nil); err != nil {
+		t.Fatalf("CreateEntry (update event) failed on read-only file: %v", err)
+	}
+
+	// Verify: file has content and correct permissions
+	fi, err = os.Stat(key)
+	if err != nil {
+		t.Fatalf("stat after update event: %v", err)
+	}
+	if fi.Size() == 0 {
+		t.Errorf("expected non-zero size after update event, got 0")
+	}
+	if fi.Mode().Perm() != 0400 {
+		t.Errorf("expected 0400 after update event, got %o", fi.Mode().Perm())
+	}
+}

weed/replication/sink/localsink/local_sink_windows.go

@@ -0,0 +1,21 @@
+//go:build windows
+
+package localsink
+
+import "strings"
+
+// sanitizeFsKey strips characters illegal in NTFS paths (notably ':') so the
+// local filesystem sink can write the key. A leading drive-letter colon (e.g.
+// "C:\...") is preserved so absolute target paths stay valid. Only the local
+// sink needs this; other sinks accept the raw key.
+func sanitizeFsKey(key string) string {
+	prefix, rest := "", key
+	if len(key) >= 2 && key[1] == ':' && isDriveLetter(key[0]) {
+		prefix, rest = key[:2], key[2:]
+	}
+	return prefix + strings.ReplaceAll(rest, ":", "")
+}
+
+func isDriveLetter(b byte) bool {
+	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')
+}

weed/replication/sink/localsink/local_sink_windows_test.go

@@ -0,0 +1,65 @@
+//go:build windows
+
+package localsink
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+)
+
+func TestSanitizeFsKeyStripsColons(t *testing.T) {
+	if got := sanitizeFsKey("/backup/a:b/c:d.txt"); got != "/backup/ab/cd.txt" {
+		t.Errorf("sanitizeFsKey() = %q, want %q", got, "/backup/ab/cd.txt")
+	}
+	if got := sanitizeFsKey(`C:\a:b`); got != `C:\ab` {
+		t.Errorf("sanitizeFsKey() = %q, want drive letter preserved %q", got, `C:\ab`)
+	}
+}
+
+// TestCreateEntryWritesSanitizedPath confirms CreateEntry strips colons from
+// the destination path before writing, so keys land at NTFS-legal paths.
+func TestCreateEntryWritesSanitizedPath(t *testing.T) {
+	tmpDir := t.TempDir()
+	sink := &LocalSink{}
+	sink.initialize(tmpDir, false)
+	sink.SetSourceFiler(&source.FilerSource{})
+
+	key := filepath.Join(tmpDir, "20:24", "fi:le.txt")
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{FileMode: uint32(0644)},
+		Content:    []byte("data"),
+	}
+	if err := sink.CreateEntry(key, entry, nil); err != nil {
+		t.Fatalf("CreateEntry failed: %v", err)
+	}
+
+	want := sanitizeFsKey(key)
+	if _, err := os.Stat(want); err != nil {
+		t.Errorf("expected file at sanitized path %q: %v", want, err)
+	}
+	if _, err := os.Stat(key); err == nil {
+		t.Errorf("unexpected file at unsanitized path %q", key)
+	}
+
+	updated := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{FileMode: uint32(0644)},
+		Content:    []byte("data-v2"),
+	}
+	if _, err := sink.UpdateEntry(key, entry, filepath.Dir(key), updated, false, nil); err != nil {
+		t.Fatalf("UpdateEntry failed: %v", err)
+	}
+	if _, err := os.Stat(want); err != nil {
+		t.Errorf("expected updated file at sanitized path %q: %v", want, err)
+	}
+
+	if err := sink.DeleteEntry(key, false, false, nil); err != nil {
+		t.Fatalf("DeleteEntry failed: %v", err)
+	}
+	if _, err := os.Stat(want); !os.IsNotExist(err) {
+		t.Errorf("expected sanitized path %q removed, stat err=%v", want, err)
+	}
+}

weed/replication/sink/replication_sink.go

@@ -0,0 +1,37 @@
+package sink
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type ReplicationSink interface {
+	GetName() string
+	Initialize(configuration util.Configuration, prefix string) error
+	DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error
+	CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error
+	UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error)
+	GetSinkToDirectory() string
+	// GetDestinationIdentity distinguishes this sink's write destination from
+	// any other destination the same sink type could write to: endpoint or
+	// account, bucket or container, and directory. filer.backup keys its resume
+	// checkpoint on it, so two configurations writing to different places must
+	// not share a value.
+	GetDestinationIdentity() string
+	SetSourceFiler(s *source.FilerSource)
+	IsIncremental() bool
+}
+
+// EntryMover is an optional capability for sinks that can relocate an entry
+// natively, in one atomic step, instead of create-then-delete. Drivers prefer
+// it for a rename so a failed copy can never leave the source deleted with no
+// committed destination, a directory move never deletes descendants before they
+// are recreated, and the entry's chunks are neither re-copied nor leaked.
+type EntryMover interface {
+	MoveEntry(oldKey, newKey string, newEntry *filer_pb.Entry, signatures []int32) error
+}
+
+var (
+	Sinks []ReplicationSink
+)

weed/replication/sink/s3sink/s3_sink.go

@@ -0,0 +1,258 @@
+package S3Sink
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/replication/repl_util"
+	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
+	"github.com/seaweedfs/seaweedfs/weed/replication/source"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type S3Sink struct {
+	conn                          s3iface.S3API
+	filerSource                   *source.FilerSource
+	isIncremental                 bool
+	keepPartSize                  bool
+	s3DisableContentMD5Validation bool
+	s3ForcePathStyle              bool
+	uploaderConcurrency           int
+	uploaderMaxUploadParts        int
+	uploaderPartSizeMb            int
+	region                        string
+	bucket                        string
+	dir                           string
+	endpoint                      string
+	acl                           string
+}
+
+func init() {
+	sink.Sinks = append(sink.Sinks, &S3Sink{})
+}
+
+func (s3sink *S3Sink) GetName() string {
+	return "s3"
+}
+
+func (s3sink *S3Sink) GetSinkToDirectory() string {
+	return s3sink.dir
+}
+
+func (s3sink *S3Sink) GetDestinationIdentity() string {
+	return s3sink.endpoint + "\x00" + s3sink.bucket + "\x00" + s3sink.dir
+}
+
+func (s3sink *S3Sink) IsIncremental() bool {
+	return s3sink.isIncremental
+}
+
+func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string) error {
+	configuration.SetDefault(prefix+"region", "us-east-2")
+	configuration.SetDefault(prefix+"directory", "/")
+	configuration.SetDefault(prefix+"keep_part_size", true)
+	configuration.SetDefault(prefix+"uploader_max_upload_parts", 1000)
+	configuration.SetDefault(prefix+"uploader_part_size_mb", 8)
+	configuration.SetDefault(prefix+"uploader_concurrency", 8)
+	configuration.SetDefault(prefix+"s3_disable_content_md5_validation", true)
+	configuration.SetDefault(prefix+"s3_force_path_style", true)
+	s3sink.region = configuration.GetString(prefix + "region")
+	s3sink.bucket = configuration.GetString(prefix + "bucket")
+	s3sink.dir = configuration.GetString(prefix + "directory")
+	s3sink.endpoint = configuration.GetString(prefix + "endpoint")
+	s3sink.acl = configuration.GetString(prefix + "acl")
+	s3sink.isIncremental = configuration.GetBool(prefix + "is_incremental")
+	s3sink.keepPartSize = configuration.GetBool(prefix + "keep_part_size")
+	s3sink.s3DisableContentMD5Validation = configuration.GetBool(prefix + "s3_disable_content_md5_validation")
+	s3sink.s3ForcePathStyle = configuration.GetBool(prefix + "s3_force_path_style")
+	s3sink.uploaderMaxUploadParts = configuration.GetInt(prefix + "uploader_max_upload_parts")
+	s3sink.uploaderPartSizeMb = configuration.GetInt(prefix + "uploader_part_size")
+	s3sink.uploaderConcurrency = configuration.GetInt(prefix + "uploader_concurrency")
+
+	glog.V(0).Infof("sink.s3.region: %v", s3sink.region)
+	glog.V(0).Infof("sink.s3.bucket: %v", s3sink.bucket)
+	glog.V(0).Infof("sink.s3.directory: %v", s3sink.dir)
+	glog.V(0).Infof("sink.s3.endpoint: %v", s3sink.endpoint)
+	glog.V(0).Infof("sink.s3.acl: %v", s3sink.acl)
+	glog.V(0).Infof("sink.s3.is_incremental: %v", s3sink.isIncremental)
+	glog.V(0).Infof("sink.s3.s3_disable_content_md5_validation: %v", s3sink.s3DisableContentMD5Validation)
+	glog.V(0).Infof("sink.s3.s3_force_path_style: %v", s3sink.s3ForcePathStyle)
+	glog.V(0).Infof("sink.s3.keep_part_size: %v", s3sink.keepPartSize)
+	if s3sink.uploaderMaxUploadParts > s3manager.MaxUploadParts {
+		s3sink.uploaderMaxUploadParts = s3manager.MaxUploadParts
+		glog.Warningf("uploader_max_upload_parts is greater than the maximum number of parts allowed when uploading multiple parts to Amazon S3")
+		glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v => %v", s3sink.uploaderMaxUploadParts, s3manager.MaxUploadParts)
+	} else {
+		glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v", s3sink.uploaderMaxUploadParts)
+	}
+	glog.V(0).Infof("sink.s3.uploader_part_size_mb: %v", s3sink.uploaderPartSizeMb)
+	glog.V(0).Infof("sink.s3.uploader_concurrency: %v", s3sink.uploaderConcurrency)
+
+	return s3sink.initialize(
+		configuration.GetString(prefix+"aws_access_key_id"),
+		configuration.GetString(prefix+"aws_secret_access_key"),
+	)
+}
+
+func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) {
+	s3sink.filerSource = s
+}
+
+func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey string) error {
+	config := &aws.Config{
+		Region:                        aws.String(s3sink.region),
+		Endpoint:                      aws.String(s3sink.endpoint),
+		S3DisableContentMD5Validation: aws.Bool(s3sink.s3DisableContentMD5Validation),
+		S3ForcePathStyle:              aws.Bool(s3sink.s3ForcePathStyle),
+	}
+	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("create aws session: %w", err)
+	}
+	s3sink.conn = s3.New(sess)
+
+	return nil
+}
+
+func (s3sink *S3Sink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
+
+	key = cleanKey(key)
+
+	if isDirectory {
+		return nil
+	}
+
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(s3sink.bucket),
+		Key:    aws.String(key),
+	}
+
+	result, err := s3sink.conn.DeleteObject(input)
+
+	if err == nil {
+		glog.V(2).Infof("[%s] delete %s: %v", s3sink.bucket, key, result)
+	} else {
+		glog.Errorf("[%s] delete %s: %v", s3sink.bucket, key, err)
+	}
+
+	return err
+
+}
+
+func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) (err error) {
+	key = cleanKey(key)
+
+	if entry.IsDirectory {
+		return nil
+	}
+
+	reader := filer.NewFileReader(s3sink.filerSource, entry)
+
+	// Decrypt SSE-encrypted objects so the destination receives plaintext
+	decryptedReader, err := repl_util.MaybeDecryptReader(reader, entry)
+	if err != nil {
+		repl_util.CloseReader(reader)
+		return fmt.Errorf("decrypt SSE object: %w", err)
+	}
+	defer repl_util.CloseMaybeDecryptedReader(reader, decryptedReader)
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploaderWithClient(s3sink.conn, func(u *s3manager.Uploader) {
+		u.PartSize = int64(s3sink.uploaderPartSizeMb * 1024 * 1024)
+		u.Concurrency = s3sink.uploaderConcurrency
+		u.MaxUploadParts = s3sink.uploaderMaxUploadParts
+	})
+
+	if s3sink.keepPartSize {
+		switch chunkCount := len(entry.Chunks); {
+		case chunkCount > 1:
+			if firstChunkSize := int64(entry.Chunks[0].Size); firstChunkSize > s3manager.MinUploadPartSize {
+				uploader.PartSize = firstChunkSize
+			}
+		default:
+			uploader.PartSize = 0
+		}
+	}
+
+	doSaveMtime := true
+	if entry.Extended == nil {
+		entry.Extended = make(map[string][]byte)
+	} else if _, ok := entry.Extended[s3_constants.AmzUserMetaMtime]; ok {
+		doSaveMtime = false
+	}
+	if doSaveMtime {
+		entry.Extended[s3_constants.AmzUserMetaMtime] = []byte(strconv.FormatInt(entry.Attributes.Mtime, 10))
+	}
+	// process tagging
+	tags := buildTaggingString(entry.Extended)
+
+	// Upload the file to S3.
+	uploadInput := s3manager.UploadInput{
+		Bucket: aws.String(s3sink.bucket),
+		Key:    aws.String(key),
+		Body:   decryptedReader,
+	}
+	if tags != "" {
+		uploadInput.Tagging = aws.String(tags)
+	}
+	if entry.Attributes.Mime != "" {
+		uploadInput.ContentType = aws.String(entry.Attributes.Mime)
+	}
+	if len(entry.Attributes.Md5) > 0 {
+		uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5)))
+	}
+	_, err = uploader.Upload(&uploadInput)
+	if err != nil {
+		if sourceErr := filer.ReaderSourceError(reader); sourceErr != nil {
+			return fmt.Errorf("read source %s: %w", key, sourceErr)
+		}
+	}
+
+	return err
+
+}
+
+func (s3sink *S3Sink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (foundExistingEntry bool, err error) {
+	key = cleanKey(key)
+	return true, s3sink.CreateEntry(key, newEntry, signatures)
+}
+
+func cleanKey(key string) string {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+	return key
+}
+
+// buildTaggingString builds the S3 Tagging header value from entry extended metadata.
+// Only keys with the AmzObjectTaggingPrefix ("X-Amz-Tagging-") are included as object
+// tags. The prefix is stripped and values are URL-encoded to produce a valid S3 tagging
+// query string.
+func buildTaggingString(extended map[string][]byte) string {
+	tagValues := url.Values{}
+	for k, v := range extended {
+		if strings.HasPrefix(k, s3_constants.AmzObjectTaggingPrefix) {
+			tagKey := k[len(s3_constants.AmzObjectTaggingPrefix):]
+			tagValues.Set(tagKey, string(v))
+		}
+	}
+	return tagValues.Encode()
+}

weed/replication/sink/s3sink/s3_sink_test.go

@@ -0,0 +1,133 @@
+package S3Sink
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildTaggingString_ShouldStripTagPrefix(t *testing.T) {
+	extended := map[string][]byte{
+		s3_constants.AmzObjectTaggingPrefix + "env": []byte("production"),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	if strings.Contains(tagging, s3_constants.AmzObjectTaggingPrefix) {
+		t.Errorf("tagging should not contain storage prefix %q, got %q", s3_constants.AmzObjectTaggingPrefix, tagging)
+	}
+
+	parsed, err := url.ParseQuery(tagging)
+	if err != nil {
+		t.Fatalf("tagging should be valid URL query: %v", err)
+	}
+	if v := parsed.Get("env"); v != "production" {
+		t.Errorf("expected tag env=production, got %q", v)
+	}
+}
+
+func TestBuildTaggingString_ShouldURLEncodeValues(t *testing.T) {
+	extended := map[string][]byte{
+		s3_constants.AmzObjectTaggingPrefix + "path": []byte("/a/b=c&d"),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	parsed, err := url.ParseQuery(tagging)
+	if err != nil {
+		t.Fatalf("tagging should be valid URL query: %v", err)
+	}
+	if v := parsed.Get("path"); v != "/a/b=c&d" {
+		t.Errorf("expected tag value /a/b=c&d after decoding, got %q", v)
+	}
+}
+
+// capturePutRoundTripper records the s3manager.Uploader's PUT and replies 200
+// so the SDK is satisfied without making a real network call.
+type capturePutRoundTripper struct {
+	uploadReq *http.Request
+}
+
+func (c *capturePutRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Method == http.MethodPut {
+		c.uploadReq = req.Clone(req.Context())
+	}
+	if req.Body != nil {
+		_, _ = io.Copy(io.Discard, req.Body)
+		_ = req.Body.Close()
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Header:     http.Header{"ETag": []string{"\"etag\""}},
+		Request:    req,
+	}, nil
+}
+
+func newCapturingS3Sink(t *testing.T) (*S3Sink, *capturePutRoundTripper) {
+	t.Helper()
+	rt := &capturePutRoundTripper{}
+	sess, err := session.NewSession(&aws.Config{
+		Region:           aws.String("us-east-1"),
+		Endpoint:         aws.String("https://example.invalid"),
+		S3ForcePathStyle: aws.Bool(true),
+		Credentials:      credentials.NewStaticCredentials("k", "s", ""),
+		HTTPClient:       &http.Client{Transport: rt},
+	})
+	require.NoError(t, err)
+	s := &S3Sink{
+		conn:                awss3.New(sess),
+		bucket:              "bucket",
+		uploaderConcurrency: 1,
+		uploaderPartSizeMb:  5,
+	}
+	return s, rt
+}
+
+func TestS3SinkCreateEntryPassesMimeAsContentType(t *testing.T) {
+	sink, rt := newCapturingS3Sink(t)
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{Mime: "text/html"},
+		Content:    []byte("<html></html>"),
+	}
+	require.NoError(t, sink.CreateEntry("/dir/test.html", entry, nil))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	require.Equal(t, "text/html", rt.uploadReq.Header.Get("Content-Type"))
+}
+
+func TestS3SinkCreateEntryOmitsContentTypeWhenMimeMissing(t *testing.T) {
+	sink, rt := newCapturingS3Sink(t)
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{},
+		Content:    []byte("data"),
+	}
+	require.NoError(t, sink.CreateEntry("/dir/test.bin", entry, nil))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	require.Equal(t, "", rt.uploadReq.Header.Get("Content-Type"))
+}
+
+func TestBuildTaggingString_EmptyWhenNoTags(t *testing.T) {
+	extended := map[string][]byte{
+		"Content-Encoding":             []byte("gzip"),
+		s3_constants.AmzUserMetaMtime:  []byte("12345"),
+		s3_constants.SeaweedFSSSES3Key: []byte(`{"algorithm":"AES256","encryptedDEK":"abc"}`),
+	}
+
+	tagging := buildTaggingString(extended)
+
+	if tagging != "" {
+		t.Errorf("expected empty tagging when no tag keys present, got %q", tagging)
+	}
+}

weed/replication/source/filer_source.go

@@ -0,0 +1,203 @@
+package source
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+// ErrVolumeNotFound reports that the source cluster has no location for a
+// chunk's volume: vacuumed away, deleted, or every replica offline. Callers
+// need it apart from a lookup that simply failed, which a later attempt can
+// still get past.
+var ErrVolumeNotFound = errors.New("volume not found")
+
+type FilerSource struct {
+	grpcAddress    string
+	grpcDialOption grpc.DialOption
+	Dir            string
+	address        string
+	proxyByFiler   bool
+	dataCenter     string
+	signature      int32
+	httpClient     *util_http_client.HTTPClient
+}
+
+func (fs *FilerSource) Initialize(configuration util.Configuration, prefix string) error {
+	fs.dataCenter = configuration.GetString(prefix + "dataCenter")
+	fs.signature = util.RandomInt32()
+	return fs.DoInitialize(
+		"",
+		configuration.GetString(prefix+"grpcAddress"),
+		configuration.GetString(prefix+"directory"),
+		false,
+	)
+}
+
+func (fs *FilerSource) DoInitialize(address, grpcAddress string, dir string, readChunkFromFiler bool) (err error) {
+	fs.address = address
+	if fs.address == "" {
+		fs.address = pb.GrpcAddressToServerAddress(grpcAddress)
+	}
+	fs.grpcAddress = grpcAddress
+	fs.Dir = dir
+	fs.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+	fs.proxyByFiler = readChunkFromFiler
+	return nil
+}
+
+func (fs *FilerSource) SetGrpcDialOption(option grpc.DialOption) {
+	fs.grpcDialOption = option
+}
+
+func (fs *FilerSource) SetHttpClient(client *util_http_client.HTTPClient) {
+	fs.httpClient = client
+}
+
+func (fs *FilerSource) LookupFileId(ctx context.Context, part string) (fileUrls []string, err error) {
+
+	vid2Locations := make(map[string]*filer_pb.Locations)
+
+	vid := volumeId(part)
+
+	err = fs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{
+			VolumeIds: []string{vid},
+		})
+		if err != nil {
+			return err
+		}
+
+		vid2Locations = resp.LocationsMap
+
+		return nil
+	})
+
+	if err != nil {
+		glog.V(1).InfofCtx(ctx, "LookupFileId volume id %s: %v", vid, err)
+		return nil, fmt.Errorf("LookupFileId volume id %s: %v", vid, err)
+	}
+
+	locations := vid2Locations[vid]
+
+	if locations == nil || len(locations.Locations) == 0 {
+		glog.V(1).InfofCtx(ctx, "LookupFileId locate volume id %s: %v", vid, ErrVolumeNotFound)
+		return nil, fmt.Errorf("LookupFileId locate volume id %s: %w", vid, ErrVolumeNotFound)
+	}
+
+	if !fs.proxyByFiler {
+		for _, loc := range locations.Locations {
+			fileUrl := fmt.Sprintf("http://%s/%s?readDeleted=true", loc.Url, part)
+			// Prefer same data center
+			if fs.dataCenter != "" && fs.dataCenter == loc.DataCenter {
+				fileUrls = append([]string{fileUrl}, fileUrls...)
+			} else {
+				fileUrls = append(fileUrls, fileUrl)
+			}
+		}
+	} else {
+		fileUrls = append(fileUrls, util_http.ProxyChunkUrl(fs.address, part))
+	}
+
+	return
+}
+
+func (fs *FilerSource) ReadPart(fileId string, offset int64) (filename string, header http.Header, resp *http.Response, err error) {
+	downloadFn := util_http.DownloadFile
+	if fs.httpClient != nil {
+		downloadFn = func(fileUrl string, jwt string, offset ...int64) (string, http.Header, *http.Response, error) {
+			return util_http.DownloadFileWithClient(fs.httpClient, fileUrl, jwt, offset...)
+		}
+	}
+
+	if fs.proxyByFiler {
+		fileUrl := util_http.ProxyChunkUrl(fs.address, fileId)
+		filename, header, resp, err = downloadFn(fileUrl, util_http.JwtForFilerServer(false), offset)
+		if err == nil {
+			err = readPartStatusError(fileUrl, resp)
+		}
+		if err != nil {
+			glog.V(0).Infof("read part %s via filer proxy %s offset %d: %v", fileId, fs.address, offset, err)
+			return "", nil, nil, err
+		}
+		glog.V(4).Infof("read part %s via filer proxy %s offset %d content-length:%s", fileId, fs.address, offset, header.Get("Content-Length"))
+		return
+	}
+
+	fileUrls, err := fs.LookupFileId(context.Background(), fileId)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	for _, fileUrl := range fileUrls {
+		filename, header, resp, err = downloadFn(fileUrl, "", offset)
+		if err == nil {
+			err = readPartStatusError(fileUrl, resp)
+		}
+		if err != nil {
+			resp = nil
+			glog.V(0).Infof("fail to read part %s from %s offset %d: %v", fileId, fileUrl, offset, err)
+			continue
+		}
+		glog.V(4).Infof("read part %s from %s offset %d content-length:%s", fileId, fileUrl, offset, header.Get("Content-Length"))
+		break
+	}
+
+	return filename, header, resp, err
+}
+
+// readPartStatusError turns a failure status into an error and closes the
+// response. Otherwise the caller copies the error page as chunk content and
+// reports it as a short read; a needle vacuum has removed answers 404, which
+// is the source having lost the data rather than corruption.
+func readPartStatusError(fileUrl string, resp *http.Response) error {
+	if resp == nil || resp.StatusCode < http.StatusBadRequest {
+		return nil
+	}
+	defer util_http.CloseResponse(resp)
+	if resp.StatusCode == http.StatusNotFound {
+		return fmt.Errorf("%s: %s: %w", fileUrl, resp.Status, util_http.ErrNotFound)
+	}
+	return fmt.Errorf("%s: %s", fileUrl, resp.Status)
+}
+
+var _ = filer_pb.FilerClient(&FilerSource{})
+
+func (fs *FilerSource) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+
+	return pb.WithGrpcClient(context.Background(), streamingMode, fs.signature, func(grpcConnection *grpc.ClientConn) error {
+		client := filer_pb.NewSeaweedFilerClient(grpcConnection)
+		return fn(client)
+	}, fs.grpcAddress, false, fs.grpcDialOption)
+
+}
+
+func (fs *FilerSource) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+func (fs *FilerSource) GetDataCenter() string {
+	return fs.dataCenter
+}
+
+func volumeId(fileId string) string {
+	lastCommaIndex := strings.LastIndex(fileId, ",")
+	if lastCommaIndex > 0 {
+		return fileId[:lastCommaIndex]
+	}
+	return fileId
+}

weed/replication/source/filer_source_test.go

@@ -0,0 +1,395 @@
+package source
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func TestMain(m *testing.M) {
+	util_http.InitGlobalHttpClient()
+	os.Exit(m.Run())
+}
+
+func TestDownloadFile_NoOffset(t *testing.T) {
+	testData := []byte("0123456789abcdefghij")
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Range") != "" {
+			t.Error("Range header should not be set when offset is 0")
+		}
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Write(testData)
+	}))
+	defer server.Close()
+
+	_, _, resp, err := util_http.DownloadFile(server.URL, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(data, testData) {
+		t.Fatalf("expected %q, got %q", testData, data)
+	}
+}
+
+func TestDownloadFile_WithOffset(t *testing.T) {
+	testData := []byte("0123456789abcdefghij")
+
+	var receivedRange string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		receivedRange = r.Header.Get("Range")
+		var offset int
+		fmt.Sscanf(receivedRange, "bytes=%d-", &offset)
+		w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, len(testData)-1, len(testData)))
+		w.WriteHeader(http.StatusPartialContent)
+		w.Write(testData[offset:])
+	}))
+	defer server.Close()
+
+	_, _, resp, err := util_http.DownloadFile(server.URL, "", 10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if receivedRange != "bytes=10-" {
+		t.Fatalf("expected Range header %q, got %q", "bytes=10-", receivedRange)
+	}
+	if resp.StatusCode != http.StatusPartialContent {
+		t.Fatalf("expected status 206, got %d", resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(data, testData[10:]) {
+		t.Fatalf("expected %q, got %q", testData[10:], data)
+	}
+}
+
+func TestDownloadFile_RejectsIgnoredRange(t *testing.T) {
+	// Server ignores Range header and returns 200 OK with full body
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("full body"))
+	}))
+	defer server.Close()
+
+	_, _, _, err := util_http.DownloadFile(server.URL, "", 100)
+	if err == nil {
+		t.Fatal("expected error when server ignores Range and returns 200")
+	}
+}
+
+func TestDownloadFile_ContentDisposition(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Disposition", `attachment; filename="test.txt"`)
+		w.Write([]byte("data"))
+	}))
+	defer server.Close()
+
+	filename, _, resp, err := util_http.DownloadFile(server.URL, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+
+	if filename != "test.txt" {
+		t.Fatalf("expected filename %q, got %q", "test.txt", filename)
+	}
+}
+
+// TestDownloadFile_PartialReadThenResume simulates a connection drop
+// after partial data, then resumes from the offset. Verifies the combined
+// data matches the original.
+func TestDownloadFile_PartialReadThenResume(t *testing.T) {
+	testData := bytes.Repeat([]byte("abcdefghij"), 100) // 1000 bytes
+	dropAfter := 500
+
+	var requestNum atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := requestNum.Add(1)
+		rangeHeader := r.Header.Get("Range")
+
+		if n == 1 && rangeHeader == "" {
+			// First request: write partial data then kill the connection
+			hj, ok := w.(http.Hijacker)
+			if !ok {
+				t.Error("server doesn't support hijacking")
+				return
+			}
+			conn, buf, _ := hj.Hijack()
+			fmt.Fprintf(buf, "HTTP/1.1 200 OK\r\n")
+			fmt.Fprintf(buf, "Content-Length: %d\r\n", len(testData))
+			fmt.Fprintf(buf, "Content-Type: application/octet-stream\r\n")
+			fmt.Fprintf(buf, "\r\n")
+			buf.Write(testData[:dropAfter])
+			buf.Flush()
+			conn.Close()
+			return
+		}
+
+		// Resume request with Range
+		var offset int
+		fmt.Sscanf(rangeHeader, "bytes=%d-", &offset)
+		w.Header().Set("Content-Range",
+			fmt.Sprintf("bytes %d-%d/%d", offset, len(testData)-1, len(testData)))
+		w.WriteHeader(http.StatusPartialContent)
+		w.Write(testData[offset:])
+	}))
+	defer server.Close()
+
+	// First read — should get partial data + error
+	_, _, resp1, err := util_http.DownloadFile(server.URL, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	partialData, readErr := io.ReadAll(resp1.Body)
+	resp1.Body.Close()
+	if readErr == nil {
+		t.Fatal("expected error from truncated response")
+	}
+	if len(partialData) == 0 {
+		t.Fatal("expected some partial data")
+	}
+	if !bytes.Equal(partialData, testData[:len(partialData)]) {
+		t.Fatal("partial data doesn't match beginning of original")
+	}
+
+	// Resume from where we left off
+	_, _, resp2, err := util_http.DownloadFile(server.URL, "", int64(len(partialData)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp2.Body.Close()
+	remainingData, readErr := io.ReadAll(resp2.Body)
+	if readErr != nil {
+		t.Fatalf("unexpected error on resume: %v", readErr)
+	}
+
+	// Combined data must match original
+	fullData := append(partialData, remainingData...)
+	if !bytes.Equal(fullData, testData) {
+		t.Fatalf("combined data mismatch: got %d bytes, want %d", len(fullData), len(testData))
+	}
+}
+
+// TestDownloadFile_GzipPartialReadThenResume verifies the tricky case
+// where the first response is gzip-encoded (and Go's HTTP client auto-
+// decompresses it), but the resume request gets uncompressed data (because
+// Go doesn't add Accept-Encoding when Range is set). The combined
+// decompressed bytes must still match the original.
+func TestDownloadFile_GzipPartialReadThenResume(t *testing.T) {
+	testData := bytes.Repeat([]byte("hello world gzip test "), 100) // ~2200 bytes
+
+	// Pre-compress
+	var compressed bytes.Buffer
+	gz := gzip.NewWriter(&compressed)
+	gz.Write(testData)
+	gz.Close()
+	compressedData := compressed.Bytes()
+	dropAfter := len(compressedData) / 2
+
+	var requestNum atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := requestNum.Add(1)
+		rangeHeader := r.Header.Get("Range")
+		acceptsGzip := r.Header.Get("Accept-Encoding") != ""
+
+		if n == 1 && acceptsGzip && rangeHeader == "" {
+			// First request: serve gzip-encoded response, drop mid-stream.
+			// Go's HTTP client auto-added Accept-Encoding: gzip (no Range),
+			// so it will auto-decompress and strip the Content-Encoding header.
+			hj, ok := w.(http.Hijacker)
+			if !ok {
+				t.Error("server doesn't support hijacking")
+				return
+			}
+			conn, buf, _ := hj.Hijack()
+			fmt.Fprintf(buf, "HTTP/1.1 200 OK\r\n")
+			fmt.Fprintf(buf, "Content-Encoding: gzip\r\n")
+			fmt.Fprintf(buf, "Content-Length: %d\r\n", len(compressedData))
+			fmt.Fprintf(buf, "Content-Type: application/octet-stream\r\n")
+			fmt.Fprintf(buf, "\r\n")
+			buf.Write(compressedData[:dropAfter])
+			buf.Flush()
+			conn.Close()
+			return
+		}
+
+		// Resume request: Range is set, Go did NOT add Accept-Encoding,
+		// so we serve decompressed data from the requested offset —
+		// mimicking what the volume server does when the client doesn't
+		// advertise gzip support.
+		var offset int
+		fmt.Sscanf(rangeHeader, "bytes=%d-", &offset)
+		remaining := testData[offset:]
+		w.Header().Set("Content-Range",
+			fmt.Sprintf("bytes %d-%d/%d", offset, len(testData)-1, len(testData)))
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.WriteHeader(http.StatusPartialContent)
+		w.Write(remaining)
+	}))
+	defer server.Close()
+
+	// First read: Go auto-decompresses; truncated stream → error + partial data
+	_, _, resp1, err := util_http.DownloadFile(server.URL, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	partialData, readErr := io.ReadAll(resp1.Body)
+	resp1.Body.Close()
+	if readErr == nil {
+		t.Fatal("expected error from truncated gzip response")
+	}
+	if len(partialData) == 0 {
+		t.Fatal("expected some decompressed partial data")
+	}
+	// Partial decompressed data must match the beginning of the original
+	if !bytes.Equal(partialData, testData[:len(partialData)]) {
+		t.Fatalf("partial decompressed data doesn't match original (got %d bytes)", len(partialData))
+	}
+
+	// Resume: offset is in the *decompressed* domain.
+	// The server (like the volume server) decompresses and serves from that offset.
+	_, _, resp2, err := util_http.DownloadFile(server.URL, "", int64(len(partialData)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp2.Body.Close()
+	remainingData, readErr := io.ReadAll(resp2.Body)
+	if readErr != nil {
+		t.Fatalf("unexpected error on resume: %v", readErr)
+	}
+
+	fullData := append(partialData, remainingData...)
+	if !bytes.Equal(fullData, testData) {
+		t.Fatalf("combined data mismatch: got %d bytes, want %d", len(fullData), len(testData))
+	}
+}
+
+// lookupFilerServer answers volume lookups with a fixed set of locations, so a
+// test can hand ReadPart several replicas, or none at all.
+type lookupFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+	locations []string
+}
+
+func (s *lookupFilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) {
+	locationsMap := make(map[string]*filer_pb.Locations)
+	for _, vid := range req.VolumeIds {
+		if len(s.locations) == 0 {
+			continue
+		}
+		locations := &filer_pb.Locations{}
+		for _, url := range s.locations {
+			locations.Locations = append(locations.Locations, &filer_pb.Location{Url: url})
+		}
+		locationsMap[vid] = locations
+	}
+	return &filer_pb.LookupVolumeResponse{LocationsMap: locationsMap}, nil
+}
+
+func startLookupFiler(t *testing.T, locations ...string) *FilerSource {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	server := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(server, &lookupFilerServer{locations: locations})
+	go server.Serve(listener)
+	t.Cleanup(server.Stop)
+
+	address := listener.Addr().String()
+	filerSource := &FilerSource{}
+	if err := filerSource.DoInitialize(address, address, "/", false); err != nil {
+		t.Fatalf("DoInitialize: %v", err)
+	}
+	filerSource.SetGrpcDialOption(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	return filerSource
+}
+
+// A volume the source cluster no longer has must be reported as ErrVolumeNotFound,
+// not as an untyped message the caller can only string-match.
+func TestLookupFileIdVolumeNotFound(t *testing.T) {
+	filerSource := startLookupFiler(t)
+
+	_, err := filerSource.LookupFileId(context.Background(), "5617,01abc")
+	if !errors.Is(err, ErrVolumeNotFound) {
+		t.Fatalf("expected ErrVolumeNotFound, got %v", err)
+	}
+}
+
+// A replica answering 404 is a failed read, not an empty file: ReadPart must move
+// on to the next replica instead of handing the error page back as content.
+func TestReadPartSkipsNotFoundReplica(t *testing.T) {
+	const body = "chunk bytes"
+
+	gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Not Found", http.StatusNotFound)
+	}))
+	defer gone.Close()
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(body))
+	}))
+	defer live.Close()
+
+	filerSource := startLookupFiler(t,
+		strings.TrimPrefix(gone.URL, "http://"), strings.TrimPrefix(live.URL, "http://"))
+
+	_, _, resp, err := filerSource.ReadPart("5617,01abc", 0)
+	if err != nil {
+		t.Fatalf("ReadPart: %v", err)
+	}
+	defer util_http.CloseResponse(resp)
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+	if string(data) != body {
+		t.Fatalf("got %q, want %q", data, body)
+	}
+}
+
+// Every replica gone: the caller must see a not-found error it can act on, and no
+// response left open behind it.
+func TestReadPartAllReplicasNotFound(t *testing.T) {
+	gone := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Not Found", http.StatusNotFound)
+	}))
+	defer gone.Close()
+
+	filerSource := startLookupFiler(t, strings.TrimPrefix(gone.URL, "http://"))
+
+	_, _, resp, err := filerSource.ReadPart("5617,01abc", 0)
+	if !errors.Is(err, util_http.ErrNotFound) {
+		t.Fatalf("expected ErrNotFound, got %v", err)
+	}
+	if resp != nil {
+		t.Fatal("expected no response alongside the error")
+	}
+}

weed/replication/sub/notification_aws_sqs.go

@@ -0,0 +1,124 @@
+package sub
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &AwsSqsInput{})
+}
+
+type AwsSqsInput struct {
+	svc      *sqs.SQS
+	queueUrl string
+}
+
+func (k *AwsSqsInput) GetName() string {
+	return "aws_sqs"
+}
+
+func (k *AwsSqsInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("replication.notification.aws_sqs.region: %v", configuration.GetString(prefix+"region"))
+	glog.V(0).Infof("replication.notification.aws_sqs.sqs_queue_name: %v", configuration.GetString(prefix+"sqs_queue_name"))
+	return k.initialize(
+		configuration.GetString(prefix+"aws_access_key_id"),
+		configuration.GetString(prefix+"aws_secret_access_key"),
+		configuration.GetString(prefix+"region"),
+		configuration.GetString(prefix+"sqs_queue_name"),
+	)
+}
+
+func (k *AwsSqsInput) initialize(awsAccessKeyId, awsSecretAccessKey, region, queueName string) (err error) {
+
+	config := &aws.Config{
+		Region:                        aws.String(region),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return fmt.Errorf("create aws session: %w", err)
+	}
+	k.svc = sqs.New(sess)
+
+	result, err := k.svc.GetQueueUrl(&sqs.GetQueueUrlInput{
+		QueueName: aws.String(queueName),
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sqs.ErrCodeQueueDoesNotExist {
+			return fmt.Errorf("unable to find queue %s", queueName)
+		}
+		return fmt.Errorf("get queue %s url: %v", queueName, err)
+	}
+
+	k.queueUrl = *result.QueueUrl
+
+	return nil
+}
+
+func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	// receive message
+	result, err := k.svc.ReceiveMessage(&sqs.ReceiveMessageInput{
+		AttributeNames: []*string{
+			aws.String(sqs.MessageSystemAttributeNameSentTimestamp),
+		},
+		MessageAttributeNames: []*string{
+			aws.String(sqs.QueueAttributeNameAll),
+		},
+		QueueUrl:            &k.queueUrl,
+		MaxNumberOfMessages: aws.Int64(1),
+		VisibilityTimeout:   aws.Int64(20), // 20 seconds
+		WaitTimeSeconds:     aws.Int64(20),
+	})
+	if err != nil {
+		err = fmt.Errorf("receive message from sqs %s: %v", k.queueUrl, err)
+		return
+	}
+	if len(result.Messages) == 0 {
+		return
+	}
+
+	// process the message
+	// fmt.Printf("messages: %+v\n", result.Messages[0])
+	keyValue := result.Messages[0].MessageAttributes["key"]
+	key = *keyValue.StringValue
+	text := *result.Messages[0].Body
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal([]byte(text), message)
+	if err != nil {
+		err = fmt.Errorf("unmarshal message from sqs %s: %w", k.queueUrl, err)
+		return
+	}
+
+	// Delete only once the message has been replicated. Deleting on receipt
+	// drops the message for good when the sink write fails; leaving it in the
+	// queue lets the visibility timeout redeliver it.
+	receiptHandle := result.Messages[0].ReceiptHandle
+	onSuccessFn = func() {
+		if _, deleteErr := k.svc.DeleteMessage(&sqs.DeleteMessageInput{
+			QueueUrl:      &k.queueUrl,
+			ReceiptHandle: receiptHandle,
+		}); deleteErr != nil {
+			glog.V(1).Infof("delete message from sqs %s: %v", k.queueUrl, deleteErr)
+		}
+	}
+	onFailureFn = func() {
+		glog.V(1).Infof("keeping message %s in sqs %s for redelivery", key, k.queueUrl)
+	}
+
+	return
+}

weed/replication/sub/notification_gocdk_pub_sub.go

@@ -0,0 +1,167 @@
+//go:build gocdk
+
+package sub
+
+import (
+	"context"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	amqp "github.com/rabbitmq/amqp091-go"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"gocloud.dev/pubsub"
+	_ "gocloud.dev/pubsub/awssnssqs"
+	"gocloud.dev/pubsub/rabbitpubsub"
+	"google.golang.org/protobuf/proto"
+
+	// _ "gocloud.dev/pubsub/azuresb"
+	_ "gocloud.dev/pubsub/gcppubsub"
+	_ "gocloud.dev/pubsub/natspubsub"
+	_ "gocloud.dev/pubsub/rabbitpubsub"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GoCDKPubSubInput{})
+}
+
+func getPath(rawUrl string) string {
+	parsedUrl, _ := url.Parse(rawUrl)
+	return path.Join(parsedUrl.Host, parsedUrl.Path)
+}
+
+func QueueDeclareAndBind(conn *amqp.Connection, exchangeUrl string, queueUrl string) error {
+	exchangeName := getPath(exchangeUrl)
+	queueName := getPath(queueUrl)
+	exchangeNameDLX := "DLX." + exchangeName
+	queueNameDLX := "DLX." + queueName
+	ch, err := conn.Channel()
+	if err != nil {
+		glog.Error(err)
+		return err
+	}
+	defer ch.Close()
+	if err := ch.ExchangeDeclare(
+		exchangeNameDLX, "fanout", true, false, false, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.ExchangeDeclare(
+		exchangeName, "fanout", true, false, false, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if _, err := ch.QueueDeclare(
+		queueName, true, false, false, false,
+		amqp.Table{"x-dead-letter-exchange": exchangeNameDLX}); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.QueueBind(queueName, "", exchangeName, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if _, err := ch.QueueDeclare(
+		queueNameDLX, true, false, false, false,
+		amqp.Table{"x-dead-letter-exchange": exchangeName, "x-message-ttl": 600000}); err != nil {
+		glog.Error(err)
+		return err
+	}
+	if err := ch.QueueBind(queueNameDLX, "", exchangeNameDLX, false, nil); err != nil {
+		glog.Error(err)
+		return err
+	}
+	return nil
+}
+
+type GoCDKPubSubInput struct {
+	sub    *pubsub.Subscription
+	subURL string
+}
+
+func (k *GoCDKPubSubInput) GetName() string {
+	return "gocdk_pub_sub"
+}
+
+func (k *GoCDKPubSubInput) Initialize(configuration util.Configuration, prefix string) error {
+	topicUrl := configuration.GetString(prefix + "topic_url")
+	k.subURL = configuration.GetString(prefix + "sub_url")
+	glog.V(0).Infof("notification.gocdk_pub_sub.sub_url: %v", k.subURL)
+	sub, err := pubsub.OpenSubscription(context.Background(), k.subURL)
+	if err != nil {
+		return err
+	}
+	var conn *amqp.Connection
+	if sub.As(&conn) {
+		ch, err := conn.Channel()
+		if err != nil {
+			return err
+		}
+		defer ch.Close()
+		_, err = ch.QueueInspect(getPath(k.subURL))
+		if err != nil {
+			if strings.HasPrefix(err.Error(), "Exception (404) Reason") {
+				if err := QueueDeclareAndBind(conn, topicUrl, k.subURL); err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+	}
+	k.sub = sub
+	return nil
+}
+
+func (k *GoCDKPubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+	ctx := context.Background()
+	msg, err := k.sub.Receive(ctx)
+	if err != nil {
+		var conn *amqp.Connection
+		if k.sub.As(&conn) && conn.IsClosed() {
+			conn.Close()
+			k.sub.Shutdown(ctx)
+			conn, err = amqp.Dial(os.Getenv("RABBIT_SERVER_URL"))
+			if err != nil {
+				glog.Error(err)
+				time.Sleep(time.Second)
+				return
+			}
+			k.sub = rabbitpubsub.OpenSubscription(conn, getPath(k.subURL), nil)
+			return
+		}
+		// This is permanent cached sub err
+		glog.Fatal(err)
+	}
+	onFailureFn = func() {
+		if msg.Nackable() {
+			isRedelivered := false
+			var delivery amqp.Delivery
+			if msg.As(&delivery) {
+				isRedelivered = delivery.Redelivered
+				glog.Warningf("onFailureFn() metadata: %+v, redelivered: %v", msg.Metadata, delivery.Redelivered)
+			}
+			if isRedelivered {
+				if err := delivery.Nack(false, false); err != nil {
+					glog.Error(err)
+				}
+			} else {
+				msg.Nack()
+			}
+		}
+	}
+	onSuccessFn = func() {
+		msg.Ack()
+	}
+	key = msg.Metadata["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Body, message)
+	if err != nil {
+		return "", nil, onSuccessFn, onFailureFn, err
+	}
+	return key, message, onSuccessFn, onFailureFn, nil
+}

weed/replication/sub/notification_google_pub_sub.go

@@ -0,0 +1,115 @@
+package sub
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"cloud.google.com/go/pubsub"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &GooglePubSubInput{})
+}
+
+type GooglePubSubInput struct {
+	sub         *pubsub.Subscription
+	topicName   string
+	messageChan chan *pubsub.Message
+}
+
+func (k *GooglePubSubInput) GetName() string {
+	return "google_pub_sub"
+}
+
+func (k *GooglePubSubInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString(prefix+"project_id"))
+	glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString(prefix+"topic"))
+	return k.initialize(
+		configuration.GetString(prefix+"google_application_credentials"),
+		configuration.GetString(prefix+"project_id"),
+		configuration.GetString(prefix+"topic"),
+	)
+}
+
+func (k *GooglePubSubInput) initialize(google_application_credentials, projectId, topicName string) (err error) {
+
+	ctx := context.Background()
+	// Creates a client.
+	if google_application_credentials == "" {
+		var found bool
+		google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
+		if !found {
+			glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml")
+		}
+	}
+
+	client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials))
+	if err != nil {
+		glog.Fatalf("Failed to create client: %v", err)
+	}
+
+	k.topicName = topicName
+	topic := client.Topic(topicName)
+	if exists, err := topic.Exists(ctx); err == nil {
+		if !exists {
+			topic, err = client.CreateTopic(ctx, topicName)
+			if err != nil {
+				glog.Fatalf("Failed to create topic %s: %v", topicName, err)
+			}
+		}
+	} else {
+		glog.Fatalf("Failed to check topic %s: %v", topicName, err)
+	}
+
+	subscriptionName := "seaweedfs_sub"
+
+	k.sub = client.Subscription(subscriptionName)
+	if exists, err := k.sub.Exists(ctx); err == nil {
+		if !exists {
+			k.sub, err = client.CreateSubscription(ctx, subscriptionName, pubsub.SubscriptionConfig{Topic: topic})
+			if err != nil {
+				glog.Fatalf("Failed to create subscription %s: %v", subscriptionName, err)
+			}
+		}
+	} else {
+		glog.Fatalf("Failed to check subscription %s: %v", topicName, err)
+	}
+
+	k.messageChan = make(chan *pubsub.Message, 1)
+
+	go k.sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) {
+		k.messageChan <- m
+	})
+
+	return err
+}
+
+func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	m := <-k.messageChan
+
+	onSuccessFn = func() {
+		m.Ack()
+	}
+	onFailureFn = func() {
+		m.Nack()
+	}
+
+	// process the message
+	key = m.Attributes["key"]
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(m.Data, message)
+
+	if err != nil {
+		err = fmt.Errorf("unmarshal message from google pubsub %s: %v", k.topicName, err)
+		return
+	}
+
+	return
+}

weed/replication/sub/notification_kafka.go

@@ -0,0 +1,203 @@
+package sub
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/Shopify/sarama"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	kafkanotif "github.com/seaweedfs/seaweedfs/weed/notification/kafka"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	NotificationInputs = append(NotificationInputs, &KafkaInput{})
+}
+
+type KafkaInput struct {
+	topic       string
+	consumer    sarama.Consumer
+	messageChan chan *sarama.ConsumerMessage
+	progress    *KafkaProgress
+}
+
+func (k *KafkaInput) GetName() string {
+	return "kafka"
+}
+
+func (k *KafkaInput) Initialize(configuration util.Configuration, prefix string) error {
+	glog.V(0).Infof("replication.notification.kafka.hosts: %v\n", configuration.GetStringSlice(prefix+"hosts"))
+	glog.V(0).Infof("replication.notification.kafka.topic: %v\n", configuration.GetString(prefix+"topic"))
+	return k.initialize(
+		configuration.GetStringSlice(prefix+"hosts"),
+		configuration.GetString(prefix+"topic"),
+		configuration.GetString(prefix+"offsetFile"),
+		configuration.GetInt(prefix+"offsetSaveIntervalSeconds"),
+		kafkanotif.SASLTLSConfig{
+			SASLEnabled:           configuration.GetBool(prefix + "sasl_enabled"),
+			SASLMechanism:         configuration.GetString(prefix + "sasl_mechanism"),
+			SASLUsername:          configuration.GetString(prefix + "sasl_username"),
+			SASLPassword:          configuration.GetString(prefix + "sasl_password"),
+			TLSEnabled:            configuration.GetBool(prefix + "tls_enabled"),
+			TLSCACert:             configuration.GetString(prefix + "tls_ca_cert"),
+			TLSClientCert:         configuration.GetString(prefix + "tls_client_cert"),
+			TLSClientKey:          configuration.GetString(prefix + "tls_client_key"),
+			TLSInsecureSkipVerify: configuration.GetBool(prefix + "tls_insecure_skip_verify"),
+		},
+	)
+}
+
+func (k *KafkaInput) initialize(hosts []string, topic string, offsetFile string, offsetSaveIntervalSeconds int, saslTLS kafkanotif.SASLTLSConfig) (err error) {
+	config := sarama.NewConfig()
+	config.Consumer.Return.Errors = true
+	if err = kafkanotif.ConfigureSASLTLS(config, saslTLS); err != nil {
+		return fmt.Errorf("kafka consumer security configuration: %w", err)
+	}
+	k.consumer, err = sarama.NewConsumer(hosts, config)
+	if err != nil {
+		return fmt.Errorf("create kafka consumer: %w", err)
+	}
+	glog.V(0).Infof("connected to %v", hosts)
+
+	k.topic = topic
+	k.messageChan = make(chan *sarama.ConsumerMessage, 1)
+
+	partitions, err := k.consumer.Partitions(topic)
+	if err != nil {
+		return fmt.Errorf("get kafka partitions for topic %q: %w", topic, err)
+	}
+
+	progress := loadProgress(offsetFile)
+	if progress == nil || progress.Topic != topic {
+		progress = &KafkaProgress{
+			Topic:            topic,
+			PartitionOffsets: make(map[int32]int64),
+		}
+	}
+	progress.lastSaveTime = time.Now()
+	progress.offsetFile = offsetFile
+	progress.offsetSaveIntervalSeconds = offsetSaveIntervalSeconds
+	progress.failedOffsets = make(map[int32]int64)
+	k.progress = progress
+
+	for _, partition := range partitions {
+		offset, found := progress.PartitionOffsets[partition]
+		if !found {
+			offset = sarama.OffsetOldest
+		} else {
+			offset += 1
+		}
+		partitionConsumer, err := k.consumer.ConsumePartition(topic, partition, offset)
+		if err != nil {
+			return fmt.Errorf("consume kafka topic %q partition %d: %w", topic, partition, err)
+		}
+		go func() {
+			for {
+				select {
+				case err := <-partitionConsumer.Errors():
+					fmt.Println(err)
+				case msg := <-partitionConsumer.Messages():
+					k.messageChan <- msg
+				}
+			}
+		}()
+	}
+
+	return nil
+}
+
+func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error) {
+
+	msg := <-k.messageChan
+
+	// Commit only once the message has been replicated. Committing on receipt
+	// leaves nothing to redeliver when the sink write fails.
+	onSuccessFn = func() {
+		if err := k.progress.setOffset(msg.Partition, msg.Offset); err != nil {
+			glog.Warningf("set kafka offset: %v", err)
+		}
+	}
+	onFailureFn = func() {
+		k.progress.markFailed(msg.Partition, msg.Offset)
+	}
+
+	key = string(msg.Key)
+	message = &filer_pb.EventNotification{}
+	err = proto.Unmarshal(msg.Value, message)
+
+	return
+}
+
+type KafkaProgress struct {
+	Topic                     string          `json:"topic"`
+	PartitionOffsets          map[int32]int64 `json:"partitionOffsets"`
+	offsetFile                string
+	lastSaveTime              time.Time
+	offsetSaveIntervalSeconds int
+	// failedOffsets is the oldest offset that failed to replicate in each
+	// partition. Nothing at or past it is ever committed, so a restart
+	// redelivers from the failure instead of resuming after it.
+	failedOffsets map[int32]int64
+	sync.Mutex
+}
+
+func loadProgress(offsetFile string) *KafkaProgress {
+	progress := &KafkaProgress{}
+	data, err := os.ReadFile(offsetFile)
+	if err != nil {
+		glog.Warningf("failed to read kafka progress file: %s", offsetFile)
+		return nil
+	}
+	err = json.Unmarshal(data, progress)
+	if err != nil {
+		glog.Warningf("failed to read kafka progress message: %s", string(data))
+		return nil
+	}
+	return progress
+}
+
+func (progress *KafkaProgress) saveProgress() error {
+	data, err := json.Marshal(progress)
+	if err != nil {
+		return fmt.Errorf("failed to marshal progress: %w", err)
+	}
+	err = util.WriteFile(progress.offsetFile, data, 0640)
+	if err != nil {
+		return fmt.Errorf("failed to save progress to %s: %v", progress.offsetFile, err)
+	}
+
+	progress.lastSaveTime = time.Now()
+	return nil
+}
+
+func (progress *KafkaProgress) setOffset(partition int32, offset int64) error {
+	progress.Lock()
+	defer progress.Unlock()
+
+	if failedOffset, found := progress.failedOffsets[partition]; found && offset >= failedOffset {
+		return nil
+	}
+
+	progress.PartitionOffsets[partition] = offset
+	if int(time.Now().Sub(progress.lastSaveTime).Seconds()) > progress.offsetSaveIntervalSeconds {
+		return progress.saveProgress()
+	}
+	return nil
+}
+
+// markFailed records an offset that could not be replicated, holding the
+// committed offset for that partition behind it.
+func (progress *KafkaProgress) markFailed(partition int32, offset int64) {
+	progress.Lock()
+	defer progress.Unlock()
+
+	if failedOffset, found := progress.failedOffsets[partition]; !found || offset < failedOffset {
+		progress.failedOffsets[partition] = offset
+		glog.Errorf("replicate kafka %s partition %d offset %d failed; holding the committed offset before it", progress.Topic, partition, offset)
+	}
+}

weed/replication/sub/notification_kafka_test.go

@@ -0,0 +1,69 @@
+package sub
+
+import (
+	"testing"
+	"time"
+)
+
+func newTestProgress() *KafkaProgress {
+	return &KafkaProgress{
+		Topic:            "test",
+		PartitionOffsets: make(map[int32]int64),
+		failedOffsets:    make(map[int32]int64),
+		lastSaveTime:     time.Now(),
+		// large enough that no test call reaches saveProgress and touches disk
+		offsetSaveIntervalSeconds: 3600,
+	}
+}
+
+// TestKafkaProgressHoldsOffsetAtFailure verifies that a message that failed to
+// replicate keeps the committed offset behind it, so a restart redelivers it
+// rather than resuming after it.
+func TestKafkaProgressHoldsOffsetAtFailure(t *testing.T) {
+	progress := newTestProgress()
+
+	if err := progress.setOffset(0, 10); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d after a replicated message, want 10", got)
+	}
+
+	progress.markFailed(0, 11)
+
+	if err := progress.setOffset(0, 12); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d after a later success, want it held at 10", got)
+	}
+
+	// a failure in one partition must not stall the others
+	if err := progress.setOffset(1, 5); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[1]; got != 5 {
+		t.Fatalf("offset = %d in an unaffected partition, want 5", got)
+	}
+}
+
+// TestKafkaProgressKeepsOldestFailure verifies that the hold point is the
+// oldest failed offset, not the most recent one.
+func TestKafkaProgressKeepsOldestFailure(t *testing.T) {
+	progress := newTestProgress()
+
+	progress.markFailed(0, 20)
+	progress.markFailed(0, 11)
+	progress.markFailed(0, 30)
+
+	if got := progress.failedOffsets[0]; got != 11 {
+		t.Fatalf("held at offset %d, want the oldest failure 11", got)
+	}
+
+	if err := progress.setOffset(0, 10); err != nil {
+		t.Fatalf("setOffset: %v", err)
+	}
+	if got := progress.PartitionOffsets[0]; got != 10 {
+		t.Fatalf("offset = %d, want an offset before the failure to still commit", got)
+	}
+}

weed/replication/sub/notifications.go

@@ -0,0 +1,18 @@
+package sub
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type NotificationInput interface {
+	// GetName gets the name to locate the configuration in sync.toml file
+	GetName() string
+	// Initialize initializes the file store
+	Initialize(configuration util.Configuration, prefix string) error
+	ReceiveMessage() (key string, message *filer_pb.EventNotification, onSuccessFn func(), onFailureFn func(), err error)
+}
+
+var (
+	NotificationInputs []NotificationInput
+)