snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/operation/assign_file_id.go

@@ -0,0 +1,228 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type VolumeAssignRequest struct {
+	Count               uint64
+	Replication         string
+	Collection          string
+	Ttl                 string
+	DiskType            string
+	DataCenter          string
+	Rack                string
+	DataNode            string
+	WritableVolumeCount uint32
+	ExpectedDataSize    uint64
+}
+
+type AssignResult struct {
+	Fid       string              `json:"fid,omitempty"`
+	Url       string              `json:"url,omitempty"`
+	PublicUrl string              `json:"publicUrl,omitempty"`
+	GrpcPort  int                 `json:"grpcPort,omitempty"`
+	Count     uint64              `json:"count,omitempty"`
+	Error     string              `json:"error,omitempty"`
+	Auth      security.EncodedJwt `json:"auth,omitempty"`
+	Replicas  []Location          `json:"replicas,omitempty"`
+	// Fsync carries the storage rule's fsync decision for the assigned path,
+	// so the upload request to the volume server can set ?fsync=true.
+	Fsync bool `json:"fsync,omitempty"`
+}
+
+func Assign(ctx context.Context, masterFn GetMasterFn, grpcDialOption grpc.DialOption, primaryRequest *VolumeAssignRequest, alternativeRequests ...*VolumeAssignRequest) (*AssignResult, error) {
+
+	var requests []*VolumeAssignRequest
+	requests = append(requests, primaryRequest)
+	requests = append(requests, alternativeRequests...)
+
+	var lastError error
+	ret := &AssignResult{}
+
+	// Compute a single deadline so all request entries (primary + fallback)
+	// share one 30s retry budget instead of each getting its own.
+	// Use a deadline-aware context so both RetryWithBackoff and per-attempt
+	// timeouts are bounded by the shared budget.
+	deadline := time.Now().Add(30 * time.Second)
+	deadlineCtx, deadlineCancel := context.WithDeadline(ctx, deadline)
+	defer deadlineCancel()
+
+	for i, request := range requests {
+		if request == nil {
+			continue
+		}
+
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+
+		lastError = util.RetryWithBackoff(deadlineCtx, "assign", remaining,
+			func(err error) bool {
+				st, ok := status.FromError(err)
+				if !ok {
+					return false
+				}
+				switch st.Code() {
+				case codes.Unavailable, codes.ResourceExhausted:
+					// ResourceExhausted: the master is shedding because volume growth
+					// is in flight; retry so we pick up the new volume once it lands.
+					return true
+				case codes.Canceled, codes.DeadlineExceeded:
+					// A stale cached gRPC channel (e.g., master restart behind
+					// a k8s Service VIP) can return Canceled/DeadlineExceeded
+					// immediately even though the caller's context is still
+					// live. The first failure invalidates the cached ClientConn
+					// via shouldInvalidateConnection; retry so the next attempt
+					// dials a fresh channel.
+					return deadlineCtx.Err() == nil
+				}
+				return false
+			},
+			func() error {
+				// Per-attempt timeout to prevent a single slow RPC from consuming the entire retry budget
+				attemptCtx, attemptCancel := context.WithTimeout(deadlineCtx, 10*time.Second)
+				defer attemptCancel()
+				// Pass attemptCtx so its expiry is not mistaken for a dead shared
+				// connection: invalidating it would cancel every other in-flight
+				// assign with "the client connection is closing".
+				return WithMasterServerClient(attemptCtx, false, masterFn(attemptCtx), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+					req := &master_pb.AssignRequest{
+						Count:               request.Count,
+						Replication:         request.Replication,
+						Collection:          request.Collection,
+						Ttl:                 request.Ttl,
+						DiskType:            request.DiskType,
+						DataCenter:          request.DataCenter,
+						Rack:                request.Rack,
+						DataNode:            request.DataNode,
+						WritableVolumeCount: request.WritableVolumeCount,
+						ExpectedDataSize:    request.ExpectedDataSize,
+					}
+					resp, grpcErr := masterClient.Assign(attemptCtx, req)
+					if grpcErr != nil {
+						return grpcErr
+					}
+
+					if resp.Error != "" {
+						return fmt.Errorf("assignRequest: %v", resp.Error)
+					}
+
+					ret.Count = resp.Count
+					ret.Fid = resp.Fid
+					ret.Url = resp.Location.Url
+					ret.PublicUrl = resp.Location.PublicUrl
+					ret.GrpcPort = int(resp.Location.GrpcPort)
+					ret.Error = resp.Error
+					ret.Auth = security.EncodedJwt(resp.Auth)
+					ret.Replicas = nil
+					for _, r := range resp.Replicas {
+						ret.Replicas = append(ret.Replicas, Location{
+							Url:        r.Url,
+							PublicUrl:  r.PublicUrl,
+							DataCenter: r.DataCenter,
+						})
+					}
+
+					return nil
+				})
+			})
+
+		if lastError != nil {
+			stats.FilerHandlerCounter.WithLabelValues(stats.ErrorChunkAssign).Inc()
+			continue
+		}
+
+		if ret.Count <= 0 {
+			lastError = fmt.Errorf("assign failure %d: %v", i+1, ret.Error)
+			continue
+		}
+
+		break
+	}
+
+	return ret, lastError
+}
+
+func LookupJwt(master pb.ServerAddress, grpcDialOption grpc.DialOption, fileId string) (token security.EncodedJwt) {
+
+	WithMasterServerClient(context.Background(), false, master, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+
+		resp, grpcErr := masterClient.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{
+			VolumeOrFileIds: []string{fileId},
+		})
+		if grpcErr != nil {
+			return grpcErr
+		}
+
+		if len(resp.VolumeIdLocations) == 0 {
+			return nil
+		}
+
+		token = security.EncodedJwt(resp.VolumeIdLocations[0].Auth)
+
+		return nil
+
+	})
+
+	return
+}
+
+type StorageOption struct {
+	Replication       string
+	DiskType          string
+	Collection        string
+	DataCenter        string
+	Rack              string
+	DataNode          string
+	TtlSeconds        int32
+	VolumeGrowthCount uint32
+	MaxFileNameLength uint32
+	Fsync             bool
+	SaveInside        bool
+}
+
+func (so *StorageOption) TtlString() string {
+	return needle.SecondsToTTL(so.TtlSeconds)
+}
+
+func (so *StorageOption) ToAssignRequests(count int) (ar *VolumeAssignRequest, altRequest *VolumeAssignRequest) {
+	ar = &VolumeAssignRequest{
+		Count:               uint64(count),
+		Replication:         so.Replication,
+		Collection:          so.Collection,
+		Ttl:                 so.TtlString(),
+		DiskType:            so.DiskType,
+		DataCenter:          so.DataCenter,
+		Rack:                so.Rack,
+		DataNode:            so.DataNode,
+		WritableVolumeCount: so.VolumeGrowthCount,
+	}
+	if so.DataCenter != "" || so.Rack != "" || so.DataNode != "" {
+		altRequest = &VolumeAssignRequest{
+			Count:               uint64(count),
+			Replication:         so.Replication,
+			Collection:          so.Collection,
+			Ttl:                 so.TtlString(),
+			DiskType:            so.DiskType,
+			DataCenter:          "",
+			Rack:                "",
+			DataNode:            "",
+			WritableVolumeCount: so.VolumeGrowthCount,
+		}
+	}
+	return
+}

weed/operation/assign_file_id_retry_test.go

@@ -0,0 +1,108 @@
+package operation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+// fakeAssignServer returns Unavailable for the first N calls, then succeeds.
+type fakeAssignServer struct {
+	master_pb.UnimplementedSeaweedServer
+	unavailableCount int32
+	callCount        atomic.Int32
+}
+
+func (s *fakeAssignServer) Assign(_ context.Context, _ *master_pb.AssignRequest) (*master_pb.AssignResponse, error) {
+	n := s.callCount.Add(1)
+	if n <= s.unavailableCount {
+		return nil, status.Errorf(codes.Unavailable, "master is warming up")
+	}
+	return &master_pb.AssignResponse{
+		Fid:   "1,abc",
+		Count: 1,
+		Location: &master_pb.Location{
+			Url:       "127.0.0.1:8080",
+			PublicUrl: "127.0.0.1:8080",
+		},
+	}, nil
+}
+
+func startFakeServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, srv)
+	go func() { _ = grpcServer.Serve(lis) }()
+	t.Cleanup(grpcServer.GracefulStop)
+
+	_, port, _ := net.SplitHostPort(lis.Addr().String())
+	// Use "0.<grpcPort>" format so ToGrpcAddress() resolves to the actual port
+	return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
+}
+
+func TestAssignRetriesOnUnavailable(t *testing.T) {
+	srv := &fakeAssignServer{unavailableCount: 3}
+	addr := startFakeServer(t, srv)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	ret, err := Assign(ctx, func(_ context.Context) pb.ServerAddress {
+		return addr
+	}, grpc.WithTransportCredentials(insecure.NewCredentials()), &VolumeAssignRequest{Count: 1})
+
+	if err != nil {
+		t.Fatalf("expected success after retries, got error: %v", err)
+	}
+	if ret.Fid != "1,abc" {
+		t.Errorf("expected fid '1,abc', got '%s'", ret.Fid)
+	}
+	if calls := srv.callCount.Load(); calls != 4 {
+		t.Errorf("expected 4 calls (3 unavailable + 1 success), got %d", calls)
+	}
+}
+
+func TestAssignStopsOnContextCancel(t *testing.T) {
+	srv := &fakeAssignServer{unavailableCount: 1000} // never succeeds
+	addr := startFakeServer(t, srv)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	start := time.Now()
+	_, err := Assign(ctx, func(_ context.Context) pb.ServerAddress {
+		return addr
+	}, grpc.WithTransportCredentials(insecure.NewCredentials()), &VolumeAssignRequest{Count: 1})
+
+	elapsed := time.Since(start)
+	if err == nil {
+		t.Fatal("expected error from context cancellation")
+	}
+	// Should stop within a reasonable time after context deadline
+	if elapsed > 5*time.Second {
+		t.Errorf("took %v, expected to stop near context deadline of 2s", elapsed)
+	}
+	// Verify the loop actually retried (not just an immediate failure)
+	if calls := srv.callCount.Load(); calls <= 1 {
+		t.Errorf("expected multiple retry attempts, got %d calls", calls)
+	}
+	// Verify the error is from context deadline
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("expected context.DeadlineExceeded, got: %v", err)
+	}
+}

weed/operation/buffer_pool.go

@@ -0,0 +1,25 @@
+package operation
+
+import (
+	"sync/atomic"
+
+	"github.com/valyala/bytebufferpool"
+)
+
+var bufferCounter int64
+
+func GetBuffer() *bytebufferpool.ByteBuffer {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, 1)
+		// println("+", bufferCounter)
+	}()
+	return bytebufferpool.Get()
+}
+
+func PutBuffer(buf *bytebufferpool.ByteBuffer) {
+	defer func() {
+		atomic.AddInt64(&bufferCounter, -1)
+		// println("-", bufferCounter)
+	}()
+	bytebufferpool.Put(buf)
+}

weed/operation/chunked_file.go

@@ -0,0 +1,238 @@
+package operation
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"sync"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	// when the remote server does not allow range requests (Accept-Ranges was not set)
+	ErrRangeRequestsNotSupported = errors.New("Range requests are not supported by the remote server")
+	// ErrInvalidRange is returned by Read when trying to read past the end of the file
+	ErrInvalidRange = errors.New("Invalid range")
+)
+
+type ChunkInfo struct {
+	Fid    string `json:"fid"`
+	Offset int64  `json:"offset"`
+	Size   int64  `json:"size"`
+}
+
+type ChunkList []*ChunkInfo
+
+type ChunkManifest struct {
+	Name   string    `json:"name,omitempty"`
+	Mime   string    `json:"mime,omitempty"`
+	Size   int64     `json:"size,omitempty"`
+	Chunks ChunkList `json:"chunks,omitempty"`
+}
+
+// seekable chunked file reader
+type ChunkedFileReader struct {
+	totalSize      int64
+	chunkList      []*ChunkInfo
+	master         pb.ServerAddress
+	pos            int64
+	pr             *io.PipeReader
+	pw             *io.PipeWriter
+	mutex          sync.Mutex
+	grpcDialOption grpc.DialOption
+}
+
+func (s ChunkList) Len() int           { return len(s) }
+func (s ChunkList) Less(i, j int) bool { return s[i].Offset < s[j].Offset }
+func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+
+func LoadChunkManifest(buffer []byte, isCompressed bool) (*ChunkManifest, error) {
+	if isCompressed {
+		var err error
+		if buffer, err = util.DecompressData(buffer); err != nil {
+			glog.V(0).Infof("fail to decompress chunk manifest: %v", err)
+		}
+	}
+	cm := ChunkManifest{}
+	if e := json.Unmarshal(buffer, &cm); e != nil {
+		return nil, e
+	}
+	sort.Sort(cm.Chunks)
+	return &cm, nil
+}
+
+func (cm *ChunkManifest) Marshal() ([]byte, error) {
+	return json.Marshal(cm)
+}
+
+func (cm *ChunkManifest) DeleteChunks(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption) error {
+	var fileIds []string
+	for _, ci := range cm.Chunks {
+		fileIds = append(fileIds, ci.Fid)
+	}
+	results := DeleteFileIds(masterFn, usePublicUrl, grpcDialOption, fileIds)
+
+	// Check for any errors in results
+	for _, result := range results {
+		if result.Error != "" {
+			glog.V(0).Infof("delete file %+v: %v", result.FileId, result.Error)
+			return fmt.Errorf("chunk delete %v: %v", result.FileId, result.Error)
+		}
+	}
+
+	return nil
+}
+
+func readChunkNeedle(fileUrl string, w io.Writer, offset int64, jwt string) (written int64, e error) {
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return written, err
+	}
+	if offset > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
+	}
+
+	resp, err := util_http.Do(req)
+	if err != nil {
+		return written, err
+	}
+	defer util_http.CloseResponse(resp)
+
+	switch resp.StatusCode {
+	case http.StatusRequestedRangeNotSatisfiable:
+		return written, ErrInvalidRange
+	case http.StatusOK:
+		if offset > 0 {
+			return written, ErrRangeRequestsNotSupported
+		}
+	case http.StatusPartialContent:
+		break
+	default:
+		return written, fmt.Errorf("Read chunk needle error: [%d] %s", resp.StatusCode, fileUrl)
+
+	}
+	return io.Copy(w, resp.Body)
+}
+
+func NewChunkedFileReader(chunkList []*ChunkInfo, master pb.ServerAddress, grpcDialOption grpc.DialOption) *ChunkedFileReader {
+	var totalSize int64
+	for _, chunk := range chunkList {
+		totalSize += chunk.Size
+	}
+	sort.Sort(ChunkList(chunkList))
+	return &ChunkedFileReader{
+		totalSize:      totalSize,
+		chunkList:      chunkList,
+		master:         master,
+		grpcDialOption: grpcDialOption,
+	}
+}
+
+func (cf *ChunkedFileReader) Seek(offset int64, whence int) (int64, error) {
+	var err error
+	switch whence {
+	case io.SeekStart:
+	case io.SeekCurrent:
+		offset += cf.pos
+	case io.SeekEnd:
+		offset = cf.totalSize + offset
+	}
+	if offset > cf.totalSize {
+		err = ErrInvalidRange
+	}
+	if cf.pos != offset {
+		cf.Close()
+	}
+	cf.pos = offset
+	return cf.pos, err
+}
+
+func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
+	chunkIndex := -1
+	chunkStartOffset := int64(0)
+	for i, ci := range cf.chunkList {
+		if cf.pos >= ci.Offset && cf.pos < ci.Offset+ci.Size {
+			chunkIndex = i
+			chunkStartOffset = cf.pos - ci.Offset
+			break
+		}
+	}
+	if chunkIndex < 0 {
+		return n, ErrInvalidRange
+	}
+	for ; chunkIndex < len(cf.chunkList); chunkIndex++ {
+		ci := cf.chunkList[chunkIndex]
+		// if we need read date from local volume server first?
+		fileUrl, jwt, lookupError := LookupFileId(func(_ context.Context) pb.ServerAddress {
+			return cf.master
+		}, cf.grpcDialOption, ci.Fid)
+		if lookupError != nil {
+			return n, lookupError
+		}
+		if wn, e := readChunkNeedle(fileUrl, w, chunkStartOffset, jwt); e != nil {
+			return n, e
+		} else {
+			n += wn
+			cf.pos += wn
+		}
+
+		chunkStartOffset = 0
+	}
+	return n, nil
+}
+
+func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
+	cf.Seek(off, 0)
+	return cf.Read(p)
+}
+
+func (cf *ChunkedFileReader) Read(p []byte) (int, error) {
+	return cf.getPipeReader().Read(p)
+}
+
+func (cf *ChunkedFileReader) Close() (e error) {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	return cf.closePipe()
+}
+
+func (cf *ChunkedFileReader) closePipe() (e error) {
+	if cf.pr != nil {
+		if err := cf.pr.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pr = nil
+	if cf.pw != nil {
+		if err := cf.pw.Close(); err != nil {
+			e = err
+		}
+	}
+	cf.pw = nil
+	return e
+}
+
+func (cf *ChunkedFileReader) getPipeReader() io.Reader {
+	cf.mutex.Lock()
+	defer cf.mutex.Unlock()
+	if cf.pr != nil && cf.pw != nil {
+		return cf.pr
+	}
+	cf.closePipe()
+	cf.pr, cf.pw = io.Pipe()
+	go func(pw *io.PipeWriter) {
+		_, e := cf.WriteTo(pw)
+		pw.CloseWithError(e)
+	}(cf.pw)
+	return cf.pr
+}

weed/operation/data_struts.go

@@ -0,0 +1,6 @@
+package operation
+
+type JoinResult struct {
+	VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"`
+	Error           string `json:"error,omitempty"`
+}

weed/operation/delete_content.go

@@ -0,0 +1,174 @@
+package operation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+type DeleteResult struct {
+	Fid    string `json:"fid"`
+	Size   int    `json:"size"`
+	Status int    `json:"status"`
+	Error  string `json:"error,omitempty"`
+}
+
+func ParseFileId(fid string) (vid string, key_cookie string, err error) {
+	commaIndex := strings.Index(fid, ",")
+	if commaIndex <= 0 {
+		return "", "", errors.New("Wrong fid format.")
+	}
+	return fid[:commaIndex], fid[commaIndex+1:], nil
+}
+
+// DeleteFileIds batch deletes a list of fileIds
+// Returns individual results for each file ID. Check result.Error for per-file failures.
+func DeleteFileIds(masterFn GetMasterFn, usePublicUrl bool, grpcDialOption grpc.DialOption, fileIds []string) []*volume_server_pb.DeleteResult {
+
+	lookupFunc := func(vids []string) (results map[string]*LookupResult, err error) {
+		results, err = LookupVolumeIds(masterFn, grpcDialOption, vids)
+		if err == nil && usePublicUrl {
+			for _, result := range results {
+				for _, loc := range result.Locations {
+					loc.Url = loc.PublicUrl
+				}
+			}
+		}
+		return
+	}
+
+	return DeleteFileIdsWithLookupVolumeId(grpcDialOption, fileIds, lookupFunc)
+
+}
+
+func DeleteFileIdsWithLookupVolumeId(grpcDialOption grpc.DialOption, fileIds []string, lookupFunc func(vid []string) (map[string]*LookupResult, error)) []*volume_server_pb.DeleteResult {
+
+	var ret []*volume_server_pb.DeleteResult
+
+	vid_to_fileIds := make(map[string][]string)
+	var vids []string
+	for _, fileId := range fileIds {
+		vid, _, err := ParseFileId(fileId)
+		if err != nil {
+			ret = append(ret, &volume_server_pb.DeleteResult{
+				FileId: fileId,
+				Status: http.StatusBadRequest,
+				Error:  err.Error()},
+			)
+			continue
+		}
+		if _, ok := vid_to_fileIds[vid]; !ok {
+			vid_to_fileIds[vid] = make([]string, 0)
+			vids = append(vids, vid)
+		}
+		vid_to_fileIds[vid] = append(vid_to_fileIds[vid], fileId)
+	}
+
+	lookupResults, err := lookupFunc(vids)
+	if err != nil {
+		// Lookup failed - return error results for all file IDs that passed parsing
+		for _, fids := range vid_to_fileIds {
+			for _, fileId := range fids {
+				ret = append(ret, &volume_server_pb.DeleteResult{
+					FileId: fileId,
+					Status: http.StatusInternalServerError,
+					Error:  fmt.Sprintf("lookup error: %v", err),
+				})
+			}
+		}
+		return ret
+	}
+
+	server_to_fileIds := make(map[pb.ServerAddress][]string)
+	for vid, result := range lookupResults {
+		if result.Error != "" {
+			// Lookup error for this volume - mark all its files as failed
+			for _, fileId := range vid_to_fileIds[vid] {
+				ret = append(ret, &volume_server_pb.DeleteResult{
+					FileId: fileId,
+					Status: http.StatusBadRequest,
+					Error:  result.Error},
+				)
+			}
+			continue
+		}
+		for _, location := range result.Locations {
+			serverAddress := location.ServerAddress()
+			if _, ok := server_to_fileIds[serverAddress]; !ok {
+				server_to_fileIds[serverAddress] = make([]string, 0)
+			}
+			server_to_fileIds[serverAddress] = append(
+				server_to_fileIds[serverAddress], vid_to_fileIds[vid]...)
+		}
+	}
+
+	resultChan := make(chan []*volume_server_pb.DeleteResult, len(server_to_fileIds))
+	var wg sync.WaitGroup
+	for server, fidList := range server_to_fileIds {
+		wg.Add(1)
+		go func(server pb.ServerAddress, fidList []string) {
+			defer wg.Done()
+
+			resultChan <- DeleteFileIdsAtOneVolumeServer(server, grpcDialOption, fidList, false)
+
+		}(server, fidList)
+	}
+	wg.Wait()
+	close(resultChan)
+
+	for result := range resultChan {
+		ret = append(ret, result...)
+	}
+
+	return ret
+}
+
+// DeleteFileIdsAtOneVolumeServer deletes a list of files that is on one volume server via gRpc
+// Returns individual results for each file ID. Check result.Error for per-file failures.
+func DeleteFileIdsAtOneVolumeServer(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fileIds []string, includeCookie bool) []*volume_server_pb.DeleteResult {
+
+	var ret []*volume_server_pb.DeleteResult
+
+	err := WithVolumeServerClient(false, volumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+		req := &volume_server_pb.BatchDeleteRequest{
+			FileIds:         fileIds,
+			SkipCookieCheck: !includeCookie,
+		}
+
+		resp, err := volumeServerClient.BatchDelete(context.Background(), req)
+
+		// fmt.Printf("deleted %v %v: %v\n", fileIds, err, resp)
+
+		if err != nil {
+			return err
+		}
+
+		ret = append(ret, resp.Results...)
+
+		return nil
+	})
+
+	if err != nil {
+		// Connection or communication error - return error results for all files
+		ret = make([]*volume_server_pb.DeleteResult, 0, len(fileIds))
+		for _, fileId := range fileIds {
+			ret = append(ret, &volume_server_pb.DeleteResult{
+				FileId: fileId,
+				Status: http.StatusInternalServerError,
+				Error:  err.Error(),
+			})
+		}
+	}
+
+	return ret
+
+}

weed/operation/grpc_client.go

@@ -0,0 +1,28 @@
+package operation
+
+import (
+	"context"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func WithVolumeServerClient(streamingMode bool, volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(volume_server_pb.VolumeServerClient) error) error {
+
+	return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
+		client := volume_server_pb.NewVolumeServerClient(grpcConnection)
+		return fn(client)
+	}, volumeServer.ToGrpcAddress(), false, grpcDialOption)
+
+}
+
+// WithMasterServerClient threads the caller's per-request context into the
+// connection-invalidation decision, so a Canceled/DeadlineExceeded from the
+// caller's own timeout does not invalidate the shared cached master connection.
+// Pass context.Background() when there is no per-request deadline to honor.
+func WithMasterServerClient(ctx context.Context, streamingMode bool, masterServer pb.ServerAddress, grpcDialOption grpc.DialOption, fn func(masterClient master_pb.SeaweedClient) error) error {
+	return pb.WithMasterClient(ctx, streamingMode, masterServer, grpcDialOption, false, fn)
+}

weed/operation/lookup.go

@@ -0,0 +1,158 @@
+package operation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+type Location struct {
+	Url               string `json:"url,omitempty"`
+	PublicUrl         string `json:"publicUrl,omitempty"`
+	DataCenter        string `json:"dataCenter,omitempty"`
+	GrpcPort          int    `json:"grpcPort,omitempty"`
+	DataInRemote      bool   `json:"dataInRemote,omitempty"`
+	ReadOnly          bool   `json:"readOnly,omitempty"`
+	ReadOnlyCanDelete bool   `json:"readOnlyCanDelete,omitempty"`
+}
+
+func (l *Location) ServerAddress() pb.ServerAddress {
+	return pb.NewServerAddressWithGrpcPort(l.Url, l.GrpcPort)
+}
+
+type LookupResult struct {
+	VolumeOrFileId string     `json:"volumeOrFileId,omitempty"`
+	Locations      []Location `json:"locations,omitempty"`
+	Jwt            string     `json:"jwt,omitempty"`
+	Error          string     `json:"error,omitempty"`
+	NotFound       bool       `json:"-"`
+}
+
+func (lr *LookupResult) String() string {
+	return fmt.Sprintf("VolumeOrFileId:%s, Locations:%v, Error:%s", lr.VolumeOrFileId, lr.Locations, lr.Error)
+}
+
+var (
+	vc VidCache // caching of volume locations, re-check if after 10 minutes
+)
+
+// LookupFileId resolves a "<vid>,<cookie>" file id to one HTTP read URL,
+// preferring a volume server whose replica holds the data locally over one
+// backed by remote-tier storage. If no local replica is known the function
+// falls back to a random remote replica. The returned jwt is the read
+// authorization the master stamped on the volume; pass it through to the
+// volume server on the read request.
+func LookupFileId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, fileId string) (fullUrl string, jwt string, err error) {
+	parts := strings.Split(fileId, ",")
+	if len(parts) != 2 {
+		return "", jwt, errors.New("Invalid fileId " + fileId)
+	}
+	lookup, lookupError := LookupVolumeId(masterFn, grpcDialOption, parts[0])
+	if lookupError != nil {
+		return "", jwt, lookupError
+	}
+	if len(lookup.Locations) == 0 {
+		return "", jwt, errors.New("File Not Found")
+	}
+
+	localUrls := make([]string, 0, len(lookup.Locations))
+	for _, loc := range lookup.Locations {
+		if !loc.DataInRemote {
+			localUrls = append(localUrls, loc.Url)
+		}
+	}
+	if len(localUrls) == 0 {
+		for _, loc := range lookup.Locations {
+			localUrls = append(localUrls, loc.Url)
+		}
+	}
+
+	return "http://" + localUrls[rand.IntN(len(localUrls))] + "/" + fileId, lookup.Jwt, nil
+}
+
+func LookupVolumeId(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid string) (*LookupResult, error) {
+	results, err := LookupVolumeIds(masterFn, grpcDialOption, []string{vid})
+	return results[vid], err
+}
+
+// InvalidateVolumeIdLocationCache drops cached locations for vid so the next lookup re-queries the master.
+func InvalidateVolumeIdLocationCache(vid string) {
+	vc.Delete(vid)
+}
+
+// LookupVolumeIds find volume locations by cache and actual lookup
+func LookupVolumeIds(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vids []string, useCache ...bool) (map[string]*LookupResult, error) {
+	ret := make(map[string]*LookupResult)
+	var unknown_vids []string
+	cacheLocations := len(useCache) == 0 || useCache[0]
+
+	//check vid cache first
+	for _, vid := range vids {
+		if cacheLocations {
+			locations, cacheErr := vc.Get(vid)
+			if cacheErr == nil {
+				ret[vid] = &LookupResult{VolumeOrFileId: vid, Locations: locations}
+				continue
+			}
+		}
+		unknown_vids = append(unknown_vids, vid)
+	}
+	//return success if all volume ids are known
+	if len(unknown_vids) == 0 {
+		return ret, nil
+	}
+
+	//only query unknown_vids
+
+	err := WithMasterServerClient(context.Background(), false, masterFn(context.Background()), grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
+
+		req := &master_pb.LookupVolumeRequest{
+			VolumeOrFileIds: unknown_vids,
+		}
+		resp, grpcErr := masterClient.LookupVolume(context.Background(), req)
+		if grpcErr != nil {
+			return grpcErr
+		}
+
+		//set newly checked vids to cache
+		for _, vidLocations := range resp.VolumeIdLocations {
+			var locations []Location
+			for _, loc := range vidLocations.Locations {
+				locations = append(locations, Location{
+					Url:               loc.Url,
+					PublicUrl:         loc.PublicUrl,
+					DataCenter:        loc.DataCenter,
+					GrpcPort:          int(loc.GrpcPort),
+					DataInRemote:      loc.DataInRemote,
+					ReadOnly:          loc.ReadOnly,
+					ReadOnlyCanDelete: loc.ReadOnlyCanDelete,
+				})
+			}
+			if cacheLocations && vidLocations.Error == "" {
+				vc.Set(vidLocations.VolumeOrFileId, locations, 10*time.Minute)
+			}
+			ret[vidLocations.VolumeOrFileId] = &LookupResult{
+				VolumeOrFileId: vidLocations.VolumeOrFileId,
+				Locations:      locations,
+				Jwt:            vidLocations.Auth,
+				Error:          vidLocations.Error,
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}

weed/operation/lookup_cache_invalidation_test.go

@@ -0,0 +1,83 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+type fakeLookupServer struct {
+	master_pb.UnimplementedSeaweedServer
+	mu        sync.Mutex
+	calls     int
+	locations int
+}
+
+func (s *fakeLookupServer) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.calls++
+	resp := &master_pb.LookupVolumeResponse{}
+	for _, vid := range req.VolumeOrFileIds {
+		var locs []*master_pb.Location
+		for i := 0; i < s.locations; i++ {
+			locs = append(locs, &master_pb.Location{Url: fmt.Sprintf("10.0.0.%d:8080", i)})
+		}
+		resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
+			VolumeOrFileId: vid,
+			Locations:      locs,
+		})
+	}
+	return resp, nil
+}
+
+// TestLookupVolumeIdCacheInvalidation reproduces the stale-lookup scenario: a
+// volume that briefly reports too few replicas would be cached for the full TTL,
+// so writes kept failing even after the master re-registered the missing replica.
+// InvalidateVolumeIdLocationCache forces the next lookup to re-query the master.
+func TestLookupVolumeIdCacheInvalidation(t *testing.T) {
+	fake := &fakeLookupServer{locations: 1}
+	master := startFakeMasterServer(t, fake)
+	masterFn := func(context.Context) pb.ServerAddress { return master }
+	dial := grpc.WithTransportCredentials(insecure.NewCredentials())
+
+	const vid = "778899"
+	InvalidateVolumeIdLocationCache(vid)
+
+	// first lookup queries the master and caches the single, under-replicated location
+	r1, err := LookupVolumeId(masterFn, dial, vid)
+	if err != nil {
+		t.Fatalf("first lookup: %v", err)
+	}
+	if len(r1.Locations) != 1 {
+		t.Fatalf("first lookup locations = %d, want 1", len(r1.Locations))
+	}
+
+	// a second lookup is served from cache, so the master is not queried again
+	if _, err := LookupVolumeId(masterFn, dial, vid); err != nil {
+		t.Fatalf("second lookup: %v", err)
+	}
+	fake.mu.Lock()
+	calls := fake.calls
+	fake.locations = 2 // master heals: both replicas are registered again
+	fake.mu.Unlock()
+	if calls != 1 {
+		t.Fatalf("expected a cache hit, master queried %d times", calls)
+	}
+
+	// invalidation drops the stale entry, so the next lookup re-queries and sees both replicas
+	InvalidateVolumeIdLocationCache(vid)
+	r3, err := LookupVolumeId(masterFn, dial, vid)
+	if err != nil {
+		t.Fatalf("third lookup: %v", err)
+	}
+	if len(r3.Locations) != 2 {
+		t.Fatalf("after invalidation locations = %d, want 2", len(r3.Locations))
+	}
+}

weed/operation/lookup_raft_leader.go

@@ -0,0 +1,54 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+)
+
+// LookupRaftLeaderMaster returns the current raft leader by querying configured
+// master peers. Followers answer GetMasterConfiguration with the leader address;
+// topology mutations such as VolumeMarkReadonly must run on that node.
+//
+// The volume server's heartbeat peer (GetMaster) may still be a follower after an
+// election until the heartbeat loop reconnects.
+func LookupRaftLeaderMaster(ctx context.Context, masters []pb.ServerAddress, grpcDialOption grpc.DialOption) (pb.ServerAddress, error) {
+	if len(masters) == 0 {
+		return "", fmt.Errorf("no master peers configured")
+	}
+	var lastErr error
+	for _, peer := range masters {
+		if err := ctx.Err(); err != nil {
+			return "", err
+		}
+		// The same timeout ctx drives both the RPC and WithMasterServerClient's
+		// connection-invalidation decision, so a self-inflicted 5s timeout leaves
+		// callCtx.Err() != nil and does not poison the shared master connection.
+		callCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		var leader pb.ServerAddress
+		err := WithMasterServerClient(callCtx, false, peer, grpcDialOption, func(client master_pb.SeaweedClient) error {
+			resp, err := client.GetMasterConfiguration(callCtx, &master_pb.GetMasterConfigurationRequest{})
+			if err != nil {
+				return err
+			}
+			if resp.Leader == "" {
+				return fmt.Errorf("master %s returned empty leader", peer)
+			}
+			leader = pb.ServerAddress(resp.Leader)
+			return nil
+		})
+		cancel()
+		if err == nil && leader != "" {
+			return leader, nil
+		}
+		lastErr = err
+	}
+	if lastErr != nil {
+		return "", fmt.Errorf("lookup raft leader from %d master peer(s): %w", len(masters), lastErr)
+	}
+	return "", fmt.Errorf("lookup raft leader: no leader from %d master peer(s)", len(masters))
+}

weed/operation/lookup_raft_leader_test.go

@@ -0,0 +1,99 @@
+package operation
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+type fakeLeaderConfigServer struct {
+	master_pb.UnimplementedSeaweedServer
+	leader string
+}
+
+func (s *fakeLeaderConfigServer) GetMasterConfiguration(_ context.Context, _ *master_pb.GetMasterConfigurationRequest) (*master_pb.GetMasterConfigurationResponse, error) {
+	return &master_pb.GetMasterConfigurationResponse{Leader: s.leader}, nil
+}
+
+func startFakeMasterServer(t *testing.T, srv master_pb.SeaweedServer) pb.ServerAddress {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	master_pb.RegisterSeaweedServer(grpcServer, srv)
+	go func() { _ = grpcServer.Serve(lis) }()
+	t.Cleanup(grpcServer.GracefulStop)
+
+	_, port, _ := net.SplitHostPort(lis.Addr().String())
+	return pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))
+}
+
+func TestLookupRaftLeaderMasterFromFollower(t *testing.T) {
+	follower := startFakeMasterServer(t, &fakeLeaderConfigServer{
+		leader: "leader.example.com:9333.19333",
+	})
+
+	leader, err := LookupRaftLeaderMaster(
+		context.Background(),
+		[]pb.ServerAddress{follower},
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err != nil {
+		t.Fatalf("lookup: %v", err)
+	}
+	if got, want := leader.ToHttpAddress(), "leader.example.com:9333"; got != want {
+		t.Fatalf("leader = %q, want %q", got, want)
+	}
+}
+
+func TestLookupRaftLeaderMasterTriesNextPeer(t *testing.T) {
+	down := pb.ServerAddress("127.0.0.1:1")
+	peer := startFakeMasterServer(t, &fakeLeaderConfigServer{
+		leader: "leader.example.com:9333.19333",
+	})
+
+	leader, err := LookupRaftLeaderMaster(
+		context.Background(),
+		[]pb.ServerAddress{down, peer},
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err != nil {
+		t.Fatalf("lookup: %v", err)
+	}
+	if got, want := leader.ToHttpAddress(), "leader.example.com:9333"; got != want {
+		t.Fatalf("leader = %q, want %q", got, want)
+	}
+}
+
+func TestLookupRaftLeaderMasterNoPeers(t *testing.T) {
+	_, err := LookupRaftLeaderMaster(context.Background(), nil, nil)
+	if err == nil {
+		t.Fatal("expected error for empty peers")
+	}
+}
+
+func TestLookupRaftLeaderMasterCanceledContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	_, err := LookupRaftLeaderMaster(
+		ctx,
+		[]pb.ServerAddress{"127.0.0.1:1"},
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err == nil {
+		t.Fatal("expected error for canceled context")
+	}
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+}

weed/operation/lookup_vid_cache.go

@@ -0,0 +1,72 @@
+package operation
+
+import (
+	"errors"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+var ErrorNotFound = errors.New("not found")
+
+type VidInfo struct {
+	Locations       []Location
+	NextRefreshTime time.Time
+}
+type VidCache struct {
+	sync.RWMutex
+	cache map[uint32]VidInfo
+}
+
+func (vc *VidCache) Get(vid string) ([]Location, error) {
+	id, err := strconv.ParseUint(vid, 10, 32)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return nil, err
+	}
+	if id == 0 {
+		return nil, ErrorNotFound
+	}
+	vc.RLock()
+	defer vc.RUnlock()
+	info, found := vc.cache[uint32(id)]
+	if !found || info.Locations == nil {
+		return nil, ErrorNotFound
+	}
+	if info.NextRefreshTime.Before(time.Now()) {
+		return nil, errors.New("expired")
+	}
+	return info.Locations, nil
+}
+
+func (vc *VidCache) Set(vid string, locations []Location, duration time.Duration) {
+	id, err := strconv.ParseUint(vid, 10, 32)
+	if err != nil {
+		glog.V(1).Infof("Unknown volume id %s", vid)
+		return
+	}
+	if id == 0 {
+		return
+	}
+	vc.Lock()
+	defer vc.Unlock()
+	if vc.cache == nil {
+		vc.cache = make(map[uint32]VidInfo)
+	}
+	vc.cache[uint32(id)] = VidInfo{
+		Locations:       locations,
+		NextRefreshTime: time.Now().Add(duration),
+	}
+}
+
+func (vc *VidCache) Delete(vid string) {
+	id, err := strconv.ParseUint(vid, 10, 32)
+	if err != nil || id == 0 {
+		return
+	}
+	vc.Lock()
+	defer vc.Unlock()
+	delete(vc.cache, uint32(id))
+}

weed/operation/lookup_vid_cache_test.go

@@ -0,0 +1,65 @@
+package operation
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestCaching(t *testing.T) {
+	var (
+		vc VidCache
+	)
+	var locations []Location
+	locations = append(locations, Location{Url: "a.com:8080"})
+	vc.Set("123", locations, time.Second)
+	ret, _ := vc.Get("123")
+	if ret == nil {
+		t.Fatal("Not found vid 123")
+	}
+	fmt.Printf("vid 123 locations = %v\n", ret)
+	time.Sleep(2 * time.Second)
+	ret, _ = vc.Get("123")
+	if ret != nil {
+		t.Fatal("Not found vid 123")
+	}
+}
+
+// a stale under-replicated result must not linger for the full cache TTL
+func TestCachingDelete(t *testing.T) {
+	var vc VidCache
+	locations := []Location{{Url: "a.com:8080"}}
+	vc.Set("123", locations, time.Minute)
+	if ret, _ := vc.Get("123"); ret == nil {
+		t.Fatal("expected vid 123 to be cached")
+	}
+	vc.Delete("123")
+	if ret, _ := vc.Get("123"); ret != nil {
+		t.Fatal("expected vid 123 to be evicted after Delete")
+	}
+	// deleting a missing or out-of-range id must not panic
+	vc.Delete("123")
+	vc.Delete("4294967296")
+}
+
+// a single large volume id must not allocate an entry per id below it
+func TestCachingLargeVolumeId(t *testing.T) {
+	var vc VidCache
+	locations := []Location{{Url: "a.com:8080"}}
+	vc.Set("32000000", locations, time.Minute)
+	if got := len(vc.cache); got != 1 {
+		t.Fatalf("expected 1 cached entry, got %d", got)
+	}
+	if ret, _ := vc.Get("32000000"); ret == nil {
+		t.Fatal("Not found vid 32000000")
+	}
+
+	// ids beyond uint32 are not real volume ids and must not wrap into the cache
+	vc.Set("4294967296", locations, time.Minute)
+	if got := len(vc.cache); got != 1 {
+		t.Fatalf("out-of-range id must not be cached, got %d entries", got)
+	}
+	if ret, _ := vc.Get("4294967296"); ret != nil {
+		t.Fatal("out-of-range vid 4294967296 should not be found")
+	}
+}

weed/operation/needle_parse_test.go

@@ -0,0 +1,146 @@
+package operation
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type MockClient struct {
+	needleHandling func(n *needle.Needle, originalSize int, e error)
+}
+
+func (m *MockClient) Do(req *http.Request) (*http.Response, error) {
+	n, originalSize, _, err := needle.CreateNeedleFromRequest(req, false, 1024*1024, &bytes.Buffer{})
+	if m.needleHandling != nil {
+		m.needleHandling(n, originalSize, err)
+	}
+	return &http.Response{
+		StatusCode: http.StatusNoContent,
+	}, io.EOF
+}
+
+/*
+
+The mime type is always the value passed in.
+
+Compress or not depends on the content detection, file name extension, and compression ratio.
+
+If the content is already compressed, need to know the content size.
+
+*/
+
+func TestCreateNeedleFromRequest(t *testing.T) {
+	mockClient := &MockClient{}
+	uploader := newUploader(mockClient)
+
+	{
+		mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+			assert.Equal(t, nil, err, "upload: %v", err)
+			assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+			assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
+			assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
+			fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)
+		}
+		uploadOption := &UploadOption{
+			UploadUrl:         "http://localhost:8080/389,0f084d17353afda0",
+			Filename:          "t.txt",
+			Cipher:            false,
+			IsInputCompressed: false,
+			MimeType:          "",
+			PairMap:           nil,
+			Jwt:               "",
+		}
+		uploadResult, err, data := uploader.Upload(context.Background(), bytes.NewReader([]byte(textContent)), uploadOption)
+		if len(data) != len(textContent) {
+			t.Errorf("data actual %d expected %d", len(data), len(textContent))
+		}
+		if err != nil {
+			fmt.Printf("err: %v\n", err)
+		}
+		fmt.Printf("uploadResult: %+v\n", uploadResult)
+	}
+
+	{
+		mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+			assert.Equal(t, nil, err, "upload: %v", err)
+			assert.Equal(t, "text/plain", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+			assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
+			assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
+			fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+		}
+		gzippedData, _ := util.GzipData([]byte(textContent))
+		uploadOption := &UploadOption{
+			UploadUrl:         "http://localhost:8080/389,0f084d17353afda0",
+			Filename:          "t.txt",
+			Cipher:            false,
+			IsInputCompressed: true,
+			MimeType:          "text/plain",
+			PairMap:           nil,
+			Jwt:               "",
+		}
+		uploader.Upload(context.Background(), bytes.NewReader(gzippedData), uploadOption)
+	}
+
+	/*
+		{
+			mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+				assert.Equal(t, nil, err, "upload: %v", err)
+				assert.Equal(t, "text/plain", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+				assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
+				assert.Equal(t, true, util.IsZstdContent(n.Data), "this should be zstd")
+				fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+			}
+			zstdData, _ := util.ZstdData([]byte(textContent))
+			Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), true, "text/plain", nil, "")
+		}
+
+		{
+			mc.needleHandling = func(n *needle.Needle, originalSize int, err error) {
+				assert.Equal(t, nil, err, "upload: %v", err)
+				assert.Equal(t, "application/zstd", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+				assert.Equal(t, false, n.IsCompressed(), "this should not be compressed")
+				assert.Equal(t, true, util.IsZstdContent(n.Data), "this should still be zstd")
+				fmt.Printf("needle: %v, dataSize:%d originalSize:%d\n", n, len(n.Data), originalSize)
+			}
+			zstdData, _ := util.ZstdData([]byte(textContent))
+			Upload("http://localhost:8080/389,0f084d17353afda0", "t.txt", false, bytes.NewReader(zstdData), false, "application/zstd", nil, "")
+		}
+	*/
+
+}
+
+var textContent = `Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+`

weed/operation/submit.go

@@ -0,0 +1,319 @@
+package operation
+
+import (
+	"context"
+	"io"
+	"math/rand/v2"
+	"mime"
+	"net/url"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+)
+
+type FilePart struct {
+	Reader   io.Reader
+	FileName string
+	FileSize int64
+	MimeType string
+	ModTime  int64 //in seconds
+	Pref     StoragePreference
+	Server   string //this comes from assign result
+	Fid      string //this comes from assign result, but customizable
+	Fsync    bool
+}
+
+type SubmitResult struct {
+	FileName string `json:"fileName,omitempty"`
+	FileUrl  string `json:"url,omitempty"`
+	Fid      string `json:"fid,omitempty"`
+	Size     uint32 `json:"size,omitempty"`
+	Error    string `json:"error,omitempty"`
+}
+
+type StoragePreference struct {
+	Replication string
+	Collection  string
+	DataCenter  string
+	Ttl         string
+	DiskType    string
+	MaxMB       int
+}
+
+type GetMasterFn func(ctx context.Context) pb.ServerAddress
+
+func SubmitFiles(masterFn GetMasterFn, grpcDialOption grpc.DialOption, files []*FilePart, pref StoragePreference, usePublicUrl bool) ([]SubmitResult, error) {
+	results := make([]SubmitResult, len(files))
+	var totalBytes int64
+	for index, file := range files {
+		results[index].FileName = file.FileName
+		totalBytes += file.FileSize
+	}
+	var avgBytes uint64
+	if n := len(files); n > 0 {
+		avgBytes = uint64((totalBytes + int64(n) - 1) / int64(n))
+	}
+	ar := &VolumeAssignRequest{
+		Count:            uint64(len(files)),
+		Replication:      pref.Replication,
+		Collection:       pref.Collection,
+		DataCenter:       pref.DataCenter,
+		Ttl:              pref.Ttl,
+		DiskType:         pref.DiskType,
+		ExpectedDataSize: avgBytes,
+	}
+	ret, err := Assign(context.Background(), masterFn, grpcDialOption, ar)
+	if err != nil {
+		for index := range files {
+			results[index].Error = err.Error()
+		}
+		return results, err
+	}
+	for index, file := range files {
+		file.Fid = ret.Fid
+		if index > 0 {
+			file.Fid = file.Fid + "_" + strconv.Itoa(index)
+		}
+		file.Server = ret.Url
+		if usePublicUrl {
+			file.Server = ret.PublicUrl
+		}
+		file.Pref = pref
+		results[index].Size, err = file.Upload(pref.MaxMB, masterFn, usePublicUrl, ret.Auth, grpcDialOption)
+		if err != nil {
+			results[index].Error = err.Error()
+		}
+		results[index].Fid = file.Fid
+		results[index].FileUrl = ret.PublicUrl + "/" + file.Fid
+	}
+	return results, nil
+}
+
+func NewFileParts(fullPathFilenames []string) (ret []*FilePart, err error) {
+	ret = make([]*FilePart, len(fullPathFilenames))
+	for index, file := range fullPathFilenames {
+		if ret[index], err = newFilePart(file); err != nil {
+			return
+		}
+	}
+	return
+}
+func newFilePart(fullPathFilename string) (ret *FilePart, err error) {
+	ret = &FilePart{}
+	fh, openErr := os.Open(fullPathFilename)
+	if openErr != nil {
+		glog.V(0).Info("Failed to open file: ", fullPathFilename)
+		return ret, openErr
+	}
+	ret.Reader = fh
+
+	fi, fiErr := fh.Stat()
+	if fiErr != nil {
+		glog.V(0).Info("Failed to stat file:", fullPathFilename)
+		return ret, fiErr
+	}
+	ret.ModTime = fi.ModTime().UTC().Unix()
+	ret.FileSize = fi.Size()
+	ext := strings.ToLower(path.Ext(fullPathFilename))
+	ret.FileName = fi.Name()
+	if ext != "" {
+		ret.MimeType = mime.TypeByExtension(ext)
+	}
+
+	return ret, nil
+}
+
+func (fi *FilePart) Upload(maxMB int, masterFn GetMasterFn, usePublicUrl bool, jwt security.EncodedJwt, grpcDialOption grpc.DialOption) (retSize uint32, err error) {
+	fileUrl := "http://" + fi.Server + "/" + fi.Fid
+	if fi.ModTime != 0 {
+		fileUrl += "?ts=" + strconv.Itoa(int(fi.ModTime))
+	}
+	if fi.Fsync {
+		fileUrl += "?fsync=true"
+	}
+	if closer, ok := fi.Reader.(io.Closer); ok {
+		defer closer.Close()
+	}
+	baseName := path.Base(fi.FileName)
+	if maxMB > 0 && fi.FileSize > int64(maxMB*1024*1024) {
+		chunkSize := int64(maxMB * 1024 * 1024)
+		chunks := fi.FileSize/chunkSize + 1
+		cm := ChunkManifest{
+			Name:   baseName,
+			Size:   fi.FileSize,
+			Mime:   fi.MimeType,
+			Chunks: make([]*ChunkInfo, 0, chunks),
+		}
+
+		var ret *AssignResult
+		var id string
+		if fi.Pref.DataCenter != "" {
+			ar := &VolumeAssignRequest{
+				Count:            uint64(chunks),
+				Replication:      fi.Pref.Replication,
+				Collection:       fi.Pref.Collection,
+				Ttl:              fi.Pref.Ttl,
+				DiskType:         fi.Pref.DiskType,
+				ExpectedDataSize: uint64(chunkSize),
+			}
+			ret, err = Assign(context.Background(), masterFn, grpcDialOption, ar)
+			if err != nil {
+				return
+			}
+		}
+		for i := int64(0); i < chunks; i++ {
+			if fi.Pref.DataCenter == "" {
+				remaining := fi.FileSize - i*chunkSize
+				thisChunk := chunkSize
+				if remaining < thisChunk {
+					thisChunk = remaining
+				}
+				ar := &VolumeAssignRequest{
+					Count:            1,
+					Replication:      fi.Pref.Replication,
+					Collection:       fi.Pref.Collection,
+					Ttl:              fi.Pref.Ttl,
+					DiskType:         fi.Pref.DiskType,
+					ExpectedDataSize: uint64(thisChunk),
+				}
+				ret, err = Assign(context.Background(), masterFn, grpcDialOption, ar)
+				if err != nil {
+					// delete all uploaded chunks
+					cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
+					return
+				}
+				id = ret.Fid
+			} else {
+				id = ret.Fid
+				if i > 0 {
+					id += "_" + strconv.FormatInt(i, 10)
+				}
+			}
+			fileUrl := genFileUrl(ret, id, usePublicUrl)
+			count, e := uploadOneChunk(
+				baseName+"-"+strconv.FormatInt(i+1, 10),
+				io.LimitReader(fi.Reader, chunkSize),
+				masterFn, fileUrl,
+				ret.Auth)
+			if e != nil {
+				// delete all uploaded chunks
+				cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
+				return 0, e
+			}
+			cm.Chunks = append(cm.Chunks,
+				&ChunkInfo{
+					Offset: i * chunkSize,
+					Size:   int64(count),
+					Fid:    id,
+				},
+			)
+			retSize += count
+		}
+		err = uploadChunkedFileManifest(fileUrl, &cm, jwt)
+		if err != nil {
+			// delete all uploaded chunks
+			cm.DeleteChunks(masterFn, usePublicUrl, grpcDialOption)
+		}
+	} else {
+		uploadOption := &UploadOption{
+			UploadUrl:         fileUrl,
+			Filename:          baseName,
+			Cipher:            false,
+			IsInputCompressed: false,
+			MimeType:          fi.MimeType,
+			PairMap:           nil,
+			Jwt:               jwt,
+		}
+
+		uploader, e := NewUploader()
+		if e != nil {
+			return 0, e
+		}
+
+		ret, e, _ := uploader.Upload(context.Background(), fi.Reader, uploadOption)
+		if e != nil {
+			return 0, e
+		}
+		return ret.Size, e
+	}
+	return
+}
+
+func genFileUrl(ret *AssignResult, id string, usePublicUrl bool) string {
+	fileUrl := "http://" + ret.Url + "/" + id
+	if usePublicUrl {
+		fileUrl = "http://" + ret.PublicUrl + "/" + id
+	}
+	for _, replica := range ret.Replicas {
+		if rand.IntN(len(ret.Replicas)+1) == 0 {
+			fileUrl = "http://" + replica.Url + "/" + id
+			if usePublicUrl {
+				fileUrl = "http://" + replica.PublicUrl + "/" + id
+			}
+		}
+	}
+	return fileUrl
+}
+
+func uploadOneChunk(filename string, reader io.Reader, masterFn GetMasterFn,
+	fileUrl string, jwt security.EncodedJwt,
+) (size uint32, e error) {
+	glog.V(4).Info("Uploading part ", filename, " to ", fileUrl, "...")
+	uploadOption := &UploadOption{
+		UploadUrl:         fileUrl,
+		Filename:          filename,
+		Cipher:            false,
+		IsInputCompressed: false,
+		MimeType:          "",
+		PairMap:           nil,
+		Jwt:               jwt,
+	}
+
+	uploader, uploaderError := NewUploader()
+	if uploaderError != nil {
+		return 0, uploaderError
+	}
+
+	uploadResult, uploadError, _ := uploader.Upload(context.Background(), reader, uploadOption)
+	if uploadError != nil {
+		return 0, uploadError
+	}
+	return uploadResult.Size, nil
+}
+
+func uploadChunkedFileManifest(fileUrl string, manifest *ChunkManifest, jwt security.EncodedJwt) error {
+	buf, e := manifest.Marshal()
+	if e != nil {
+		return e
+	}
+	glog.V(4).Info("Uploading chunks manifest ", manifest.Name, " to ", fileUrl, "...")
+	u, _ := url.Parse(fileUrl)
+	q := u.Query()
+	q.Set("cm", "true")
+	u.RawQuery = q.Encode()
+	uploadOption := &UploadOption{
+		UploadUrl:         u.String(),
+		Filename:          manifest.Name,
+		Cipher:            false,
+		IsInputCompressed: false,
+		MimeType:          "application/json",
+		PairMap:           nil,
+		Jwt:               jwt,
+	}
+
+	uploader, e := NewUploader()
+	if e != nil {
+		return e
+	}
+
+	_, e = uploader.UploadData(context.Background(), buf, uploadOption)
+	return e
+}

weed/operation/sync_volume.go

@@ -0,0 +1,22 @@
+package operation
+
+import (
+	"context"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+)
+
+func GetVolumeSyncStatus(server pb.ServerAddress, grpcDialOption grpc.DialOption, vid uint32) (resp *volume_server_pb.VolumeSyncStatusResponse, err error) {
+
+	WithVolumeServerClient(false, server, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+
+		resp, err = client.VolumeSyncStatus(context.Background(), &volume_server_pb.VolumeSyncStatusRequest{
+			VolumeId: vid,
+		})
+		return nil
+	})
+
+	return
+}

weed/operation/tail_volume.go

@@ -0,0 +1,94 @@
+package operation
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TailVolume(masterFn GetMasterFn, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, timeoutSeconds int, fn func(n *needle.Needle) error) error {
+	// find volume location, replication, ttl info
+	lookup, err := LookupVolumeId(masterFn, grpcDialOption, vid.String())
+	if err != nil {
+		return fmt.Errorf("look up volume %d: %v", vid, err)
+	}
+	if len(lookup.Locations) == 0 {
+		return fmt.Errorf("unable to locate volume %d", vid)
+	}
+
+	volumeServer := lookup.Locations[0].ServerAddress()
+
+	return TailVolumeFromSource(volumeServer, grpcDialOption, vid, sinceNs, timeoutSeconds, fn)
+}
+
+func TailVolumeFromSource(volumeServer pb.ServerAddress, grpcDialOption grpc.DialOption, vid needle.VolumeId, sinceNs uint64, idleTimeoutSeconds int, fn func(n *needle.Needle) error) error {
+	return WithVolumeServerClient(true, volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		stream, err := client.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
+			VolumeId:           uint32(vid),
+			SinceNs:            sinceNs,
+			IdleTimeoutSeconds: uint32(idleTimeoutSeconds),
+		})
+		if err != nil {
+			return err
+		}
+
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				} else {
+					return recvErr
+				}
+			}
+
+			needleHeader := resp.NeedleHeader
+			needleBody := resp.NeedleBody
+			version := needle.Version(resp.Version)
+			if version == 0 {
+				version = needle.GetCurrentVersion()
+			}
+
+			if len(needleHeader) == 0 {
+				continue
+			}
+
+			for !resp.IsLastChunk {
+				resp, recvErr = stream.Recv()
+				if recvErr != nil {
+					if recvErr == io.EOF {
+						break
+					} else {
+						return recvErr
+					}
+				}
+				needleBody = append(needleBody, resp.NeedleBody...)
+			}
+
+			n := new(needle.Needle)
+			n.ParseNeedleHeader(needleHeader)
+			err = n.ReadNeedleBodyBytes(needleBody, version)
+			if err != nil {
+				return err
+			}
+
+			err = fn(n)
+
+			if err != nil {
+				return err
+			}
+
+		}
+		return nil
+	})
+}

weed/operation/upload_chunked.go

@@ -0,0 +1,431 @@
+package operation
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// ErrTruncatedBody tags a source read that ended before the expected bytes
+// arrived, so callers can tell a truncated input (client abort, reverse-proxy
+// timeout) apart from a volume-server upload fault.
+var ErrTruncatedBody = errors.New("truncated request body")
+
+// ChunkedUploadResult contains the result of a chunked upload
+type ChunkedUploadResult struct {
+	FileChunks   []*filer_pb.FileChunk
+	Md5Hash      hash.Hash
+	TotalSize    int64
+	SmallContent []byte // For files smaller than threshold
+}
+
+// ChunkedUploadOption contains options for chunked uploads
+type ChunkedUploadOption struct {
+	ChunkSize       int32
+	SmallFileLimit  int64
+	Collection      string
+	Replication     string
+	DataCenter      string
+	SaveSmallInline bool
+	Jwt             security.EncodedJwt
+	MimeType        string
+	Cipher          bool // encrypt data on volume servers
+	AssignFunc      func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error)
+	UploadFunc      func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) // Optional: for testing
+}
+
+var chunkBufferPool = sync.Pool{
+	New: func() interface{} {
+		return new(bytes.Buffer)
+	},
+}
+
+// UploadReaderInChunks reads from reader and uploads in chunks to volume servers
+// This prevents OOM by processing the stream in fixed-size chunks
+// Returns file chunks, MD5 hash, total size, and any small content stored inline
+func UploadReaderInChunks(ctx context.Context, reader io.Reader, opt *ChunkedUploadOption) (*ChunkedUploadResult, error) {
+
+	md5Hash := md5.New()
+	var partReader = io.TeeReader(reader, md5Hash)
+
+	var fileChunks []*filer_pb.FileChunk
+	var fileChunksLock sync.Mutex
+	var uploadErr error
+	var uploadErrLock sync.Mutex
+	var chunkOffset int64 = 0
+
+	var wg sync.WaitGroup
+	const bytesBufferCounter = 4
+	bytesBufferLimitChan := make(chan struct{}, bytesBufferCounter)
+
+	// objectFailed reports whether another chunk has already doomed the upload,
+	// so an in-flight chunk stops spending its retry budget on bytes nobody
+	// will reference.
+	objectFailed := func() bool {
+		uploadErrLock.Lock()
+		defer uploadErrLock.Unlock()
+		return uploadErr != nil
+	}
+
+uploadLoop:
+	for {
+		// Throttle buffer usage
+		bytesBufferLimitChan <- struct{}{}
+
+		// Check for errors from parallel uploads
+		uploadErrLock.Lock()
+		if uploadErr != nil {
+			<-bytesBufferLimitChan
+			uploadErrLock.Unlock()
+			break
+		}
+		uploadErrLock.Unlock()
+
+		// Check for context cancellation
+		select {
+		case <-ctx.Done():
+			<-bytesBufferLimitChan
+			uploadErrLock.Lock()
+			if uploadErr == nil {
+				uploadErr = ctx.Err()
+			}
+			uploadErrLock.Unlock()
+			break uploadLoop
+		default:
+		}
+
+		// Get buffer from pool
+		bytesBuffer := chunkBufferPool.Get().(*bytes.Buffer)
+		limitedReader := io.LimitReader(partReader, int64(opt.ChunkSize))
+		bytesBuffer.Reset()
+
+		// Read one chunk
+		dataSize, err := bytesBuffer.ReadFrom(limitedReader)
+		if err != nil {
+			// Attach offset + bytes-read to distinguish client disconnect
+			// before any data (offset=0,got=0) from mid-stream truncation.
+			// A bare io.ErrUnexpectedEOF is not actionable on its own (see #9149).
+			wrapped := fmt.Errorf("read chunk at offset %d (got %d bytes): %w", chunkOffset, dataSize, err)
+			if errors.Is(err, io.ErrUnexpectedEOF) {
+				wrapped = fmt.Errorf("%w: %w", ErrTruncatedBody, wrapped)
+			}
+			glog.V(2).Infof("UploadReaderInChunks: %v", wrapped)
+			chunkBufferPool.Put(bytesBuffer)
+			<-bytesBufferLimitChan
+			uploadErrLock.Lock()
+			if uploadErr == nil {
+				uploadErr = wrapped
+			}
+			uploadErrLock.Unlock()
+			break
+		}
+		// If no data was read, we've reached EOF
+		// Only break if we've already read some data (chunkOffset > 0) or if this is truly EOF
+		if dataSize == 0 {
+			if chunkOffset == 0 {
+				// Empty objects are valid for S3/HTTP uploads (e.g. zero-byte files).
+				// Keep this at verbose level to avoid warning noise in normal operation.
+				glog.V(4).Infof("UploadReaderInChunks: received 0 bytes on first read - creating empty file")
+			}
+			chunkBufferPool.Put(bytesBuffer)
+			<-bytesBufferLimitChan
+			// If we've already read some chunks, this is normal EOF
+			// If we haven't read anything yet (chunkOffset == 0), this could be an empty file
+			// which is valid (e.g., touch command creates 0-byte files)
+			break
+		}
+
+		// For small files at offset 0, store inline instead of uploading
+		if chunkOffset == 0 && opt.SaveSmallInline && dataSize < opt.SmallFileLimit {
+			smallContent := make([]byte, dataSize)
+			n, readErr := io.ReadFull(bytesBuffer, smallContent)
+			chunkBufferPool.Put(bytesBuffer)
+			<-bytesBufferLimitChan
+
+			if readErr != nil {
+				return nil, fmt.Errorf("failed to read small content: read %d of %d bytes: %w", n, dataSize, readErr)
+			}
+
+			return &ChunkedUploadResult{
+				FileChunks:   nil,
+				Md5Hash:      md5Hash,
+				TotalSize:    dataSize,
+				SmallContent: smallContent,
+			}, nil
+		}
+
+		// Upload chunk in parallel goroutine
+		wg.Add(1)
+		go func(offset int64, buf *bytes.Buffer, size int64) {
+			defer func() {
+				chunkBufferPool.Put(buf)
+				<-bytesBufferLimitChan
+				wg.Done()
+			}()
+
+			// Assign volume for this chunk
+			_, assignResult, assignErr := opt.AssignFunc(ctx, 1, uint64(size))
+			if assignErr != nil {
+				uploadErrLock.Lock()
+				if uploadErr == nil {
+					uploadErr = fmt.Errorf("assign volume: %w", assignErr)
+				}
+				uploadErrLock.Unlock()
+				return
+			}
+
+			// Use per-assignment JWT if present, otherwise fall back to the original JWT
+			// This is critical for secured clusters where each volume assignment has its own JWT
+			jwt := opt.Jwt
+			if assignResult.Auth != "" {
+				jwt = assignResult.Auth
+			}
+
+			// Calculate MD5 for the chunk
+			chunkMd5 := md5.Sum(buf.Bytes())
+			chunkMd5B64 := base64.StdEncoding.EncodeToString(chunkMd5[:])
+
+			var uploadResult *UploadResult
+			var uploadResultErr error
+
+			// A target that fills up, loses its replica peer, or goes away fails
+			// every attempt against the same fid, so ask for a fresh assignment and
+			// retry there rather than losing the whole object to one bad volume.
+			for attempt := 1; ; attempt++ {
+				uploadResult, uploadResultErr = uploadChunk(ctx, assignResult, buf.Bytes(), jwt, chunkMd5B64, opt)
+				if uploadResultErr == nil {
+					break
+				}
+				// The volume server commits the needle before replicating, so a
+				// failed write can still leave a copy behind. No chunk will name
+				// this fid whether we retry or give up here, and an unreferenced
+				// needle is not garbage vacuum can find, so drop it either way.
+				deleteChunkFromHolders(chunkHolders(assignResult), assignResult.Fid, jwt)
+				if attempt == chunkAssignAttempts || !ShouldReassignUpload(uploadResultErr) || objectFailed() {
+					break
+				}
+				glog.V(2).Infof("re-assigning chunk at offset %d after attempt %d/%d: %v", offset, attempt, chunkAssignAttempts, uploadResultErr)
+				_, assignResult, assignErr = opt.AssignFunc(ctx, 1, uint64(size))
+				if assignErr != nil {
+					uploadResultErr = fmt.Errorf("reassign volume after %w: %w", uploadResultErr, assignErr)
+					break
+				}
+				jwt = opt.Jwt
+				if assignResult.Auth != "" {
+					jwt = assignResult.Auth
+				}
+			}
+
+			if uploadResultErr != nil {
+				uploadErrLock.Lock()
+				if uploadErr == nil {
+					uploadErr = fmt.Errorf("upload chunk: %w", uploadResultErr)
+				}
+				uploadErrLock.Unlock()
+				return
+			}
+
+			// Create chunk entry
+			// Set ModifiedTsNs to current time (nanoseconds) to track when upload completed
+			// This is critical for multipart uploads where the same part may be uploaded multiple times
+			// The part with the latest ModifiedTsNs is selected as the authoritative version
+			fid, _ := filer_pb.ToFileIdObject(assignResult.Fid)
+			chunk := &filer_pb.FileChunk{
+				FileId:       assignResult.Fid,
+				Offset:       offset,
+				Size:         uint64(uploadResult.Size),
+				ModifiedTsNs: time.Now().UnixNano(),
+				ETag:         uploadResult.ContentMd5,
+				Fid:          fid,
+				CipherKey:    uploadResult.CipherKey,
+				IsCompressed: uploadResult.Gzip > 0,
+			}
+			fileChunksLock.Lock()
+			fileChunks = append(fileChunks, chunk)
+			fileChunksLock.Unlock()
+
+		}(chunkOffset, bytesBuffer, dataSize)
+
+		// Update offset for next chunk
+		chunkOffset += dataSize
+
+		// If this was a partial chunk, we're done
+		if dataSize < int64(opt.ChunkSize) {
+			break
+		}
+	}
+
+	// Wait for all uploads to complete
+	wg.Wait()
+
+	// Sort chunks by offset (do this even if there's an error, for cleanup purposes)
+	sort.Slice(fileChunks, func(i, j int) bool {
+		return fileChunks[i].Offset < fileChunks[j].Offset
+	})
+
+	// Check for errors - return partial results for cleanup
+	if uploadErr != nil {
+		glog.Errorf("chunked upload failed: %v (returning %d partial chunks for cleanup)", uploadErr, len(fileChunks))
+		// IMPORTANT: Return partial results even on error so caller can cleanup orphaned chunks
+		return &ChunkedUploadResult{
+			FileChunks:   fileChunks,
+			Md5Hash:      md5Hash,
+			TotalSize:    chunkOffset,
+			SmallContent: nil,
+		}, uploadErr
+	}
+
+	return &ChunkedUploadResult{
+		FileChunks:   fileChunks,
+		Md5Hash:      md5Hash,
+		TotalSize:    chunkOffset,
+		SmallContent: nil,
+	}, nil
+}
+
+// chunkAssignAttempts bounds how many volumes one chunk may be offered to
+// before the upload gives up.
+const chunkAssignAttempts = 3
+
+// uploadChunk writes one chunk to its assigned volume. It fans out to every
+// holder, except for cipher: per-call encryption would give each replica
+// different bytes, so keep its relay path.
+func uploadChunk(ctx context.Context, assignResult *AssignResult, data []byte, jwt security.EncodedJwt, md5b64 string, opt *ChunkedUploadOption) (*UploadResult, error) {
+	holders := chunkHolders(assignResult)
+	if opt.UploadFunc == nil && !opt.Cipher && len(holders) > 1 {
+		return uploadChunkToHolders(ctx, holders, assignResult.Fid, data, jwt, md5b64, assignResult.Fsync, opt)
+	}
+	uploadOption := &UploadOption{
+		UploadUrl:         fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid),
+		Cipher:            opt.Cipher,
+		IsInputCompressed: false,
+		MimeType:          opt.MimeType,
+		PairMap:           nil,
+		Jwt:               jwt,
+		Md5:               md5b64,
+		// One upload call per assignment: the caller retries everything a
+		// same-URL retry would, and a different volume is the better second try.
+		// This bounds retriedUploadData only — doUploadData still reissues once
+		// on a connection reset, with a rewound body, which is a transport
+		// stutter rather than a fresh attempt at the volume. The fan-out path
+		// keeps its retries, where absorbing a blip locally beats cancelling
+		// every holder and re-uploading the chunk.
+		MaxAttempts: 1,
+	}
+	if assignResult.Fsync {
+		uploadOption.UploadUrl += "?fsync=true"
+	}
+	// Use mock upload function if provided (for testing), otherwise use real uploader
+	if opt.UploadFunc != nil {
+		return opt.UploadFunc(ctx, data, uploadOption)
+	}
+	uploader, err := NewUploader()
+	if err != nil {
+		return nil, fmt.Errorf("create uploader: %w", err)
+	}
+	return uploader.UploadData(ctx, data, uploadOption)
+}
+
+// chunkHolders returns the assigned volume plus its replica holders.
+func chunkHolders(assignResult *AssignResult) []string {
+	hosts := []string{assignResult.Url}
+	for _, replica := range assignResult.Replicas {
+		if replica.Url != "" && replica.Url != assignResult.Url {
+			hosts = append(hosts, replica.Url)
+		}
+	}
+	return hosts
+}
+
+// uploadChunkToHolders writes the chunk to every holder concurrently (each with
+// type=replicate). On the first failure it cancels the remaining uploads and
+// deletes any copies that already landed, so a partial fan-out leaves no
+// orphaned needle the caller cannot see.
+func uploadChunkToHolders(ctx context.Context, hosts []string, fid string, data []byte, jwt security.EncodedJwt, md5b64 string, fsync bool, opt *ChunkedUploadOption) (*UploadResult, error) {
+	uploader, err := NewUploader()
+	if err != nil {
+		return nil, fmt.Errorf("create uploader: %w", err)
+	}
+	glog.V(4).Infof("replica fan-out: writing chunk %s to %d holders %v", fid, len(hosts), hosts)
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	type outcome struct {
+		host   string
+		result *UploadResult
+		err    error
+	}
+	outcomes := make(chan outcome, len(hosts))
+	for _, host := range hosts {
+		go func(host string) {
+			uploadUrl := fmt.Sprintf("http://%s/%s?type=replicate", host, fid)
+			if fsync {
+				uploadUrl += "&fsync=true"
+			}
+			uploadOption := &UploadOption{
+				UploadUrl:         uploadUrl,
+				Cipher:            false,
+				IsInputCompressed: false,
+				MimeType:          opt.MimeType,
+				PairMap:           nil,
+				Jwt:               jwt,
+				Md5:               md5b64,
+			}
+			r, e := uploader.UploadData(ctx, data, uploadOption)
+			outcomes <- outcome{host, r, e}
+		}(host)
+	}
+	var first *UploadResult
+	var firstErr error
+	var succeeded []string
+	for range hosts {
+		o := <-outcomes
+		if o.err != nil {
+			// Once one holder fails the rest are cancelled, so errors arrive in
+			// no fixed order. Prefer one the caller can act on, or the choice of
+			// which host to report — and whether to retry elsewhere — turns on
+			// goroutine scheduling.
+			if firstErr == nil {
+				firstErr = o.err
+				cancel()
+			} else if !ShouldReassignUpload(firstErr) && ShouldReassignUpload(o.err) {
+				firstErr = o.err
+			}
+		} else {
+			succeeded = append(succeeded, o.host)
+			if first == nil {
+				first = o.result
+			}
+		}
+	}
+	if firstErr != nil {
+		// A failed fan-out records no chunk, so roll back the copies that landed
+		// before the cancel rather than leaking them as orphans.
+		deleteChunkFromHolders(succeeded, fid, jwt)
+		return nil, firstErr
+	}
+	return first, nil
+}
+
+// deleteChunkFromHolders best-effort removes a needle from each holder it landed
+// on, using type=replicate so the volume drops only its local copy. A failed
+// delete falls back to vacuum reclaiming the orphan.
+func deleteChunkFromHolders(hosts []string, fid string, jwt security.EncodedJwt) {
+	for _, host := range hosts {
+		if err := util_http.Delete(fmt.Sprintf("http://%s/%s?type=replicate", host, fid), string(jwt)); err != nil {
+			glog.Warningf("replica fan-out cleanup: delete %s from %s: %v", fid, host, err)
+		}
+	}
+}

weed/operation/upload_chunked_test.go

@@ -0,0 +1,681 @@
+package operation
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+)
+
+// TestUploadReaderInChunksReturnsPartialResultsOnError verifies that when
+// UploadReaderInChunks fails mid-upload, it returns partial results containing
+// the chunks that were successfully uploaded before the error occurred.
+// This allows the caller to cleanup orphaned chunks and prevent resource leaks.
+func TestUploadReaderInChunksReturnsPartialResultsOnError(t *testing.T) {
+	// Create test data larger than one chunk to force multiple chunk uploads
+	testData := bytes.Repeat([]byte("test data for chunk upload failure testing"), 1000) // ~40KB
+	reader := bytes.NewReader(testData)
+
+	uploadAttempts := 0
+
+	// Create a mock assign function that succeeds for first chunk, then fails
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		uploadAttempts++
+
+		if uploadAttempts == 1 {
+			// First chunk succeeds
+			return nil, &AssignResult{
+				Fid:       "test-fid-1,1234",
+				Url:       "http://test-volume-1:8080",
+				PublicUrl: "http://test-volume-1:8080",
+				Count:     1,
+			}, nil
+		}
+
+		// Second chunk fails (simulating volume server down or network error)
+		return nil, nil, errors.New("simulated volume assignment failure")
+	}
+
+	// Mock upload function that simulates successful upload
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return &UploadResult{
+			Name:       "test-file",
+			Size:       uint32(len(data)),
+			ContentMd5: "mock-md5-hash",
+			Error:      "",
+		}, nil
+	}
+
+	// Attempt upload with small chunk size to trigger multiple uploads
+	result, err := UploadReaderInChunks(context.Background(), reader, &ChunkedUploadOption{
+		ChunkSize:       8 * 1024, // 8KB chunks
+		SmallFileLimit:  256,
+		Collection:      "test",
+		DataCenter:      "",
+		SaveSmallInline: false,
+		AssignFunc:      assignFunc,
+		UploadFunc:      uploadFunc,
+	})
+
+	// VERIFICATION 1: Error should be returned
+	if err == nil {
+		t.Fatal("Expected error from UploadReaderInChunks, got nil")
+	}
+	t.Logf("✓ Got expected error: %v", err)
+
+	// VERIFICATION 2: Result should NOT be nil (this is the fix)
+	if result == nil {
+		t.Fatal("CRITICAL: UploadReaderInChunks returned nil result on error - caller cannot cleanup orphaned chunks!")
+	}
+	t.Log("✓ Result is not nil (partial results returned)")
+
+	// VERIFICATION 3: Result should contain partial chunks from successful uploads
+	// Note: In reality, the first chunk upload would succeed before assignment fails for chunk 2
+	// But in this test, assignment fails immediately for chunk 2, so we may have 0 chunks
+	// The important thing is that the result struct is returned, not that it has chunks
+	t.Logf("✓ Result contains %d chunks (may be 0 if all assignments failed)", len(result.FileChunks))
+
+	// VERIFICATION 4: MD5 hash should be available even on partial failure
+	if result.Md5Hash == nil {
+		t.Error("Expected Md5Hash to be non-nil")
+	} else {
+		t.Log("✓ Md5Hash is available for partial data")
+	}
+
+	// VERIFICATION 5: TotalSize should reflect bytes read before failure
+	if result.TotalSize < 0 {
+		t.Errorf("Expected non-negative TotalSize, got %d", result.TotalSize)
+	} else {
+		t.Logf("✓ TotalSize = %d bytes read before failure", result.TotalSize)
+	}
+}
+
+// TestUploadReaderInChunksSuccessPath verifies normal successful upload behavior
+func TestUploadReaderInChunksSuccessPath(t *testing.T) {
+	testData := []byte("small test data")
+	reader := bytes.NewReader(testData)
+
+	// Mock assign function that always succeeds
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		return nil, &AssignResult{
+			Fid:       "test-fid,1234",
+			Url:       "http://test-volume:8080",
+			PublicUrl: "http://test-volume:8080",
+			Count:     1,
+		}, nil
+	}
+
+	// Mock upload function that simulates successful upload
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return &UploadResult{
+			Name:       "test-file",
+			Size:       uint32(len(data)),
+			ContentMd5: "mock-md5-hash",
+			Error:      "",
+		}, nil
+	}
+
+	result, err := UploadReaderInChunks(context.Background(), reader, &ChunkedUploadOption{
+		ChunkSize:       8 * 1024,
+		SmallFileLimit:  256,
+		Collection:      "test",
+		DataCenter:      "",
+		SaveSmallInline: false,
+		AssignFunc:      assignFunc,
+		UploadFunc:      uploadFunc,
+	})
+
+	// VERIFICATION 1: No error should occur
+	if err != nil {
+		t.Fatalf("Expected successful upload, got error: %v", err)
+	}
+	t.Log("✓ Upload completed without error")
+
+	// VERIFICATION 2: Result should not be nil
+	if result == nil {
+		t.Fatal("Expected non-nil result")
+	}
+	t.Log("✓ Result is not nil")
+
+	// VERIFICATION 3: Should have file chunks
+	if len(result.FileChunks) == 0 {
+		t.Error("Expected at least one file chunk")
+	} else {
+		t.Logf("✓ Result contains %d file chunk(s)", len(result.FileChunks))
+	}
+
+	// VERIFICATION 4: Total size should match input data
+	if result.TotalSize != int64(len(testData)) {
+		t.Errorf("Expected TotalSize=%d, got %d", len(testData), result.TotalSize)
+	} else {
+		t.Logf("✓ TotalSize=%d matches input data", result.TotalSize)
+	}
+
+	// VERIFICATION 5: MD5 hash should be available
+	if result.Md5Hash == nil {
+		t.Error("Expected non-nil Md5Hash")
+	} else {
+		t.Log("✓ Md5Hash is available")
+	}
+
+	// VERIFICATION 6: Chunk should have expected properties
+	if len(result.FileChunks) > 0 {
+		chunk := result.FileChunks[0]
+		if chunk.FileId != "test-fid,1234" {
+			t.Errorf("Expected chunk FileId='test-fid,1234', got '%s'", chunk.FileId)
+		}
+		if chunk.Offset != 0 {
+			t.Errorf("Expected chunk Offset=0, got %d", chunk.Offset)
+		}
+		if chunk.Size != uint64(len(testData)) {
+			t.Errorf("Expected chunk Size=%d, got %d", len(testData), chunk.Size)
+		}
+		t.Logf("✓ Chunk properties validated: FileId=%s, Offset=%d, Size=%d",
+			chunk.FileId, chunk.Offset, chunk.Size)
+	}
+}
+
+// TestUploadReaderInChunksContextCancellation verifies behavior when context is cancelled
+func TestUploadReaderInChunksContextCancellation(t *testing.T) {
+	testData := bytes.Repeat([]byte("test data"), 10000) // ~80KB
+	reader := bytes.NewReader(testData)
+
+	// Create a context that we'll cancel
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Cancel immediately to trigger cancellation handling
+	cancel()
+
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		return nil, &AssignResult{
+			Fid:       "test-fid,1234",
+			Url:       "http://test-volume:8080",
+			PublicUrl: "http://test-volume:8080",
+			Count:     1,
+		}, nil
+	}
+
+	// Mock upload function that simulates successful upload
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return &UploadResult{
+			Name:       "test-file",
+			Size:       uint32(len(data)),
+			ContentMd5: "mock-md5-hash",
+			Error:      "",
+		}, nil
+	}
+
+	result, err := UploadReaderInChunks(ctx, reader, &ChunkedUploadOption{
+		ChunkSize:       8 * 1024,
+		SmallFileLimit:  256,
+		Collection:      "test",
+		DataCenter:      "",
+		SaveSmallInline: false,
+		AssignFunc:      assignFunc,
+		UploadFunc:      uploadFunc,
+	})
+
+	// Should get context cancelled error
+	if err == nil {
+		t.Error("Expected context cancellation error")
+	}
+
+	// Should still get partial results for cleanup
+	if result == nil {
+		t.Error("Expected non-nil result even on context cancellation")
+	} else {
+		t.Logf("✓ Got partial result on cancellation: chunks=%d", len(result.FileChunks))
+	}
+}
+
+// TestUploadChunkToHoldersRollsBackOnPartialFailure verifies that when a fan-out
+// chunk write fails on one holder, the copies that already landed on the other
+// holders are deleted (type=replicate, local-only) so nothing is left orphaned.
+func TestUploadChunkToHoldersRollsBackOnPartialFailure(t *testing.T) {
+	const fid = "3,01abcdef"
+	var goodDeletes, badDeletes int32
+
+	// Sequence the failure strictly after the good upload so the test does not
+	// depend on timing: the failing holder returns its error only once the good
+	// holder has stored the chunk, so the good copy is always what gets rolled back.
+	goodUploaded := make(chan struct{})
+	var once sync.Once
+
+	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete {
+			if strings.Contains(r.URL.Path, "01abcdef") && r.URL.Query().Get("type") == "replicate" {
+				atomic.AddInt32(&goodDeletes, 1)
+			}
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		fmt.Fprintf(w, `{"name":"f","size":11}`)
+		once.Do(func() { close(goodUploaded) })
+	}))
+	defer good.Close()
+
+	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete {
+			atomic.AddInt32(&badDeletes, 1)
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		select {
+		case <-goodUploaded:
+		case <-time.After(5 * time.Second):
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		io.WriteString(w, "boom")
+	}))
+	defer bad.Close()
+
+	hosts := []string{strings.TrimPrefix(good.URL, "http://"), strings.TrimPrefix(bad.URL, "http://")}
+	_, err := uploadChunkToHolders(context.Background(), hosts, fid, []byte("hello world"), "", "", false, &ChunkedUploadOption{})
+
+	if err == nil {
+		t.Fatal("expected error from a partial fan-out")
+	}
+	if got := atomic.LoadInt32(&goodDeletes); got != 1 {
+		t.Errorf("expected the succeeded holder to receive 1 cleanup DELETE, got %d", got)
+	}
+	if got := atomic.LoadInt32(&badDeletes); got != 0 {
+		t.Errorf("expected no cleanup DELETE to the failed holder, got %d", got)
+	}
+}
+
+// mockFailingReader simulates a reader that fails after reading some data
+type mockFailingReader struct {
+	data      []byte
+	pos       int
+	failAfter int
+}
+
+func (m *mockFailingReader) Read(p []byte) (n int, err error) {
+	if m.pos >= m.failAfter {
+		return 0, errors.New("simulated read failure")
+	}
+
+	remaining := m.failAfter - m.pos
+	toRead := len(p)
+	if toRead > remaining {
+		toRead = remaining
+	}
+	if toRead > len(m.data)-m.pos {
+		toRead = len(m.data) - m.pos
+	}
+
+	if toRead == 0 {
+		return 0, io.EOF
+	}
+
+	copy(p, m.data[m.pos:m.pos+toRead])
+	m.pos += toRead
+	return toRead, nil
+}
+
+// TestUploadReaderInChunksReaderFailure verifies behavior when reader fails mid-read
+func TestUploadReaderInChunksReaderFailure(t *testing.T) {
+	testData := bytes.Repeat([]byte("test"), 5000) // 20KB
+	failingReader := &mockFailingReader{
+		data:      testData,
+		pos:       0,
+		failAfter: 10000, // Fail after 10KB
+	}
+
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		return nil, &AssignResult{
+			Fid:       "test-fid,1234",
+			Url:       "http://test-volume:8080",
+			PublicUrl: "http://test-volume:8080",
+			Count:     1,
+		}, nil
+	}
+
+	// Mock upload function that simulates successful upload
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return &UploadResult{
+			Name:       "test-file",
+			Size:       uint32(len(data)),
+			ContentMd5: "mock-md5-hash",
+			Error:      "",
+		}, nil
+	}
+
+	result, err := UploadReaderInChunks(context.Background(), failingReader, &ChunkedUploadOption{
+		ChunkSize:       8 * 1024, // 8KB chunks
+		SmallFileLimit:  256,
+		Collection:      "test",
+		DataCenter:      "",
+		SaveSmallInline: false,
+		AssignFunc:      assignFunc,
+		UploadFunc:      uploadFunc,
+	})
+
+	// Should get read error
+	if err == nil {
+		t.Error("Expected read failure error")
+	}
+
+	// Should still get partial results
+	if result == nil {
+		t.Fatal("Expected non-nil result on read failure")
+	}
+
+	t.Logf("✓ Got partial result on read failure: chunks=%d, totalSize=%d",
+		len(result.FileChunks), result.TotalSize)
+}
+
+// truncatedReader yields some bytes then reports io.ErrUnexpectedEOF, mimicking a
+// request body cut short by a client abort or reverse-proxy timeout.
+type truncatedReader struct {
+	data []byte
+	pos  int
+}
+
+func (r *truncatedReader) Read(p []byte) (int, error) {
+	if r.pos >= len(r.data) {
+		return 0, io.ErrUnexpectedEOF
+	}
+	n := copy(p, r.data[r.pos:])
+	r.pos += n
+	return n, nil
+}
+
+func TestUploadReaderInChunksTagsTruncatedBody(t *testing.T) {
+	reader := &truncatedReader{data: bytes.Repeat([]byte("x"), 10000)}
+
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		return nil, &AssignResult{Fid: "test-fid,1234", Url: "http://test-volume:8080", Count: 1}, nil
+	}
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return &UploadResult{Size: uint32(len(data))}, nil
+	}
+
+	_, err := UploadReaderInChunks(context.Background(), reader, &ChunkedUploadOption{
+		ChunkSize:  8 * 1024,
+		Collection: "test",
+		AssignFunc: assignFunc,
+		UploadFunc: uploadFunc,
+	})
+	if err == nil {
+		t.Fatal("expected an error for a truncated body")
+	}
+	if !errors.Is(err, ErrTruncatedBody) {
+		t.Errorf("expected ErrTruncatedBody, got %v", err)
+	}
+	if !errors.Is(err, io.ErrUnexpectedEOF) {
+		t.Errorf("expected io.ErrUnexpectedEOF to remain in the chain, got %v", err)
+	}
+}
+
+// uploadSingleChunk runs a single-chunk upload against uploadFunc, handing out a
+// fresh volume on every assignment, and reports how many assignments it took.
+func uploadSingleChunk(t *testing.T, uploadFunc func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error)) (*ChunkedUploadResult, int, error) {
+	t.Helper()
+
+	assigns := 0
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		assigns++
+		return nil, &AssignResult{
+			Fid:   fmt.Sprintf("%d,0a0b0c0d", assigns),
+			Url:   fmt.Sprintf("volume-%d:8080", assigns),
+			Auth:  security.EncodedJwt(fmt.Sprintf("jwt-%d", assigns)),
+			Count: 1,
+		}, nil
+	}
+
+	result, err := UploadReaderInChunks(context.Background(), bytes.NewReader(bytes.Repeat([]byte("x"), 4096)), &ChunkedUploadOption{
+		ChunkSize:  8 * 1024,
+		Collection: "test",
+		AssignFunc: assignFunc,
+		UploadFunc: uploadFunc,
+	})
+	return result, assigns, err
+}
+
+// A volume at capacity answers every attempt against the same fid with a 500,
+// so the chunk has to move to a freshly assigned volume rather than take the
+// whole object down with it.
+func TestUploadReaderInChunksReassignsOnFullVolume(t *testing.T) {
+	var jwts []security.EncodedJwt
+	result, assigns, err := uploadSingleChunk(t, func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		jwts = append(jwts, option.Jwt)
+		if strings.Contains(option.UploadUrl, "volume-1:") {
+			return nil, &uploadStatusError{
+				StatusCode: http.StatusInternalServerError,
+				err:        errors.New("failed to write to local disk: Volume Size 34361499680 Exceeded 34359738368"),
+			}
+		}
+		return &UploadResult{Size: uint32(len(data))}, nil
+	})
+
+	if err != nil {
+		t.Fatalf("expected the chunk to land on the reassigned volume, got %v", err)
+	}
+	if assigns != 2 {
+		t.Fatalf("expected 2 assignments, got %d", assigns)
+	}
+	// A secured cluster mints a JWT per assignment; presenting the full volume's
+	// token to the new one would 401.
+	if len(jwts) != 2 || jwts[0] != "jwt-1" || jwts[1] != "jwt-2" {
+		t.Errorf("expected each attempt to carry its own assignment JWT, got %v", jwts)
+	}
+	if len(result.FileChunks) != 1 {
+		t.Fatalf("expected 1 chunk, got %d", len(result.FileChunks))
+	}
+	if got := result.FileChunks[0].FileId; got != "2,0a0b0c0d" {
+		t.Errorf("expected the chunk to record the reassigned fid, got %s", got)
+	}
+}
+
+func TestUploadReaderInChunksDoesNotReassignOnClientError(t *testing.T) {
+	_, assigns, err := uploadSingleChunk(t, func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return nil, &uploadStatusError{StatusCode: http.StatusBadRequest, err: errors.New("mismatching cookie")}
+	})
+
+	if err == nil {
+		t.Fatal("expected a 4xx to fail the upload")
+	}
+	// Another volume would reject the same request the same way.
+	if assigns != 1 {
+		t.Fatalf("expected 1 assignment, got %d", assigns)
+	}
+}
+
+func TestUploadReaderInChunksBoundsReassignment(t *testing.T) {
+	_, assigns, err := uploadSingleChunk(t, func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		return nil, &uploadStatusError{StatusCode: http.StatusInternalServerError, err: errors.New("volume full")}
+	})
+
+	if err == nil {
+		t.Fatal("expected the upload to fail once every volume it is offered is full")
+	}
+	// Spelled out rather than compared to chunkAssignAttempts: the budget is
+	// what bounds this against retriedUploadData's own attempts, so raising it
+	// should fail here, not pass silently.
+	if assigns != 3 {
+		t.Fatalf("expected 3 assignments, got %d", assigns)
+	}
+}
+
+// The fan-out path is what a real multi-replica cluster takes, and it behaves
+// differently from the relay path under failure: it cancels its siblings and
+// rolls back the copies that landed. Drive the reassignment loop through it.
+func TestUploadReaderInChunksReassignsAcrossHolders(t *testing.T) {
+	var primaryDeletes, fullDeletes int32
+
+	healthy := func(deletes *int32) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method == http.MethodDelete {
+				atomic.AddInt32(deletes, 1)
+				w.WriteHeader(http.StatusAccepted)
+				return
+			}
+			w.WriteHeader(http.StatusCreated)
+			fmt.Fprint(w, `{"name":"chunk","size":4096}`)
+		}))
+	}
+
+	full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete {
+			atomic.AddInt32(&fullDeletes, 1)
+			w.WriteHeader(http.StatusAccepted)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, `{"error":"failed to write to local disk: Volume Size 34361499680 Exceeded 34359738368"}`)
+	}))
+	defer full.Close()
+
+	var secondaryDeletes int32
+	primary, secondary := healthy(&primaryDeletes), healthy(&secondaryDeletes)
+	defer primary.Close()
+	defer secondary.Close()
+
+	host := func(s *httptest.Server) string { return strings.TrimPrefix(s.URL, "http://") }
+
+	assigns := 0
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		assigns++
+		if assigns == 1 {
+			// Two holders, so uploadChunk takes the fan-out path; one is full.
+			return nil, &AssignResult{
+				Fid:      "1,0a0b0c0d",
+				Url:      host(primary),
+				Replicas: []Location{{Url: host(full)}},
+				Count:    1,
+			}, nil
+		}
+		return nil, &AssignResult{
+			Fid:      "2,0a0b0c0d",
+			Url:      host(primary),
+			Replicas: []Location{{Url: host(secondary)}},
+			Count:    1,
+		}, nil
+	}
+
+	result, err := UploadReaderInChunks(context.Background(), bytes.NewReader(bytes.Repeat([]byte("x"), 4096)), &ChunkedUploadOption{
+		ChunkSize:  8 * 1024,
+		Collection: "test",
+		AssignFunc: assignFunc,
+	})
+
+	if err != nil {
+		t.Fatalf("expected the chunk to land on the reassigned volume, got %v", err)
+	}
+	if assigns != 2 {
+		t.Fatalf("expected 2 assignments, got %d", assigns)
+	}
+	if len(result.FileChunks) != 1 || result.FileChunks[0].FileId != "2,0a0b0c0d" {
+		t.Fatalf("expected the chunk to record the reassigned fid, got %+v", result.FileChunks)
+	}
+	// The copy that landed on the healthy holder before its peer reported full
+	// is referenced by nothing, so it has to be rolled back.
+	if atomic.LoadInt32(&primaryDeletes) == 0 {
+		t.Error("expected the copy that landed to be rolled back")
+	}
+	// The full holder is not in uploadChunkToHolders' succeeded set, so only the
+	// loop's own rollback reaches it — and it has to, because a 5xx there can
+	// still mean the needle was committed before replication failed.
+	if atomic.LoadInt32(&fullDeletes) == 0 {
+		t.Error("expected the abandoned fid to be deleted from the failed holder too")
+	}
+	if atomic.LoadInt32(&secondaryDeletes) != 0 {
+		t.Error("the reassigned volume kept the chunk; it must not be rolled back")
+	}
+}
+
+// retriedUploadData retries the same URL three times by default. On the relay
+// path the reassignment loop retries everything that would, so leaving both in
+// place would multiply into nine upload calls for one chunk.
+//
+// This counts calls that reached a handler and answered. doUploadData reissues
+// once more on a connection reset before any of them return, which no HTTP
+// status can provoke; upload_content_test covers that separately.
+func TestUploadReaderInChunksDoesNotMultiplyRelayAttempts(t *testing.T) {
+	var posts, deletes int32
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete {
+			atomic.AddInt32(&deletes, 1)
+			w.WriteHeader(http.StatusAccepted)
+			return
+		}
+		atomic.AddInt32(&posts, 1)
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprint(w, `{"error":"failed to write to local disk: Volume Size 34361499680 Exceeded 34359738368"}`)
+	}))
+	defer dead.Close()
+
+	assigns := 0
+	_, err := UploadReaderInChunks(context.Background(), bytes.NewReader(bytes.Repeat([]byte("x"), 4096)), &ChunkedUploadOption{
+		ChunkSize:  8 * 1024,
+		Collection: "test",
+		AssignFunc: func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+			assigns++
+			// One holder, so uploadChunk takes the relay path.
+			return nil, &AssignResult{Fid: fmt.Sprintf("%d,0a0b0c0d", assigns), Url: strings.TrimPrefix(dead.URL, "http://"), Count: 1}, nil
+		},
+	})
+
+	if err == nil {
+		t.Fatal("expected the upload to fail once every volume it is offered is full")
+	}
+	if got := atomic.LoadInt32(&posts); got != 3 {
+		t.Errorf("expected 3 upload calls, one per assignment, got %d", got)
+	}
+	// Including the last one: giving up does not make the needle it may have
+	// committed anyone else's to find, and no chunk will ever name that fid.
+	if got := atomic.LoadInt32(&deletes); got != 3 {
+		t.Errorf("expected every abandoned fid to be rolled back, got %d deletes", got)
+	}
+}
+
+// TestUploadReaderInChunksAppendsFsyncWhenAssigned verifies that a chunk
+// assignment carrying the storage rule's fsync decision lands as ?fsync=true
+// on the volume server upload URL.
+func TestUploadReaderInChunksAppendsFsyncWhenAssigned(t *testing.T) {
+	testData := []byte("data needing fsync")
+	reader := bytes.NewReader(testData)
+
+	assignFunc := func(ctx context.Context, count int, expectedDataSize uint64) (*VolumeAssignRequest, *AssignResult, error) {
+		return nil, &AssignResult{
+			Fid:       "test-fid,1234",
+			Url:       "http://test-volume:8080",
+			PublicUrl: "http://test-volume:8080",
+			Count:     1,
+			Fsync:     true,
+		}, nil
+	}
+
+	var gotUploadUrl string
+	uploadFunc := func(ctx context.Context, data []byte, option *UploadOption) (*UploadResult, error) {
+		gotUploadUrl = option.UploadUrl
+		return &UploadResult{
+			Size:       uint32(len(data)),
+			ContentMd5: "mock-md5-hash",
+		}, nil
+	}
+
+	if _, err := UploadReaderInChunks(context.Background(), reader, &ChunkedUploadOption{
+		ChunkSize:       8 * 1024,
+		SmallFileLimit:  256,
+		SaveSmallInline: false,
+		AssignFunc:      assignFunc,
+		UploadFunc:      uploadFunc,
+	}); err != nil {
+		t.Fatalf("Expected successful upload, got error: %v", err)
+	}
+	if !strings.Contains(gotUploadUrl, "?fsync=true") {
+		t.Errorf("Expected the upload URL to carry ?fsync=true when the assignment says fsync, got %q", gotUploadUrl)
+	}
+}

weed/operation/upload_content.go

@@ -0,0 +1,592 @@
+package operation
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"mime/multipart"
+	"net/http"
+	"net/textproto"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+	"github.com/valyala/bytebufferpool"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+// GenUploadUrlProxy returns a function that builds a chunk upload URL via the
+// filer proxy.  Identical to the inline logic used by weed mount -filerProxy
+// and weed filer gateway:
+//
+//   - without filerProxy: "http://{host}/{fileId}"
+//   - with filerProxy:     "http://{filerAddress}/?proxyChunkId={fileId}"
+func GenUploadUrlProxy(filerAddress string) func(host, fileId string) string {
+	return func(host, fileId string) string {
+		if filerAddress == "" {
+			return fmt.Sprintf("http://%s/%s", host, fileId)
+		}
+		return util_http.ProxyChunkUrl(filerAddress, fileId)
+	}
+}
+
+type UploadOption struct {
+	UploadUrl         string
+	Filename          string
+	Cipher            bool
+	IsInputCompressed bool
+	IsReplication     bool // preserve the source needle's compression state
+	MimeType          string
+	PairMap           map[string]string
+	Jwt               security.EncodedJwt
+	RetryForever      bool
+	Md5               string
+	WantMd5           bool // compute Content-MD5 from the data when Md5 is unset and the upload is not ciphered
+	BytesBuffer       *bytes.Buffer
+	SourceUrl         string                           // optional: for logging when reading from a remote source
+	MaxAttempts       int                              // <=0 uses the default
+	GenUploadUrl      func(host, fileId string) string // if nil → fallback "http://{host}/{fileId}"
+}
+
+type UploadResult struct {
+	Name       string `json:"name,omitempty"`
+	Size       uint32 `json:"size,omitempty"`
+	Error      string `json:"error,omitempty"`
+	ETag       string `json:"eTag,omitempty"`
+	CipherKey  []byte `json:"cipherKey,omitempty"`
+	Mime       string `json:"mime,omitempty"`
+	Gzip       uint32 `json:"gzip,omitempty"`
+	ContentMd5 string `json:"contentMd5,omitempty"`
+	RetryCount int    `json:"-"`
+}
+
+func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64, tsNs int64) *filer_pb.FileChunk {
+	fid, _ := filer_pb.ToFileIdObject(fileId)
+	return &filer_pb.FileChunk{
+		FileId:       fileId,
+		Offset:       offset,
+		Size:         uint64(uploadResult.Size),
+		ModifiedTsNs: tsNs,
+		ETag:         uploadResult.ContentMd5,
+		CipherKey:    uploadResult.CipherKey,
+		IsCompressed: uploadResult.Gzip > 0,
+		Fid:          fid,
+	}
+}
+
+// ToPbFileChunkWithSSE creates a FileChunk with SSE metadata
+func (uploadResult *UploadResult) ToPbFileChunkWithSSE(fileId string, offset int64, tsNs int64, sseType filer_pb.SSEType, sseMetadata []byte) *filer_pb.FileChunk {
+	fid, _ := filer_pb.ToFileIdObject(fileId)
+	chunk := &filer_pb.FileChunk{
+		FileId:       fileId,
+		Offset:       offset,
+		Size:         uint64(uploadResult.Size),
+		ModifiedTsNs: tsNs,
+		ETag:         uploadResult.ContentMd5,
+		CipherKey:    uploadResult.CipherKey,
+		IsCompressed: uploadResult.Gzip > 0,
+		Fid:          fid,
+	}
+
+	// Add SSE metadata if provided
+	chunk.SseType = sseType
+	if len(sseMetadata) > 0 {
+		chunk.SseMetadata = sseMetadata
+	}
+
+	return chunk
+}
+
+var (
+	uploader    *Uploader
+	uploaderErr error
+	once        sync.Once
+)
+
+// uploadStatusError carries the volume-server HTTP status of a failed upload so
+// the retry gate can decide on the status code instead of the message text.
+// StatusCode 0 means the request never got a response — a transport failure.
+type uploadStatusError struct {
+	StatusCode int
+	err        error
+}
+
+func (e *uploadStatusError) Error() string { return e.err.Error() }
+func (e *uploadStatusError) Unwrap() error { return e.err }
+
+// ShouldReassignUpload reports whether an upload error means the client should
+// ask for a fresh volume assignment and retry on another volume.
+//
+// On the write path a volume server only 5xxs on a ReplicatedWrite failure
+// (local disk, replica peer down, or under-replication) — all of which a
+// different volume dodges — so any 5xx is reassignable. A status of 0 means the
+// assigned target never answered (down/unreachable), also reassignable. A 4xx
+// is a genuine client error and is surfaced. Errors without a status come from
+// the AssignVolume RPC or request setup; retry only transient transport ones.
+func ShouldReassignUpload(err error) bool {
+	if err == nil {
+		return false
+	}
+	var se *uploadStatusError
+	if errors.As(err, &se) {
+		return se.StatusCode == 0 || se.StatusCode >= 500
+	}
+	return strings.Contains(err.Error(), "transport")
+}
+
+// assignVolumeTimeout bounds a single AssignVolume RPC so an overwhelmed filer
+// can't block the caller forever. Overridable in tests.
+var assignVolumeTimeout = 30 * time.Second
+
+// HTTPClient interface for testing
+type HTTPClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+// Uploader
+type Uploader struct {
+	httpClient HTTPClient
+}
+
+func NewUploader() (*Uploader, error) {
+	once.Do(func() {
+		// With Dial context
+		var httpClient *util_http_client.HTTPClient
+		httpClient, uploaderErr = util_http.NewGlobalHttpClient(util_http_client.AddDialContext)
+		if uploaderErr != nil {
+			uploaderErr = fmt.Errorf("error initializing the loader: %s", uploaderErr)
+		}
+		if httpClient != nil {
+			uploader = newUploader(httpClient)
+		}
+	})
+	return uploader, uploaderErr
+}
+
+func newUploader(httpClient HTTPClient) *Uploader {
+	return &Uploader{
+		httpClient: httpClient,
+	}
+}
+
+// NewUploaderWithHttpClient creates an Uploader that uses the provided HTTP
+// client instead of the global one. This is used by filer.sync to upload to
+// remote clusters that use different TLS certificates.
+func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader {
+	return &Uploader{
+		httpClient: httpClient,
+	}
+}
+
+func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, data []byte) (fileId string, uploadResult *UploadResult, err error) {
+	doUploadFunc := func() error {
+		var host string
+		var auth security.EncodedJwt
+		fileId, host, auth, err = assignFn()
+		if err != nil {
+			return err
+		}
+
+		genUrl := uploadOption.GenUploadUrl
+		if genUrl == nil {
+			genUrl = func(host, fileId string) string { return fmt.Sprintf("http://%s/%s", host, fileId) }
+		}
+		uploadOption.UploadUrl = genUrl(host, fileId)
+		uploadOption.Jwt = auth
+		if util_http.IsProxyChunkUrl(uploadOption.UploadUrl) {
+			// The request addresses the filer, which authorizes it and mints the
+			// volume credential itself. The AssignVolume token is not a filer
+			// credential and gets the caller nowhere here.
+			uploadOption.Jwt = security.EncodedJwt(util_http.JwtForFilerServer(true))
+		}
+
+		uploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)
+		return err
+	}
+
+	if uploadOption.RetryForever {
+		util.RetryUntil("uploadWithRetryForever", doUploadFunc, func(err error) (shouldContinue bool) {
+			glog.V(0).Infof("upload content: %v", err)
+			return true
+		})
+	} else {
+		err = util.RetryOnError("uploadWithRetry", ShouldReassignUpload, doUploadFunc)
+	}
+
+	return
+}
+
+// UploadWithRetry will retry both assigning volume request and uploading content
+// The option parameter does not need to specify UploadUrl and Jwt, which will come from assigning volume.
+func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) {
+	bytesReader, ok := reader.(*util.BytesReader)
+	if ok {
+		data = bytesReader.Bytes
+	} else {
+		data, err = io.ReadAll(reader)
+		if err != nil {
+			glog.V(0).Infof("upload read input %s: %v", uploadOption.SourceUrl, err)
+			err = fmt.Errorf("read input: %w", err)
+			return
+		}
+		glog.V(4).Infof("upload read %d bytes from %s", len(data), uploadOption.SourceUrl)
+	}
+
+	// Tell the master the real chunk size so its effectiveSize accounting
+	// doesn't fall back to the 1 MB DefaultNeedleSizeEstimate per fid.
+	if assignRequest.ExpectedDataSize == 0 {
+		assignRequest.ExpectedDataSize = uint64(len(data))
+	}
+
+	// Hash the buffer we already hold so the server echoes Content-MD5 back as
+	// the chunk ETag (std-base64 of the raw digest, the form ParseUpload
+	// verifies). Under cipher the server never sees the header, but the digest
+	// still becomes the chunk ETag.
+	if uploadOption.WantMd5 && uploadOption.Md5 == "" {
+		digest := md5.Sum(data)
+		uploadOption.Md5 = base64.StdEncoding.EncodeToString(digest[:])
+	}
+
+	fileId, uploadResult, err = uploader.uploadWithRetryData(func() (fileId string, host string, auth security.EncodedJwt, err error) {
+		// grpc assign volume
+		if grpcAssignErr := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			assignCtx, assignCancel := context.WithTimeout(context.Background(), assignVolumeTimeout)
+			defer assignCancel()
+			resp, assignErr := client.AssignVolume(assignCtx, assignRequest)
+			if assignErr != nil {
+				glog.V(0).Infof("assign volume failure %v: %v", assignRequest, assignErr)
+				return assignErr
+			}
+			if resp.Error != "" {
+				return fmt.Errorf("assign volume failure %v: %v", assignRequest, resp.Error)
+			}
+
+			fileId, auth = resp.FileId, security.EncodedJwt(resp.Auth)
+			loc := resp.Location
+			host = filerClient.AdjustedUrl(loc)
+
+			return nil
+		}); grpcAssignErr != nil {
+			err = fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)
+		}
+		return
+	}, uploadOption, data)
+	return
+}
+
+// Upload sends a POST request to a volume server to upload the content with adjustable compression level
+func (uploader *Uploader) UploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
+	uploadResult, err = uploader.retriedUploadData(ctx, data, option)
+	return
+}
+
+// Upload sends a POST request to a volume server to upload the content with fast compression
+func (uploader *Uploader) Upload(ctx context.Context, reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
+	uploadResult, err, data = uploader.doUpload(ctx, reader, option)
+	return
+}
+
+func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option *UploadOption) (uploadResult *UploadResult, err error, data []byte) {
+	bytesReader, ok := reader.(*util.BytesReader)
+	if ok {
+		data = bytesReader.Bytes
+	} else {
+		data, err = io.ReadAll(reader)
+		if err != nil {
+			err = fmt.Errorf("read input: %w", err)
+			return
+		}
+	}
+	uploadResult, uploadErr := uploader.retriedUploadData(ctx, data, option)
+	return uploadResult, uploadErr, data
+}
+
+func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
+	maxAttempts := option.MaxAttempts
+	if maxAttempts <= 0 {
+		maxAttempts = 3
+	}
+	for i := 0; i < maxAttempts; i++ {
+		if i > 0 {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(time.Millisecond * time.Duration(237*(i+1))):
+			}
+		}
+		uploadResult, err = uploader.doUploadData(ctx, data, option)
+		if err == nil {
+			uploadResult.RetryCount = i
+			return
+		}
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		glog.WarningfCtx(ctx, "uploading %d to %s: %v", i, option.UploadUrl, err)
+	}
+	return
+}
+
+func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) {
+	contentIsGzipped := option.IsInputCompressed
+	shouldGzipNow := false
+	if !option.IsInputCompressed && !option.IsReplication {
+		if option.MimeType == "" {
+			option.MimeType = http.DetectContentType(data)
+			// println("detect1 mimetype to", MimeType)
+			if option.MimeType == "application/octet-stream" {
+				option.MimeType = ""
+			}
+		}
+		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(filepath.Base(option.Filename), option.MimeType); iAmSure && shouldBeCompressed {
+			shouldGzipNow = true
+		} else if !iAmSure && option.MimeType == "" && len(data) > 16*1024 {
+			var compressed []byte
+			compressed, err = util.GzipData(data[0:128])
+			if err != nil {
+				return
+			}
+			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
+		}
+	}
+
+	var clearDataLen int
+
+	// gzip if possible
+	// this could be double copying
+	clearDataLen = len(data)
+	clearData := data
+	if shouldGzipNow {
+		compressed, compressErr := util.GzipData(data)
+		// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
+		if compressErr == nil {
+			if len(compressed) < len(data) {
+				data = compressed
+				contentIsGzipped = true
+			}
+		}
+	} else if option.IsInputCompressed && !option.IsReplication {
+		// decompress only to report the clear data length; replication discards the result, so skip it
+		clearData, err = util.DecompressData(data)
+		if err == nil {
+			clearDataLen = len(clearData)
+		}
+	}
+
+	if option.Cipher {
+		// encrypt(gzip(data))
+
+		// encrypt
+		cipherKey := util.GenCipherKey()
+		encryptedData, encryptionErr := util.Encrypt(data, cipherKey)
+		if encryptionErr != nil {
+			err = fmt.Errorf("encrypt input: %w", encryptionErr)
+			return
+		}
+
+		// upload data
+		uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
+			_, err = w.Write(encryptedData)
+			return
+		}, len(encryptedData), &UploadOption{
+			UploadUrl:         option.UploadUrl,
+			Filename:          "",
+			Cipher:            false,
+			IsInputCompressed: false,
+			MimeType:          "",
+			PairMap:           nil,
+			Jwt:               option.Jwt,
+		})
+		if uploadResult == nil {
+			return
+		}
+		uploadResult.Name = option.Filename
+		uploadResult.Mime = option.MimeType
+		uploadResult.CipherKey = cipherKey
+		// The volume server only ever hashes the ciphertext it stored, so the
+		// chunk ETag has to come from the caller's plaintext digest.
+		uploadResult.ContentMd5 = option.Md5
+		uploadResult.Size = uint32(clearDataLen)
+		if contentIsGzipped {
+			uploadResult.Gzip = 1
+		}
+	} else {
+		// upload data
+		uploadResult, err = uploader.upload_content(ctx, func(w io.Writer) (err error) {
+			_, err = w.Write(data)
+			return
+		}, len(data), &UploadOption{
+			UploadUrl:         option.UploadUrl,
+			Filename:          option.Filename,
+			Cipher:            false,
+			IsInputCompressed: contentIsGzipped,
+			MimeType:          option.MimeType,
+			PairMap:           option.PairMap,
+			Jwt:               option.Jwt,
+			Md5:               option.Md5,
+			BytesBuffer:       option.BytesBuffer,
+		})
+		if uploadResult == nil {
+			return
+		}
+		uploadResult.Size = uint32(clearDataLen)
+		if contentIsGzipped {
+			uploadResult.Gzip = 1
+		}
+	}
+
+	return uploadResult, err
+}
+
+func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction func(w io.Writer) error, originalDataSize int, option *UploadOption) (*UploadResult, error) {
+	var body_writer *multipart.Writer
+	var reqReader *bytes.Reader
+	var buf *bytebufferpool.ByteBuffer
+	if option.BytesBuffer == nil {
+		buf = GetBuffer()
+		defer PutBuffer(buf)
+		body_writer = multipart.NewWriter(buf)
+	} else {
+		option.BytesBuffer.Reset()
+		body_writer = multipart.NewWriter(option.BytesBuffer)
+	}
+	h := make(textproto.MIMEHeader)
+	// Use mime.FormatMediaType for RFC 6266 compliant Content-Disposition,
+	// properly handling non-ASCII characters and special characters
+	h.Set("Content-Disposition", mime.FormatMediaType("form-data", map[string]string{"name": "file", "filename": option.Filename}))
+	h.Set("Idempotency-Key", option.UploadUrl)
+	if option.MimeType == "" {
+		option.MimeType = mime.TypeByExtension(strings.ToLower(filepath.Ext(option.Filename)))
+	}
+	if option.MimeType != "" {
+		h.Set("Content-Type", option.MimeType)
+	}
+	if option.IsInputCompressed {
+		h.Set("Content-Encoding", "gzip")
+	}
+	if option.Md5 != "" {
+		h.Set("Content-MD5", option.Md5)
+	}
+
+	file_writer, cp_err := body_writer.CreatePart(h)
+	if cp_err != nil {
+		glog.V(0).InfolnCtx(ctx, "error creating form file", cp_err.Error())
+		return nil, cp_err
+	}
+	if err := fillBufferFunction(file_writer); err != nil {
+		glog.V(0).InfolnCtx(ctx, "error copying data", err)
+		return nil, err
+	}
+	content_type := body_writer.FormDataContentType()
+	if err := body_writer.Close(); err != nil {
+		glog.V(0).InfolnCtx(ctx, "error closing body", err)
+		return nil, err
+	}
+	if option.BytesBuffer == nil {
+		reqReader = bytes.NewReader(buf.Bytes())
+	} else {
+		reqReader = bytes.NewReader(option.BytesBuffer.Bytes())
+	}
+	req, postErr := http.NewRequestWithContext(ctx, http.MethodPost, option.UploadUrl, reqReader)
+	if postErr != nil {
+		glog.V(1).InfofCtx(ctx, "create upload request %s: %v", option.UploadUrl, postErr)
+		return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr)
+	}
+	req.Header.Set("Content-Type", content_type)
+	for k, v := range option.PairMap {
+		req.Header.Set(k, v)
+	}
+	if option.Jwt != "" {
+		req.Header.Set("Authorization", security.BearerPrefix+string(option.Jwt))
+	}
+
+	request_id.InjectToRequest(ctx, req)
+
+	// print("+")
+	resp, post_err := uploader.httpClient.Do(req)
+	defer util_http.CloseResponse(resp)
+	if post_err != nil {
+		if strings.Contains(post_err.Error(), "connection reset by peer") ||
+			strings.Contains(post_err.Error(), "use of closed network connection") {
+			glog.V(1).InfofCtx(ctx, "repeat error upload request %s: %v", option.UploadUrl, post_err)
+			stats.FilerHandlerCounter.WithLabelValues(stats.RepeatErrorUploadContent).Inc()
+			// The first attempt already consumed (or partially consumed) the
+			// body, so retrying with the same *http.Request would send 0 bytes
+			// and Go's transport would surface "ContentLength=N with Body
+			// length 0". http.NewRequestWithContext sets GetBody for
+			// *bytes.Reader bodies; use it to attach a fresh body for retry.
+			// If we can't rewind, skip the inner retry and let the outer
+			// retriedUploadData loop reissue the request with a fresh body —
+			// retrying here with a consumed body would mask the original
+			// "connection reset" error with a misleading "Body length 0".
+			if req.GetBody != nil {
+				if newBody, gbErr := req.GetBody(); gbErr == nil {
+					req.Body = newBody
+					resp, post_err = uploader.httpClient.Do(req)
+					defer util_http.CloseResponse(resp)
+				} else {
+					glog.V(1).InfofCtx(ctx, "skip inner retry for %s: GetBody returned %v", option.UploadUrl, gbErr)
+				}
+			} else {
+				glog.V(1).InfofCtx(ctx, "skip inner retry for %s: req.GetBody is nil", option.UploadUrl)
+			}
+		}
+	}
+	if post_err != nil {
+		stats.UploadErrorCounter.WithLabelValues("0").Inc()
+		return nil, &uploadStatusError{StatusCode: 0, err: fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)}
+	}
+	// print("-")
+
+	var ret UploadResult
+	etag := getEtag(resp)
+	if resp.StatusCode == http.StatusNoContent {
+		ret.ETag = etag
+		ret.ContentMd5 = resp.Header.Get("Content-MD5")
+		return &ret, nil
+	}
+
+	resp_body, ra_err := io.ReadAll(resp.Body)
+	if ra_err != nil {
+		stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
+		return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)}
+	}
+
+	unmarshal_err := json.Unmarshal(resp_body, &ret)
+	if unmarshal_err != nil {
+		stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
+		glog.ErrorfCtx(ctx, "unmarshal %s: %v", option.UploadUrl, string(resp_body))
+		return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("unmarshal %v: %w", option.UploadUrl, unmarshal_err)}
+	}
+	if ret.Error != "" {
+		stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
+		return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)}
+	}
+	ret.ETag = etag
+	ret.ContentMd5 = resp.Header.Get("Content-MD5")
+	return &ret, nil
+}
+
+func getEtag(r *http.Response) (etag string) {
+	etag = r.Header.Get("ETag")
+	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
+		etag = etag[1 : len(etag)-1]
+	}
+	return
+}

weed/operation/upload_content_test.go

@@ -0,0 +1,517 @@
+package operation
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+type scriptedHTTPResponse struct {
+	status int
+	body   string
+}
+
+type scriptedHTTPClient struct {
+	mu        sync.Mutex
+	responses map[string][]scriptedHTTPResponse
+	calls     []string
+}
+
+func (c *scriptedHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	url := req.URL.String()
+	c.calls = append(c.calls, url)
+
+	plans := c.responses[url]
+	if len(plans) == 0 {
+		return nil, fmt.Errorf("unexpected request to %s", url)
+	}
+	plan := plans[0]
+	c.responses[url] = plans[1:]
+
+	return &http.Response{
+		StatusCode: plan.status,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(plan.body)),
+	}, nil
+}
+
+func TestShouldReassignUpload(t *testing.T) {
+	testCases := []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "nil", err: nil, want: false},
+		{name: "no response (transport)", err: &uploadStatusError{StatusCode: 0, err: fmt.Errorf("dial tcp: no such host")}, want: true},
+		{name: "500 replica write failed", err: &uploadStatusError{StatusCode: 500, err: fmt.Errorf("failed to write to replicas")}, want: true},
+		{name: "503 service unavailable", err: &uploadStatusError{StatusCode: 503, err: fmt.Errorf("busy")}, want: true},
+		{name: "400 bad request", err: &uploadStatusError{StatusCode: 400, err: fmt.Errorf("bad needle")}, want: false},
+		{name: "401 unauthorized", err: &uploadStatusError{StatusCode: 401, err: fmt.Errorf("wrong jwt")}, want: false},
+		{name: "assign rpc transport", err: fmt.Errorf("assign volume: transport is closing"), want: true},
+		{name: "other permanent", err: fmt.Errorf("mismatching cookie"), want: false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := ShouldReassignUpload(tc.err); got != tc.want {
+				t.Fatalf("ShouldReassignUpload(%v) = %v, want %v", tc.err, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestUploadWithRetryDataReassignsOnVolumeSizeExceeded(t *testing.T) {
+	httpClient := &scriptedHTTPClient{
+		responses: map[string][]scriptedHTTPResponse{
+			"http://volume-a/1,first": {
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+				{status: http.StatusInternalServerError, body: `{"error":"failed to write to local disk: append to volume 1 size 0 actualSize 0: Volume Size 33555976 Exceeded 33554432"}`},
+			},
+			"http://volume-b/2,second": {
+				{status: http.StatusCreated, body: `{"name":"test.bin","size":3}`},
+			},
+		},
+	}
+	uploader := newUploader(httpClient)
+
+	assignCalls := 0
+	fileID, uploadResult, err := uploader.uploadWithRetryData(func() (string, string, security.EncodedJwt, error) {
+		assignCalls++
+		if assignCalls == 1 {
+			return "1,first", "volume-a", "", nil
+		}
+		return "2,second", "volume-b", "", nil
+	}, &UploadOption{Filename: "test.bin"}, []byte("abc"))
+
+	if err != nil {
+		t.Fatalf("expected success after reassignment, got %v", err)
+	}
+	if fileID != "2,second" {
+		t.Fatalf("expected second file id, got %s", fileID)
+	}
+	if assignCalls != 2 {
+		t.Fatalf("expected 2 assign attempts, got %d", assignCalls)
+	}
+	if uploadResult == nil || uploadResult.Name != "test.bin" {
+		t.Fatalf("expected successful upload result, got %#v", uploadResult)
+	}
+	if len(httpClient.calls) != 4 {
+		t.Fatalf("expected 4 upload attempts (3 same-url retries + 1 reassigned upload), got %d", len(httpClient.calls))
+	}
+	if httpClient.calls[0] != "http://volume-a/1,first" || httpClient.calls[3] != "http://volume-b/2,second" {
+		t.Fatalf("unexpected upload call sequence: %#v", httpClient.calls)
+	}
+}
+
+// A volume whose replica peer is down reports "failed to write to replicas":
+// the primary write lands but replication fails. The client must re-assign to a
+// volume on live nodes and retry rather than losing the write.
+func TestUploadWithRetryDataReassignsOnReplicaWriteFailure(t *testing.T) {
+	replicaErr := `{"error":"failed to write to replicas for volume 1: [volume-down:8080]: dial tcp: lookup volume-down: no such host"}`
+	httpClient := &scriptedHTTPClient{
+		responses: map[string][]scriptedHTTPResponse{
+			"http://volume-a/1,first": {
+				{status: http.StatusInternalServerError, body: replicaErr},
+				{status: http.StatusInternalServerError, body: replicaErr},
+				{status: http.StatusInternalServerError, body: replicaErr},
+			},
+			"http://volume-b/2,second": {
+				{status: http.StatusCreated, body: `{"name":"test.bin","size":3}`},
+			},
+		},
+	}
+	uploader := newUploader(httpClient)
+
+	assignCalls := 0
+	fileID, uploadResult, err := uploader.uploadWithRetryData(func() (string, string, security.EncodedJwt, error) {
+		assignCalls++
+		if assignCalls == 1 {
+			return "1,first", "volume-a", "", nil
+		}
+		return "2,second", "volume-b", "", nil
+	}, &UploadOption{Filename: "test.bin"}, []byte("abc"))
+
+	if err != nil {
+		t.Fatalf("expected success after reassignment, got %v", err)
+	}
+	if fileID != "2,second" {
+		t.Fatalf("expected reassigned file id, got %s", fileID)
+	}
+	if assignCalls != 2 {
+		t.Fatalf("expected 2 assign attempts, got %d", assignCalls)
+	}
+	if uploadResult == nil || uploadResult.Name != "test.bin" {
+		t.Fatalf("expected successful upload result, got %#v", uploadResult)
+	}
+}
+
+// bodyCapturingHTTPClient drains req.Body on every Do, optionally failing the
+// first attempt with a transport error so we can verify upload_content rewinds
+// the body before retrying.
+type bodyCapturingHTTPClient struct {
+	mu          sync.Mutex
+	bodies      [][]byte
+	failFirst   string
+	successJSON string
+}
+
+func (c *bodyCapturingHTTPClient) Do(req *http.Request) (*http.Response, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var captured []byte
+	if req.Body != nil {
+		b, err := io.ReadAll(req.Body)
+		if err != nil {
+			return nil, err
+		}
+		captured = b
+	}
+	c.bodies = append(c.bodies, captured)
+
+	if len(c.bodies) == 1 && c.failFirst != "" {
+		return nil, errors.New(c.failFirst)
+	}
+	return &http.Response{
+		StatusCode: http.StatusCreated,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(strings.NewReader(c.successJSON)),
+	}, nil
+}
+
+// hangingAssignSeaweedClient is a SeaweedFilerClient whose AssignVolume blocks
+// until its context is done, so a test can prove UploadWithRetry bounds the RPC.
+type hangingAssignSeaweedClient struct {
+	filer_pb.SeaweedFilerClient
+	sawDeadline chan bool
+}
+
+func (c *hangingAssignSeaweedClient) AssignVolume(ctx context.Context, in *filer_pb.AssignVolumeRequest, opts ...grpc.CallOption) (*filer_pb.AssignVolumeResponse, error) {
+	_, ok := ctx.Deadline()
+	// Non-blocking so a retry that calls this more than once can't wedge here.
+	select {
+	case c.sawDeadline <- ok:
+	default:
+	}
+	<-ctx.Done() // simulate an overwhelmed filer that never answers
+	return nil, ctx.Err()
+}
+
+type hangingAssignFilerClient struct {
+	inner *hangingAssignSeaweedClient
+}
+
+func (c *hangingAssignFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(c.inner)
+}
+func (c *hangingAssignFilerClient) AdjustedUrl(loc *filer_pb.Location) string { return loc.GetUrl() }
+func (c *hangingAssignFilerClient) GetDataCenter() string                     { return "" }
+
+// TestUploadWithRetryBoundsAssignVolume covers the case where an AssignVolume
+// against an overwhelmed filer carried no deadline, so the upload (and the FUSE
+// flush driving it) blocked forever. UploadWithRetry must give the RPC a
+// deadline and return once it expires instead of hanging.
+func TestUploadWithRetryBoundsAssignVolume(t *testing.T) {
+	original := assignVolumeTimeout
+	assignVolumeTimeout = 200 * time.Millisecond
+	t.Cleanup(func() { assignVolumeTimeout = original })
+
+	client := &hangingAssignFilerClient{inner: &hangingAssignSeaweedClient{sawDeadline: make(chan bool, 1)}}
+	uploader := newUploader(&scriptedHTTPClient{responses: map[string][]scriptedHTTPResponse{}})
+
+	done := make(chan error, 1)
+	go func() {
+		_, _, err, _ := uploader.UploadWithRetry(client,
+			&filer_pb.AssignVolumeRequest{Count: 1},
+			&UploadOption{Filename: "test.bin"},
+			bytes.NewReader([]byte("abc")),
+		)
+		done <- err
+	}()
+
+	select {
+	case sawDeadline := <-client.inner.sawDeadline:
+		if !sawDeadline {
+			t.Fatal("AssignVolume received a context with no deadline")
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("AssignVolume was never called")
+	}
+
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("expected UploadWithRetry to fail once the assign deadline expired")
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("UploadWithRetry hung well past the assign timeout")
+	}
+}
+
+// TestUploadRewindsBodyOnConnectionReset reproduces issue #9139 follow-up:
+// when the inner Do retry fires on a "connection reset" / "closed network"
+// error, the *bytes.Reader body has already been consumed, so without an
+// explicit rewind the second attempt sends 0 bytes and Go's transport surfaces
+// "ContentLength=N with Body length 0".
+func TestUploadRewindsBodyOnConnectionReset(t *testing.T) {
+	for _, transient := range []string{
+		"connection reset by peer",
+		"use of closed network connection",
+	} {
+		t.Run(transient, func(t *testing.T) {
+			client := &bodyCapturingHTTPClient{
+				failFirst:   transient,
+				successJSON: `{"name":"test.bin","size":42}`,
+			}
+			uploader := newUploader(client)
+
+			payload := bytes.Repeat([]byte("payload-"), 256) // 2048 bytes
+			_, err := uploader.UploadData(context.Background(), payload, &UploadOption{
+				UploadUrl: "http://volume/1,abc",
+				Filename:  "test.bin",
+			})
+			if err != nil {
+				t.Fatalf("upload should succeed after inner retry, got %v", err)
+			}
+			if len(client.bodies) != 2 {
+				t.Fatalf("expected 2 Do attempts, got %d", len(client.bodies))
+			}
+			if len(client.bodies[0]) == 0 {
+				t.Fatalf("first attempt sent an empty body; test setup wrong")
+			}
+			if !bytes.Equal(client.bodies[0], client.bodies[1]) {
+				t.Fatalf("retry body length=%d differs from first attempt length=%d (body was not rewound)",
+					len(client.bodies[1]), len(client.bodies[0]))
+			}
+		})
+	}
+}
+
+func TestReplicationUploadPreservesUncompressedNeedle(t *testing.T) {
+	testCases := []struct {
+		name    string
+		payload []byte
+	}{
+		{
+			name:    "unknown binary sampling",
+			payload: bytes.Repeat([]byte{0}, 32*1024),
+		},
+		{
+			name:    "receiver MIME detection",
+			payload: bytes.Repeat([]byte("plain text content\n"), 2048),
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var replicatedNeedle *needle.Needle
+			var parseErr error
+
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				replicatedNeedle, _, _, parseErr = needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusCreated)
+				_, _ = io.WriteString(w, `{"name":"payload.custom","size":32768}`)
+			}))
+			defer server.Close()
+
+			uploader := newUploader(server.Client())
+			_, err := uploader.UploadData(context.Background(), tc.payload, &UploadOption{
+				UploadUrl:     server.URL + "/3,01637037d6?type=replicate",
+				Filename:      "payload.custom",
+				IsReplication: true,
+				MaxAttempts:   1,
+			})
+			if err != nil {
+				t.Fatalf("replication upload failed: %v", err)
+			}
+			if parseErr != nil {
+				t.Fatalf("parse replicated upload: %v", parseErr)
+			}
+			if replicatedNeedle == nil {
+				t.Fatal("replica did not receive a needle")
+			}
+			if replicatedNeedle.IsCompressed() {
+				t.Fatal("replication changed an uncompressed needle to compressed")
+			}
+			if !bytes.Equal(replicatedNeedle.Data, tc.payload) {
+				t.Fatalf("replicated data differs: got %d bytes, want %d", len(replicatedNeedle.Data), len(tc.payload))
+			}
+		})
+	}
+}
+
+// deadReplicaClient simulates an unreachable replica: every request blocks for
+// dialDelay (standing in for the TCP dial timeout) and then fails, unless the
+// request context is cancelled first.
+type deadReplicaClient struct {
+	mu        sync.Mutex
+	calls     int
+	dialDelay time.Duration
+}
+
+func (c *deadReplicaClient) attempts() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+func (c *deadReplicaClient) Do(req *http.Request) (*http.Response, error) {
+	c.mu.Lock()
+	c.calls++
+	c.mu.Unlock()
+	select {
+	case <-time.After(c.dialDelay):
+		return nil, fmt.Errorf("dial tcp %s: i/o timeout", req.URL.Host)
+	case <-req.Context().Done():
+		return nil, req.Context().Err()
+	}
+}
+
+// TestUploadToDeadReplicaRetriesThreeTimes reproduces the dead-replica upload
+// stall: a synchronous replica write to an unreachable host pays the dial
+// timeout three times over, so one dead replica stalls the caller for ~3x the
+// dial timeout. In production dialDelay is the 10s dialer timeout, so the
+// caller blocks ~30s before the failure surfaces.
+func TestUploadToDeadReplicaRetriesThreeTimes(t *testing.T) {
+	client := &deadReplicaClient{dialDelay: 100 * time.Millisecond}
+	uploader := newUploader(client)
+
+	start := time.Now()
+	_, err := uploader.UploadData(context.Background(), []byte("hello"), &UploadOption{
+		UploadUrl: "http://dead-replica:8080/3,01",
+		Filename:  "test.bin",
+	})
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("expected an error uploading to a dead replica")
+	}
+	if got := client.attempts(); got != 3 {
+		t.Fatalf("dial attempts = %d, want 3 (each attempt pays the full dial timeout)", got)
+	}
+	t.Logf("dead replica stalled the caller for %v across %d attempts", elapsed, client.attempts())
+}
+
+// TestUploadToDeadReplicaSingleAttempt verifies the fix: a synchronous replica
+// write makes a single attempt (MaxAttempts=1), so a dead replica fails after
+// one dial timeout instead of three. The outer client write still retries.
+func TestUploadToDeadReplicaSingleAttempt(t *testing.T) {
+	client := &deadReplicaClient{dialDelay: 100 * time.Millisecond}
+	uploader := newUploader(client)
+
+	_, err := uploader.UploadData(context.Background(), []byte("hello"), &UploadOption{
+		UploadUrl:   "http://dead-replica:8080/3,01",
+		Filename:    "test.bin",
+		MaxAttempts: 1,
+	})
+	if err == nil {
+		t.Fatal("expected an error uploading to a dead replica")
+	}
+	if got := client.attempts(); got != 1 {
+		t.Fatalf("dial attempts = %d, want 1", got)
+	}
+}
+
+// TestUploadRetryStopsOnContextCancel verifies the retry loop honors context
+// cancellation so DistributedOperation can abandon a slow replica once the
+// write outcome is already decided by another replica.
+func TestUploadRetryStopsOnContextCancel(t *testing.T) {
+	client := &deadReplicaClient{dialDelay: time.Hour} // would block ~forever per attempt
+	uploader := newUploader(client)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel() // already decided elsewhere
+
+	start := time.Now()
+	_, err := uploader.UploadData(ctx, []byte("hello"), &UploadOption{
+		UploadUrl: "http://dead-replica:8080/3,01",
+		Filename:  "test.bin",
+	})
+	if err == nil {
+		t.Fatal("expected a context error")
+	}
+	if elapsed := time.Since(start); elapsed > 10*time.Second {
+		t.Fatalf("upload did not abort on cancellation: took %v", elapsed)
+	}
+	if got := client.attempts(); got > 1 {
+		t.Fatalf("dial attempts = %d, want at most 1 after cancellation", got)
+	}
+}
+
+// A ciphered upload hands the volume server ciphertext, so it cannot echo a
+// Content-MD5 back; without the caller's plaintext digest the chunk lands with
+// no ETag and every ETag computed from those chunks comes out empty
+// (issue #10968).
+func TestCipherUploadKeepsPlaintextETag(t *testing.T) {
+	payload := bytes.Repeat([]byte("encrypt me\n"), 4096)
+	digest := md5.Sum(payload)
+	wantMd5 := base64.StdEncoding.EncodeToString(digest[:])
+
+	var storedNeedle *needle.Needle
+	var sentContentMd5 string
+	var parseErr error
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		sentContentMd5 = r.Header.Get("Content-MD5")
+		storedNeedle, _, _, parseErr = needle.CreateNeedleFromRequest(r, false, 1024*1024, &bytes.Buffer{})
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusCreated)
+		_, _ = io.WriteString(w, `{"name":"payload","size":45056}`)
+	}))
+	defer server.Close()
+
+	uploader := newUploader(server.Client())
+	result, err := uploader.UploadData(context.Background(), payload, &UploadOption{
+		UploadUrl:   server.URL + "/3,01637037d6",
+		Cipher:      true,
+		Md5:         wantMd5,
+		MaxAttempts: 1,
+	})
+	if err != nil {
+		t.Fatalf("cipher upload failed: %v", err)
+	}
+	if parseErr != nil {
+		t.Fatalf("parse ciphered upload: %v", parseErr)
+	}
+	if sentContentMd5 != "" {
+		t.Fatalf("Content-MD5 %q sent to the volume server, which only sees ciphertext", sentContentMd5)
+	}
+	if result.ContentMd5 != wantMd5 {
+		t.Fatalf("chunk ETag = %q, want the plaintext digest %q", result.ContentMd5, wantMd5)
+	}
+	if bytes.Equal(storedNeedle.Data, payload) {
+		t.Fatal("volume server stored plaintext for a ciphered upload")
+	}
+	decrypted, err := util.Decrypt(storedNeedle.Data, util.CipherKey(result.CipherKey))
+	if err != nil {
+		t.Fatalf("decrypt stored needle: %v", err)
+	}
+	if result.Gzip > 0 {
+		if decrypted, err = util.DecompressData(decrypted); err != nil {
+			t.Fatalf("decompress stored needle: %v", err)
+		}
+	}
+	if !bytes.Equal(decrypted, payload) {
+		t.Fatalf("decrypted %d bytes, want %d", len(decrypted), len(payload))
+	}
+}

weed/operation/volume_move/ec_move.go

@@ -0,0 +1,215 @@
+package volume_move
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// EcShardMove is one planned relocation of mounted shards to another server.
+type EcShardMove struct {
+	VolumeId   needle.VolumeId
+	Collection string
+	ShardIds   []erasure_coding.ShardId
+	Source     pb.ServerAddress
+	Target     pb.ServerAddress
+	// TargetDisk picks the destination disk (0 lets the server pick).
+	TargetDisk uint32
+}
+
+// EcMoveOptions control MoveEcShards.
+type EcMoveOptions struct {
+	// IoBytePerSecond limits the shard copy rate; 0 falls back to the volume
+	// server's maintenance rate.
+	IoBytePerSecond int64
+	// Writer receives human-readable progress lines (nil discards them).
+	Writer io.Writer
+	// Progress, when set, receives percent/stage callbacks as the move advances.
+	Progress func(percent float64, stage string)
+}
+
+// MoveEcShards relocates mounted shards: copy+mount on the target, verify the
+// target registered them, then unmount+delete on the source. The verification
+// gates the destructive half — a copy/mount RPC can return OK while the shard
+// is not loadable on the target, and deleting the source then would lose it;
+// on a mismatch the source is kept so the caller can retry.
+func (m *Mover) MoveEcShards(ctx context.Context, move EcShardMove, opts EcMoveOptions) error {
+	writer := opts.Writer
+	if writer == nil {
+		writer = io.Discard
+	}
+	progress := opts.Progress
+	if progress == nil {
+		progress = func(float64, string) {}
+	}
+
+	// A same-server "move" cannot be expressed with these RPCs: the source
+	// delete is server-wide, so it would erase the just-copied shard. Removing
+	// a duplicate shard in place is RemoveEcShards. SameServer, not ==:
+	// "node:8080" and "node:8080.18080" are one server.
+	if SameServer(move.Source, move.Target) {
+		return fmt.Errorf("refusing EC shard move of volume %d shard(s) %v onto its own server %s: the source delete is server-wide", move.VolumeId, move.ShardIds, move.Source)
+	}
+
+	progress(10, fmt.Sprintf("copying EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
+	if err := m.CopyAndMountEcShards(ctx, move.VolumeId, move.Collection, move.ShardIds, move.Source, move.Target, move.TargetDisk, opts.IoBytePerSecond, writer); err != nil {
+		return err
+	}
+
+	progress(40, fmt.Sprintf("verifying EC shard(s) %d.%v on %s", move.VolumeId, move.ShardIds, move.Target))
+	if err := m.VerifyEcShards(ctx, move.VolumeId, move.Target, move.ShardIds); err != nil {
+		return err
+	}
+
+	progress(50, fmt.Sprintf("unmounting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
+	fmt.Fprintf(writer, "unmount %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
+	if err := m.UnmountEcShards(ctx, move.VolumeId, move.Source, move.ShardIds); err != nil {
+		return fmt.Errorf("unmount %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
+	}
+
+	progress(75, fmt.Sprintf("deleting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
+	fmt.Fprintf(writer, "delete %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
+	if err := m.DeleteEcShards(ctx, move.VolumeId, move.Collection, move.Source, move.ShardIds); err != nil {
+		return fmt.Errorf("delete %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
+	}
+
+	progress(100, fmt.Sprintf("moved EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
+	return nil
+}
+
+// CopyAndMountEcShards has the target copy the shards (with their .ecx/.ecj/
+// .vif/.ecsum sidecars) from source and mount them. A same-address call skips
+// the copy and just mounts — ec.encode uses that to bring freshly generated
+// shards online in place.
+func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, source, target pb.ServerAddress, targetDisk uint32, ioBytePerSecond int64, writer io.Writer) error {
+	if writer == nil {
+		writer = io.Discard
+	}
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if !SameServer(target, source) {
+		if err := checkDialable(source); err != nil {
+			return err
+		}
+	}
+	return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
+		if !SameServer(target, source) {
+			fmt.Fprintf(writer, "copy %d.%v %s => %s\n", volumeId, shardIds, source, target)
+			_, copyErr := client.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:        uint32(volumeId),
+				Collection:      collection,
+				ShardIds:        erasure_coding.ShardIdsToUint32(shardIds),
+				CopyEcxFile:     true,
+				CopyEcjFile:     true,
+				CopyVifFile:     true,
+				CopyEcsumFile:   true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
+				SourceDataNode:  string(source),
+				DiskId:          targetDisk,
+				IoBytePerSecond: ioBytePerSecond,
+			})
+			if copyErr != nil {
+				return fmt.Errorf("copy %d.%v %s => %s: %v", volumeId, shardIds, source, target, copyErr)
+			}
+		}
+		fmt.Fprintf(writer, "mount %d.%v on %s\n", volumeId, shardIds, target)
+		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		if mountErr != nil {
+			return fmt.Errorf("mount %d.%v on %s: %v", volumeId, shardIds, target, mountErr)
+		}
+		return nil
+	})
+}
+
+// VerifyEcShards confirms server has every shard in shardIds registered for
+// the volume.
+func (m *Mover) VerifyEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, err := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+			VolumeId: uint32(volumeId),
+		})
+		if err != nil {
+			return fmt.Errorf("verify EC shard(s) on %s for volume %d: %v", server, volumeId, err)
+		}
+		var bits, zeroSized erasure_coding.ShardBits
+		for _, s := range resp.EcShardInfos {
+			if s.VolumeId != uint32(volumeId) || s.ShardId >= erasure_coding.MaxShardCount {
+				continue
+			}
+			// A zero-sized shard is residue of a failed operation, not a
+			// shard; counting it as present would let a broken copy pass
+			// verification and the source be deleted behind it.
+			if s.Size <= 0 {
+				zeroSized = zeroSized.Set(erasure_coding.ShardId(s.ShardId))
+				continue
+			}
+			bits = bits.Set(erasure_coding.ShardId(s.ShardId))
+		}
+		for _, sid := range shardIds {
+			if bits.Has(sid) {
+				continue
+			}
+			if zeroSized.Has(sid) {
+				return fmt.Errorf("%s has a zero-sized EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
+			}
+			return fmt.Errorf("%s missing EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
+		}
+		return nil
+	})
+}
+
+// MountEcShards mounts shards already on server.
+func (m *Mover) MountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return mountErr
+	})
+}
+
+// UnmountEcShards unmounts shards on server.
+func (m *Mover) UnmountEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, unmountErr := client.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+			VolumeId: uint32(volumeId),
+			ShardIds: erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return unmountErr
+	})
+}
+
+// DeleteEcShards deletes shards on server. The delete is server-wide: it
+// removes the shards from every disk of the server.
+func (m *Mover) DeleteEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, deleteErr := client.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return deleteErr
+	})
+}
+
+// RemoveEcShards unmounts then deletes shards in place — the dedup path for a
+// shard that already has a copy elsewhere.
+func (m *Mover) RemoveEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	if err := m.UnmountEcShards(ctx, volumeId, server, shardIds); err != nil {
+		return fmt.Errorf("unmount %d.%v from %s: %v", volumeId, shardIds, server, err)
+	}
+	if err := m.DeleteEcShards(ctx, volumeId, collection, server, shardIds); err != nil {
+		return fmt.Errorf("delete %d.%v from %s: %v", volumeId, shardIds, server, err)
+	}
+	return nil
+}

weed/operation/volume_move/ec_move_test.go

@@ -0,0 +1,137 @@
+package volume_move
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+func ecMove(shardIds ...erasure_coding.ShardId) EcShardMove {
+	return EcShardMove{
+		VolumeId:   7,
+		Collection: "c1",
+		ShardIds:   shardIds,
+		Source:     srcAddr,
+		Target:     dstAddr,
+		TargetDisk: 2,
+	}
+}
+
+func dstShards(shardIds ...uint32) []*volume_server_pb.EcShardInfo {
+	var infos []*volume_server_pb.EcShardInfo
+	for _, sid := range shardIds {
+		infos = append(infos, &volume_server_pb.EcShardInfo{VolumeId: 7, ShardId: sid, Size: 1024})
+	}
+	return infos
+}
+
+func TestMoveEcShardsSequence(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3, 4)
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{IoBytePerSecond: 77})
+	if err != nil {
+		t.Fatalf("MoveEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"dst:8080 VolumeEcShardsCopy",
+		"dst:8080 VolumeEcShardsMount",
+		"dst:8080 VolumeEcShardsInfo",
+		"src:8080 VolumeEcShardsUnmount",
+		"src:8080 VolumeEcShardsDelete",
+	})
+
+	copyReq := cluster.ecCopyReqs[0]
+	if !copyReq.CopyEcxFile || !copyReq.CopyEcjFile || !copyReq.CopyVifFile || !copyReq.CopyEcsumFile {
+		t.Errorf("shard sidecars not all copied: %+v", copyReq)
+	}
+	if copyReq.DiskId != 2 || copyReq.SourceDataNode != string(srcAddr) || copyReq.Collection != "c1" || copyReq.IoBytePerSecond != 77 {
+		t.Errorf("copy request not propagated: %+v", copyReq)
+	}
+}
+
+func TestMoveEcShardsVerifyFailureKeepsSource(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3) // shard 4 didn't register
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "missing EC shard 7.4") {
+		t.Fatalf("expected missing-shard error, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeEcShardsUnmount" || call == "src:8080 VolumeEcShardsDelete" {
+			t.Fatalf("source touched despite verification failure: %v", cluster.callList())
+		}
+	}
+}
+
+func TestMoveEcShardsZeroSizedDestinationKeepsSource(t *testing.T) {
+	// A zero-sized shard on the destination is residue of a failed operation
+	// (seaweedfs issue 10730); verification must not count it as a delivered
+	// shard, or the source is deleted behind a broken copy.
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3, 4)
+	cluster.ecShards[string(dstAddr)][1].Size = 0 // shard 4 landed as an empty file
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "zero-sized EC shard 7.4") {
+		t.Fatalf("expected zero-sized-shard rejection, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeEcShardsUnmount" || call == "src:8080 VolumeEcShardsDelete" {
+			t.Fatalf("source touched despite zero-sized destination shard: %v", cluster.callList())
+		}
+	}
+}
+
+func TestMoveEcShardsRejectsSameServer(t *testing.T) {
+	// The second target is the same server written with an explicit grpc port;
+	// the guard must see through the representation difference.
+	for _, target := range []pb.ServerAddress{srcAddr, pb.ServerAddress("src:8080.18080")} {
+		cluster := newFakeCluster()
+		move := ecMove(3)
+		move.Target = target
+
+		err := cluster.mover().MoveEcShards(context.Background(), move, EcMoveOptions{})
+		if err == nil || !strings.Contains(err.Error(), "its own server") {
+			t.Fatalf("target %q: expected same-server rejection, got: %v", target, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Fatalf("target %q: RPCs issued for a rejected move: %v", target, cluster.callList())
+		}
+	}
+}
+
+func TestRemoveEcShards(t *testing.T) {
+	cluster := newFakeCluster()
+
+	err := cluster.mover().RemoveEcShards(context.Background(), 7, "c1", srcAddr, []erasure_coding.ShardId{3})
+	if err != nil {
+		t.Fatalf("RemoveEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeEcShardsUnmount",
+		"src:8080 VolumeEcShardsDelete",
+	})
+}
+
+func TestCopyAndMountEcShardsSameAddressMountsOnly(t *testing.T) {
+	cluster := newFakeCluster()
+
+	err := cluster.mover().CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, srcAddr, srcAddr, 0, 0, nil)
+	if err != nil {
+		t.Fatalf("CopyAndMountEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeEcShardsMount",
+	})
+}

weed/operation/volume_move/fake_client_test.go

@@ -0,0 +1,230 @@
+package volume_move
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// fakeCluster fakes the volume servers a move touches. It records every RPC as
+// "<server> <method>" so tests can assert the exact sequence, and lets tests
+// inject per-RPC errors and per-server state.
+type fakeCluster struct {
+	mu    sync.Mutex
+	calls []string
+
+	// errs fails an RPC, keyed "<server> <method>".
+	errs map[string]error
+	// readonly is each server's VolumeStatus.IsReadOnly answer.
+	readonly map[string]bool
+	// status is each server's ReadVolumeFileStatus answer.
+	status map[string]*volume_server_pb.ReadVolumeFileStatusResponse
+	// statusFailures fails a server's next N ReadVolumeFileStatus calls, e.g.
+	// to model a target that has no volume before the copy but one after. The
+	// failures use statusFailureErr, defaulting to a server-side (gRPC code
+	// Unknown) volume-not-found error.
+	statusFailures   map[string]int
+	statusFailureErr error
+	// statusSeq overrides a server's next ReadVolumeFileStatus answers in
+	// order, e.g. to model a source that changes between two reads.
+	statusSeq map[string][]*volume_server_pb.ReadVolumeFileStatusResponse
+	// ecShards is each server's VolumeEcShardsInfo answer.
+	ecShards map[string][]*volume_server_pb.EcShardInfo
+
+	lastAppendAtNs uint64
+
+	copyReqs   []*volume_server_pb.VolumeCopyRequest
+	deleteReqs []*volume_server_pb.VolumeDeleteRequest
+	ecCopyReqs []*volume_server_pb.VolumeEcShardsCopyRequest
+}
+
+func newFakeCluster() *fakeCluster {
+	return &fakeCluster{
+		errs:           make(map[string]error),
+		readonly:       make(map[string]bool),
+		status:         make(map[string]*volume_server_pb.ReadVolumeFileStatusResponse),
+		statusFailures: make(map[string]int),
+		statusSeq:      make(map[string][]*volume_server_pb.ReadVolumeFileStatusResponse),
+		ecShards:       make(map[string][]*volume_server_pb.EcShardInfo),
+	}
+}
+
+func (c *fakeCluster) mover() *Mover {
+	return NewMoverWithClientFunc(func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error {
+		return fn(&fakeClient{cluster: c, addr: string(addr)})
+	})
+}
+
+func (c *fakeCluster) record(addr, method string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls = append(c.calls, addr+" "+method)
+	return c.errs[addr+" "+method]
+}
+
+func (c *fakeCluster) callList() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]string(nil), c.calls...)
+}
+
+// fakeClient answers as the volume server at addr. Unused VolumeServerClient
+// methods panic via the embedded nil interface.
+type fakeClient struct {
+	volume_server_pb.VolumeServerClient
+	cluster *fakeCluster
+	addr    string
+}
+
+func (f *fakeClient) VolumeStatus(ctx context.Context, req *volume_server_pb.VolumeStatusRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeStatusResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeStatus"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeStatusResponse{IsReadOnly: f.cluster.readonly[f.addr]}, nil
+}
+
+func (f *fakeClient) VolumeMarkReadonly(ctx context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeMarkReadonly"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
+}
+
+func (f *fakeClient) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeMarkWritableResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeMarkWritable"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeMarkWritableResponse{}, nil
+}
+
+func (f *fakeClient) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
+	if err := f.cluster.record(f.addr, "ReadVolumeFileStatus"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	failures := f.cluster.statusFailures[f.addr]
+	if failures > 0 {
+		f.cluster.statusFailures[f.addr] = failures - 1
+	}
+	failureErr := f.cluster.statusFailureErr
+	var seqResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if failures == 0 {
+		if seq := f.cluster.statusSeq[f.addr]; len(seq) > 0 {
+			seqResp = seq[0]
+			f.cluster.statusSeq[f.addr] = seq[1:]
+		}
+	}
+	resp := f.cluster.status[f.addr]
+	f.cluster.mu.Unlock()
+	if failures > 0 {
+		if failureErr != nil {
+			return nil, failureErr
+		}
+		return nil, status.Error(codes.Unknown, fmt.Sprintf("not found volume id %d", req.VolumeId))
+	}
+	if seqResp != nil {
+		return seqResp, nil
+	}
+	if resp == nil {
+		return nil, status.Error(codes.Unknown, fmt.Sprintf("not found volume id %d", req.VolumeId))
+	}
+	return resp, nil
+}
+
+func (f *fakeClient) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[volume_server_pb.VolumeCopyResponse], error) {
+	if err := f.cluster.record(f.addr, "VolumeCopy"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.copyReqs = append(f.cluster.copyReqs, req)
+	lastAppendAtNs := f.cluster.lastAppendAtNs
+	f.cluster.mu.Unlock()
+	return &fakeCopyStream{resps: []*volume_server_pb.VolumeCopyResponse{
+		{ProcessedBytes: 1024},
+		{LastAppendAtNs: lastAppendAtNs},
+	}}, nil
+}
+
+func (f *fakeClient) VolumeTailReceiver(ctx context.Context, req *volume_server_pb.VolumeTailReceiverRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeTailReceiverResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeTailReceiver"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
+}
+
+func (f *fakeClient) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeDeleteResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeDelete"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.deleteReqs = append(f.cluster.deleteReqs, req)
+	f.cluster.mu.Unlock()
+	return &volume_server_pb.VolumeDeleteResponse{}, nil
+}
+
+func (f *fakeClient) VolumeConfigure(ctx context.Context, req *volume_server_pb.VolumeConfigureRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeConfigureResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeConfigure"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeConfigureResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsCopy"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.ecCopyReqs = append(f.cluster.ecCopyReqs, req)
+	f.cluster.mu.Unlock()
+	return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsMount"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsUnmount"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsDelete"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsInfo"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsInfoResponse{EcShardInfos: f.cluster.ecShards[f.addr]}, nil
+}
+
+// fakeCopyStream feeds the canned VolumeCopy responses, then io.EOF.
+type fakeCopyStream struct {
+	grpc.ClientStream
+	resps []*volume_server_pb.VolumeCopyResponse
+}
+
+func (s *fakeCopyStream) Recv() (*volume_server_pb.VolumeCopyResponse, error) {
+	if len(s.resps) == 0 {
+		return nil, io.EOF
+	}
+	resp := s.resps[0]
+	s.resps = s.resps[1:]
+	return resp, nil
+}

weed/operation/volume_move/mover.go

@@ -0,0 +1,113 @@
+// Package volume_move implements the volume and EC shard move sequences shared
+// by the interactive shell commands (volume.move, volume.balance, ec.balance,
+// volume.tier.move, ...) and the maintenance workers (balance, ec_balance,
+// volume_tiering). The RPCs go through an injectable ClientFunc so every
+// sequence can be unit tested against a fake volume server client.
+package volume_move
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+// ClientFunc runs fn with a client for the volume server at addr. It matches
+// operation.WithVolumeServerClient, which production movers use to dial real
+// servers; tests substitute a fake client.
+type ClientFunc func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error
+
+// Mover executes volume and EC shard moves against volume servers.
+type Mover struct {
+	withClient ClientFunc
+}
+
+func NewMover(grpcDialOption grpc.DialOption) *Mover {
+	return &Mover{withClient: func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error {
+		// Validated here, the single point every mover RPC dials through:
+		// handing a malformed address (an unvalidated -source/-target flag)
+		// to the dialer would abort the whole process instead of failing the
+		// move.
+		if err := checkDialable(addr); err != nil {
+			return err
+		}
+		return operation.WithVolumeServerClient(streamingMode, addr, grpcDialOption, fn)
+	}}
+}
+
+// checkDialable rejects an address whose grpc normalization would abort the
+// process: for the "host:port" form, ServerAddress.ToGrpcAddress falls back
+// to a parser that calls glog.Fatalf when the port is not numeric. Everything
+// else either normalizes cleanly or fails at dial time as an ordinary error.
+// It also guards the source addresses embedded in copy/tail requests: the
+// receiving volume server dials those through the same fatal parser, so an
+// unchecked malformed source would terminate the destination server.
+func checkDialable(addr pb.ServerAddress) error {
+	s := string(addr)
+	colon := strings.LastIndex(s, ":")
+	if colon < 0 || colon+1 >= len(s) {
+		return nil // no port part; handed to the dialer untouched
+	}
+	ports := s[colon+1:]
+	if dot := strings.LastIndex(ports, "."); dot >= 0 {
+		// "port.grpcPort": different dial paths canonicalize a half-malformed
+		// form differently (the method splices the grpc part in unparsed; the
+		// string helper falls back to the http part + 10000), so a bad
+		// component could dial an unintended server or reach the fatal
+		// parser. Require both components numeric.
+		if _, err := strconv.ParseUint(ports[:dot], 10, 64); err != nil {
+			return fmt.Errorf("invalid volume server address %q: port %q is not a number", s, ports[:dot])
+		}
+		if _, err := strconv.ParseUint(ports[dot+1:], 10, 64); err != nil {
+			return fmt.Errorf("invalid volume server address %q: grpc port %q is not a number", s, ports[dot+1:])
+		}
+		return nil
+	}
+	if _, err := strconv.ParseUint(ports, 10, 64); err != nil {
+		return fmt.Errorf("invalid volume server address %q: port %q is not a number", s, ports)
+	}
+	return nil
+}
+
+// NewMoverWithClientFunc builds a Mover on a custom transport.
+func NewMoverWithClientFunc(withClient ClientFunc) *Mover {
+	return &Mover{withClient: withClient}
+}
+
+// SameServer reports whether two addresses name the same volume server. The
+// gRPC endpoint identifies the server process — each has exactly one — so
+// "node:8080" and "node:8080.18080" compare equal, while two servers sharing
+// a degenerate HTTP address (e.g. port 0 in test harnesses) stay distinct.
+// The normalization is non-fatal, unlike ServerAddress.ToGrpcAddress, whose
+// parser exits the process on a malformed port; anything unparsable compares
+// as its literal self.
+func SameServer(a, b pb.ServerAddress) bool {
+	return grpcEndpoint(string(a)) == grpcEndpoint(string(b))
+}
+
+// grpcEndpoint mirrors ServerAddress.ToGrpcAddress ("host:port" gets the
+// +10000 grpc port; "host:port.grpcPort" names it explicitly) but returns a
+// malformed address unchanged instead of exiting.
+func grpcEndpoint(addr string) string {
+	colon := strings.LastIndex(addr, ":")
+	if colon < 0 || colon+1 >= len(addr) {
+		return addr
+	}
+	host, ports := addr[:colon], addr[colon+1:]
+	if dot := strings.LastIndex(ports, "."); dot >= 0 {
+		if grpcPort, err := strconv.Atoi(ports[dot+1:]); err == nil {
+			return util.JoinHostPort(host, grpcPort)
+		}
+		return addr
+	}
+	httpPort, err := strconv.Atoi(ports)
+	if err != nil {
+		return addr
+	}
+	return util.JoinHostPort(host, httpPort+10000)
+}

weed/operation/volume_move/volume_move.go

@@ -0,0 +1,503 @@
+package volume_move
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ErrSourceKeptReadonly marks a move failure whose recovery deliberately keeps
+// the source volume readonly — the target copy may hold data the source lacks
+// (an ambiguous source delete, or an undeletable stale copy). Callers that
+// froze the source themselves must not thaw it on this error.
+var ErrSourceKeptReadonly = errors.New("the source volume is deliberately kept readonly")
+
+// VolumeMoveOptions control LiveMoveVolume.
+type VolumeMoveOptions struct {
+	// DiskType changes the volume's disk type on the target ("" keeps it).
+	DiskType string
+	// IoBytePerSecond throttles the copy (0 = unlimited).
+	IoBytePerSecond int64
+	// IdleTimeout is how long the tail phase waits for in-flight requests to
+	// drain (default 5s).
+	IdleTimeout time.Duration
+	// Writer receives human-readable progress lines (nil discards them).
+	Writer io.Writer
+	// Progress, when set, receives percent/stage callbacks as the move advances.
+	Progress func(percent float64, stage string)
+}
+
+func (o *VolumeMoveOptions) fillDefaults() {
+	if o.IdleTimeout <= 0 {
+		o.IdleTimeout = 5 * time.Second
+	}
+	if o.Writer == nil {
+		o.Writer = io.Discard
+	}
+	if o.Progress == nil {
+		o.Progress = func(float64, string) {}
+	}
+}
+
+// LiveMoveVolume moves one volume between volume servers while it keeps serving
+// reads: freeze the source (readonly), copy, tail to drain in-flight requests,
+// verify the target matches the source, then delete the source. A failure
+// before the source delete restores the source's writability if this move is
+// what froze it.
+func (m *Mover) LiveMoveVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, opts VolumeMoveOptions) (err error) {
+	opts.fillDefaults()
+
+	// VolumeCopy tears down any existing copy on the target first, so a
+	// same-server "move" would delete the volume and then fail to read it.
+	// SameServer, not ==: "node:8080" and "node:8080.18080" are one server.
+	if SameServer(source, target) {
+		return fmt.Errorf("refusing to move volume %d onto its own server %s", volumeId, source)
+	}
+
+	opts.Progress(10, fmt.Sprintf("marking volume %d readonly on %s", volumeId, source))
+	sourceWasWritable, err := m.ensureVolumeReadonly(ctx, volumeId, source, true)
+	// The source stays authoritative until its delete succeeds; on any earlier
+	// failure undo the freeze this move added and clean up the target copy.
+	// Installed before the error check: the marking RPC itself can fail after
+	// the server applied the mark (e.g. its master notification failed).
+	var copyStarted, copyCompleted, sourceDeleteStarted bool
+	var targetHadVolume, targetStateKnown bool
+	defer func() {
+		if err != nil {
+			if sourceDeleteStarted {
+				// Past verification: the target is complete and may already hold
+				// new writes, and the delete may or may not have reached the
+				// source. Deleting the target or reopening the source could fork
+				// the volume; keep the source readonly.
+				fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: the source delete failed with the verified copy on %s mounted\n", volumeId, source, target)
+				glog.Warningf("volume %d is left readonly on %s: the source delete failed with the verified copy on %s mounted", volumeId, source, target)
+				err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+				return
+			}
+			// The cleanup runs on its own deadline so an abort via cancelled
+			// context still cleans up.
+			cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+			defer cleanupCancel()
+			cleanupTarget := copyCompleted
+			if copyStarted && !copyCompleted {
+				// The server can finish the copy and mount the target even when
+				// the client loses the stream, so probe rather than assume.
+				exists, known := m.probeVolume(cleanupCtx, volumeId, target)
+				switch {
+				case known && !exists:
+					// nothing mounted on the target; just undo the freeze below
+				case known && exists && targetStateKnown && !targetHadVolume:
+					// the failed copy created it; remove it so the source stays
+					// the only replica
+					cleanupTarget = true
+				default:
+					// A copy may sit mounted on the target and its provenance
+					// cannot be proven: the prior state is unknown, the target
+					// held a replica before the move, or the probe failed.
+					// Deleting it risks someone else's replica; reopening the
+					// source beside it risks two writable replicas taking
+					// divergent writes. Keep the source readonly.
+					fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: a copy may exist on %s but its origin cannot be determined; delete one side explicitly, then re-run the move\n", volumeId, source, target)
+					glog.Warningf("volume %d is left readonly on %s: a copy may exist on %s but its origin cannot be determined; delete one side explicitly, then re-run the move", volumeId, source, target)
+					err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+					return
+				}
+			}
+			if cleanupTarget {
+				// The target copy may be missing tailed entries; remove it so
+				// the source stays the only replica.
+				if dErr := m.DeleteVolume(cleanupCtx, volumeId, target, false, true); dErr != nil {
+					// Restoring the source while the stale target stays mounted
+					// risks divergent replicas; keep the source readonly. A
+					// re-run refuses while the copy exists, so name the fix.
+					fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: failed to delete the incomplete copy on %s: %v; delete that copy, then re-run the move\n", volumeId, source, target, dErr)
+					glog.Warningf("volume %d is left readonly on %s: failed to delete the incomplete copy on %s: %v; delete that copy, then re-run the move", volumeId, source, target, dErr)
+					err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+					return
+				}
+			}
+			if sourceWasWritable {
+				m.restoreVolumeWritable(volumeId, source)
+			} else {
+				// The hard freeze added on top of a readonly-reporting status is
+				// kept: it is indistinguishable from an operator's mark. It is
+				// not persisted, so a volume server restart clears it.
+				fmt.Fprintf(opts.Writer, "volume %d on %s keeps the readonly mark the move added; volume.mark -writable clears it if the prior readonly state was transient\n", volumeId, source)
+			}
+		}
+	}()
+	if err != nil {
+		return fmt.Errorf("mark volume %d readonly on %s: %v", volumeId, source, err)
+	}
+
+	targetHadVolume, targetStateKnown = m.probeVolume(ctx, volumeId, target)
+	if !sourceWasWritable {
+		// A readonly source next to an existing copy is the signature of a
+		// previous move whose source delete failed — that copy may be the
+		// authoritative one, serving writes the source never saw, and the
+		// copy below would tear it down. No client-side observation proves
+		// otherwise: compaction revision is shared by ordinary replicas,
+		// aggregate sizes shrink under compaction, and any snapshot can be
+		// invalidated by a write right after it. Fail closed on any existing
+		// copy or unknown state; the operator deletes one side explicitly.
+		if !targetStateKnown {
+			err = fmt.Errorf("%w: cannot determine whether %s already holds a copy of volume %d; refusing to overwrite it while the source is readonly", ErrSourceKeptReadonly, target, volumeId)
+			return err
+		}
+		if targetHadVolume {
+			err = fmt.Errorf("%w: volume %d already has a copy on %s while the source on %s is readonly — likely a previous move that did not finish; delete the source to keep that copy, or delete the copy before re-running the move", ErrSourceKeptReadonly, volumeId, target, source)
+			return err
+		}
+	}
+
+	opts.Progress(20, fmt.Sprintf("copying volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "copying volume %d from %s to %s\n", volumeId, source, target)
+	copyStarted = true
+	lastAppendAtNs, err := m.copyVolumeData(ctx, volumeId, source, target, opts.DiskType, opts.IoBytePerSecond, opts.Writer)
+	if err != nil {
+		return fmt.Errorf("copy volume %d from %s to %s: %v", volumeId, source, target, err)
+	}
+	copyCompleted = true
+
+	opts.Progress(70, fmt.Sprintf("tailing volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "tailing volume %d from %s to %s\n", volumeId, source, target)
+	tailFailed := false
+	if tailErr := m.TailVolume(ctx, volumeId, source, target, lastAppendAtNs, opts.IdleTimeout); tailErr != nil {
+		// A tail failure is tolerable only when the volume was already readonly
+		// before this move began: frozen for the whole copy, it should have no
+		// in-flight writes for the tail to drain — and the stability check
+		// below still proves it. A volume this move froze can have stragglers
+		// admitted just before the freeze that only the tail delivers, so
+		// losing the tail there must abort the move.
+		if sourceWasWritable {
+			return fmt.Errorf("tail volume %d from %s to %s: %v", volumeId, source, target, tailErr)
+		}
+		tailFailed = true
+		fmt.Fprintf(opts.Writer, "tail volume %d from %s to %s: %v\n", volumeId, source, target, tailErr)
+		glog.Warningf("tail volume %d from %s to %s: %v", volumeId, source, target, tailErr)
+	}
+
+	// Verify before the point of no return: the source is deleted only when the
+	// target holds at least everything the source does. The source status is
+	// read here, after the tail — a write in flight when the source was frozen
+	// can still land after an earlier read, and a stale snapshot would wave
+	// through an incomplete target.
+	opts.Progress(85, fmt.Sprintf("verifying volume %d on %s", volumeId, target))
+	sourceStatus, err := m.ReadVolumeFileStatus(ctx, volumeId, source)
+	if err != nil {
+		return fmt.Errorf("read volume %d status on %s: %v", volumeId, source, err)
+	}
+	if tailFailed {
+		// A successful tail proves the source held still for the idle window; a
+		// tolerated tail failure proved nothing, so substitute the same drain
+		// barrier here: the source status must not change across the window.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(opts.IdleTimeout):
+		}
+		secondStatus, statusErr := m.ReadVolumeFileStatus(ctx, volumeId, source)
+		if statusErr != nil {
+			return fmt.Errorf("re-read volume %d status on %s: %v", volumeId, source, statusErr)
+		}
+		if secondStatus.DatFileSize != sourceStatus.DatFileSize || secondStatus.IdxFileSize != sourceStatus.IdxFileSize || secondStatus.FileCount != sourceStatus.FileCount {
+			return fmt.Errorf("volume %d on %s is still changing after a failed tail; aborting the move", volumeId, source)
+		}
+		sourceStatus = secondStatus
+	}
+	targetStatus, err := m.ReadVolumeFileStatus(ctx, volumeId, target)
+	if err != nil {
+		return fmt.Errorf("verify volume %d on target %s before deleting source: %v", volumeId, target, err)
+	}
+	if err = verifyTargetNotBehind(volumeId, sourceStatus, targetStatus); err != nil {
+		return err
+	}
+	if targetStatus.DatFileSize > sourceStatus.DatFileSize || targetStatus.IdxFileSize > sourceStatus.IdxFileSize || targetStatus.FileCount > sourceStatus.FileCount {
+		// The target announced itself writable when the copy mounted it, so
+		// clients may have written to it during the tail. Those writes live on
+		// the surviving copy; committing keeps them.
+		fmt.Fprintf(opts.Writer, "volume %d on %s has writes beyond the source; they stay with the moved volume\n", volumeId, target)
+	}
+
+	opts.Progress(90, fmt.Sprintf("deleting volume %d from %s", volumeId, source))
+	fmt.Fprintf(opts.Writer, "deleting volume %d from %s\n", volumeId, source)
+	sourceDeleteStarted = true
+	if err = m.DeleteVolume(ctx, volumeId, source, false, true); err != nil {
+		return fmt.Errorf("delete volume %d from %s: %v", volumeId, source, err)
+	}
+
+	opts.Progress(100, fmt.Sprintf("moved volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "moved volume %d from %s to %s\n", volumeId, source, target)
+	return nil
+}
+
+// verifyTargetNotBehind fails when the target holds less than the (frozen)
+// source — the copy or tail missed data and deleting the source would lose it.
+// A target that is ahead is not an error: the target serves writes during the
+// tail, and those belong to the copy that survives the move.
+func verifyTargetNotBehind(volumeId needle.VolumeId, source, target *volume_server_pb.ReadVolumeFileStatusResponse) error {
+	if target.DatFileSize < source.DatFileSize {
+		return fmt.Errorf("volume %d target is behind the source: .dat %d < %d bytes", volumeId, target.DatFileSize, source.DatFileSize)
+	}
+	if target.IdxFileSize < source.IdxFileSize {
+		return fmt.Errorf("volume %d target is behind the source: .idx %d < %d bytes", volumeId, target.IdxFileSize, source.IdxFileSize)
+	}
+	if target.FileCount < source.FileCount {
+		return fmt.Errorf("volume %d target is behind the source: %d < %d files", volumeId, target.FileCount, source.FileCount)
+	}
+	return nil
+}
+
+// CopyVolume freezes the volume on source, copies it to target, and reports the
+// stamp of the last entry copied. restoreWritable also restores the source's
+// writability on success, for copies that leave the source serving; a failed
+// copy always undoes the freeze this call added.
+func (m *Mover) CopyVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, restoreWritable bool, writer io.Writer) (lastAppendAtNs uint64, err error) {
+	if writer == nil {
+		writer = io.Discard
+	}
+	if SameServer(source, target) {
+		return 0, fmt.Errorf("refusing to copy volume %d onto its own server %s", volumeId, source)
+	}
+	// The copy is non-destructive, so a readonly-reporting source is taken as
+	// is (force=false): missed concurrent deletes only make the new replica
+	// trivially stale, and hard-marking here could pin a transiently readonly
+	// (e.g. low-disk) source readonly with nothing to gate on it.
+	sourceWasWritable, err := m.ensureVolumeReadonly(ctx, volumeId, source, false)
+	defer func() {
+		if sourceWasWritable && (err != nil || restoreWritable) {
+			m.restoreVolumeWritable(volumeId, source)
+		}
+	}()
+	if err != nil {
+		return 0, err
+	}
+	return m.copyVolumeData(ctx, volumeId, source, target, diskType, ioBytePerSecond, writer)
+}
+
+// ensureVolumeReadonly freezes the volume on server and reports whether it was
+// writable beforehand — only then may a failure path undo the freeze, since a
+// volume that was already readonly (e.g. full, or operator-set) must stay so.
+// With force, the mark is issued even when the status already reports
+// readonly: that answer also covers transient low-disk state and the
+// readonly-but-can-delete flag, and neither blocks needle deletes, so only the
+// hard mark makes the volume immutable — required before deleting the source
+// of a move. Without force, a readonly-reporting volume is left untouched.
+func (m *Mover) ensureVolumeReadonly(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, force bool) (wasWritable bool, err error) {
+	err = m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, statusErr := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
+			VolumeId: uint32(volumeId),
+		})
+		if statusErr != nil {
+			return statusErr
+		}
+		wasWritable = !resp.IsReadOnly
+		if !wasWritable && !force {
+			return nil
+		}
+		_, readonlyErr := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+			VolumeId: uint32(volumeId),
+			Persist:  false,
+		})
+		return readonlyErr
+	})
+	return
+}
+
+// probeVolume reports whether server currently has the volume, and whether the
+// answer is trustworthy. The Go server answers a missing volume with a plain
+// error (gRPC code Unknown — the only way this RPC fails there); the Rust
+// server answers with codes.NotFound. Both mean the server responded and the
+// volume is absent. A transport-level failure means the state is unknown, and
+// callers deciding to delete must treat unknown as hands-off.
+func (m *Mover) probeVolume(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress) (exists bool, known bool) {
+	_, err := m.ReadVolumeFileStatus(ctx, volumeId, server)
+	if err == nil {
+		return true, true
+	}
+	if s, ok := status.FromError(err); ok && (s.Code() == codes.Unknown || s.Code() == codes.NotFound) {
+		return false, true
+	}
+	return false, false
+}
+
+// restoreVolumeWritable undoes a readonly mark after a failed move. It runs on
+// its own deadline so an abort via cancelled context still restores the source,
+// and only logs a failure — the caller is already returning the move error.
+func (m *Mover) restoreVolumeWritable(volumeId needle.VolumeId, server pb.ServerAddress) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if err := m.MarkVolumeWritable(ctx, volumeId, server, true, false); err != nil {
+		glog.Warningf("restore volume %d writable on %s: %v", volumeId, server, err)
+	}
+}
+
+// copyVolumeData streams a VolumeCopy on the target, which pulls the volume
+// from source and mounts it, and returns the stamp of the last entry copied
+// for the tail phase.
+func (m *Mover) copyVolumeData(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, writer io.Writer) (lastAppendAtNs uint64, err error) {
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err = checkDialable(source); err != nil {
+		return 0, err
+	}
+	err = m.withClient(true, target, func(client volume_server_pb.VolumeServerClient) error {
+		stream, replicateErr := client.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+			VolumeId:        uint32(volumeId),
+			SourceDataNode:  string(source),
+			DiskType:        diskType,
+			IoBytePerSecond: ioBytePerSecond,
+		})
+		if replicateErr != nil {
+			return replicateErr
+		}
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				}
+				return recvErr
+			}
+			if resp.LastAppendAtNs != 0 {
+				lastAppendAtNs = resp.LastAppendAtNs
+			} else {
+				fmt.Fprintf(writer, "%s => %s volume %d processed %s\n", source, target, volumeId, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
+			}
+		}
+		return nil
+	})
+	return
+}
+
+// TailVolume has the target follow the source's appends since sinceNs until the
+// source stays idle for idleTimeout, draining requests in flight when the move
+// froze the source.
+func (m *Mover) TailVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, sinceNs uint64, idleTimeout time.Duration) error {
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err := checkDialable(source); err != nil {
+		return err
+	}
+	return m.withClient(true, target, func(client volume_server_pb.VolumeServerClient) error {
+		_, replicateErr := client.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+			VolumeId:           uint32(volumeId),
+			SinceNs:            sinceNs,
+			IdleTimeoutSeconds: uint32(idleTimeout.Seconds()),
+			SourceVolumeServer: string(source),
+		})
+		return replicateErr
+	})
+}
+
+// ReadVolumeFileStatus reads the volume's file sizes and needle count on server.
+func (m *Mover) ReadVolumeFileStatus(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress) (resp *volume_server_pb.ReadVolumeFileStatusResponse, err error) {
+	err = m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		var statusErr error
+		resp, statusErr = client.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+			VolumeId: uint32(volumeId),
+		})
+		return statusErr
+	})
+	return
+}
+
+// DeleteVolume removes the volume from server. When keepRemoteData is true, the
+// cloud-tier object backing the volume is left intact — used on the source side
+// of a move where another server is taking over the same .vif.
+func (m *Mover) DeleteVolume(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, deleteErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+			VolumeId:       uint32(volumeId),
+			OnlyEmpty:      onlyEmpty,
+			KeepRemoteData: keepRemoteData,
+		})
+		return deleteErr
+	})
+}
+
+// MarkVolumeWritable marks the volume writable (or readonly when writable is
+// false, persisted per persist) on server.
+func (m *Mover) MarkVolumeWritable(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, writable, persist bool) (err error) {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		if writable {
+			_, err = client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{
+				VolumeId: uint32(volumeId),
+			})
+		} else {
+			_, err = client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+				VolumeId: uint32(volumeId),
+				Persist:  persist,
+			})
+		}
+		return err
+	})
+}
+
+// ReplicateVolume copies a volume from source to target without touching the
+// source — the replica-creation half of a move. ioBytePerSecond limits the
+// copy rate; 0 falls back to the volume server's maintenance rate.
+func (m *Mover) ReplicateVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, writer io.Writer) error {
+	if writer == nil {
+		writer = io.Discard
+	}
+	if SameServer(source, target) {
+		return fmt.Errorf("refusing to replicate volume %d onto its own server %s", volumeId, source)
+	}
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err := checkDialable(source); err != nil {
+		return err
+	}
+	return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
+		stream, replicateErr := client.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+			VolumeId:        uint32(volumeId),
+			SourceDataNode:  string(source),
+			DiskType:        diskType,
+			IoBytePerSecond: ioBytePerSecond,
+		})
+		if replicateErr != nil {
+			return replicateErr
+		}
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				}
+				return recvErr
+			}
+			if resp.ProcessedBytes > 0 {
+				fmt.Fprintf(writer, "volume %d processed %s bytes\n", volumeId, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
+			}
+		}
+		return nil
+	})
+}
+
+// ConfigureVolumeReplication sets the replication setting on the volume at server.
+func (m *Mover) ConfigureVolumeReplication(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, replication string) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, configureErr := client.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+			VolumeId:    uint32(volumeId),
+			Replication: replication,
+		})
+		if configureErr != nil {
+			return configureErr
+		}
+		if resp.Error != "" {
+			return errors.New(resp.Error)
+		}
+		return nil
+	})
+}

weed/operation/volume_move/volume_move_test.go

@@ -0,0 +1,633 @@
+package volume_move
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	srcAddr = pb.ServerAddress("src:8080")
+	dstAddr = pb.ServerAddress("dst:8080")
+)
+
+func volumeStatus(datSize, idxSize, fileCount uint64) *volume_server_pb.ReadVolumeFileStatusResponse {
+	return &volume_server_pb.ReadVolumeFileStatusResponse{
+		DatFileSize: datSize,
+		IdxFileSize: idxSize,
+		FileCount:   fileCount,
+	}
+}
+
+func TestSameServer(t *testing.T) {
+	if !SameServer("node:8080", "node:8080.18080") {
+		t.Error("one server written with and without an explicit grpc port compared unequal")
+	}
+	// Test harnesses run several servers on HTTP port 0 with distinct grpc
+	// ports; the grpc endpoint is the server's identity.
+	if SameServer("127.0.0.1:0.36825", "127.0.0.1:0.38609") {
+		t.Error("two servers sharing a degenerate HTTP address compared equal")
+	}
+	// Malformed addresses (an unvalidated -source/-target flag) must compare
+	// without touching the fatal ToGrpcAddress parser.
+	if !SameServer("node:abc", "node:abc") {
+		t.Error("identical malformed addresses compared unequal")
+	}
+	if SameServer("node:abc", "node:8080") {
+		t.Error("malformed and well-formed addresses compared equal")
+	}
+	if SameServer("", "node:8080") {
+		t.Error("empty and well-formed addresses compared equal")
+	}
+}
+
+func TestCheckDialable(t *testing.T) {
+	// Rejected: the class the fatal parser aborts on, plus dotted forms with
+	// a non-numeric component — those canonicalize differently across dial
+	// paths ("node:8080.bad" dials as node:bad on one and node:18080 on the
+	// other, silently aliasing another server).
+	for _, addr := range []pb.ServerAddress{"node:abc", "host.domain.com:x8080", "[::1]:abc",
+		"node:8080.bad", "node:bad.8080", "node:.bad", "node:8080."} {
+		if err := checkDialable(addr); err == nil {
+			t.Errorf("%q accepted although it is unsafe to dial", addr)
+		}
+	}
+	// Accepted: normalized cleanly, or handed to the dialer untouched where
+	// a bad address fails as an ordinary error.
+	for _, addr := range []pb.ServerAddress{"node:8080", "node:8080.18080", "127.0.0.1:0.36825", "", "node", "node:"} {
+		if err := checkDialable(addr); err != nil {
+			t.Errorf("%q rejected: %v", addr, err)
+		}
+	}
+}
+
+func TestEmbeddedSourceAddressValidated(t *testing.T) {
+	// The target dials the source embedded in copy/tail requests through the
+	// same fatal parser; the client must reject a malformed source before
+	// issuing any RPC.
+	bad := pb.ServerAddress("node:abc")
+
+	ops := map[string]func(m *Mover) error{
+		"ReplicateVolume": func(m *Mover) error {
+			return m.ReplicateVolume(context.Background(), 7, bad, dstAddr, "", 0, nil)
+		},
+		"TailVolume": func(m *Mover) error {
+			return m.TailVolume(context.Background(), 7, bad, dstAddr, 0, time.Second)
+		},
+		"CopyAndMountEcShards": func(m *Mover) error {
+			return m.CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, bad, dstAddr, 0, 0, nil)
+		},
+		"MoveEcShards": func(m *Mover) error {
+			move := ecMove(3)
+			move.Source = bad
+			return m.MoveEcShards(context.Background(), move, EcMoveOptions{})
+		},
+	}
+	for name, op := range ops {
+		cluster := newFakeCluster()
+		err := op(cluster.mover())
+		if err == nil || !strings.Contains(err.Error(), "invalid volume server address") {
+			t.Errorf("%s: expected invalid-address error, got: %v", name, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Errorf("%s: RPCs issued with a malformed embedded source: %v", name, cluster.callList())
+		}
+	}
+}
+
+func TestNewMoverRejectsMalformedAddressWithoutDialing(t *testing.T) {
+	// The production dial path must fail the move, not the process, when a
+	// caller passes an unvalidated malformed address.
+	mover := NewMover(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	err := mover.LiveMoveVolume(context.Background(), 7, "node:abc", "node:8080", VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "invalid volume server address") {
+		t.Fatalf("expected invalid-address error, got: %v", err)
+	}
+}
+
+func assertCalls(t *testing.T, got, want []string) {
+	t.Helper()
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("RPC sequence mismatch:\n got:  %v\n want: %v", got, want)
+	}
+}
+
+func TestLiveMoveVolumeSequence(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.lastAppendAtNs = 12345
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{
+		DiskType:        "ssd",
+		IoBytePerSecond: 42,
+	})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume: %v", err)
+	}
+
+	// The source status read must come after the tail: a write in flight when
+	// the source was frozen can land after an earlier read, and verifying
+	// against that stale snapshot would miss a missed tail. The first target
+	// status read probes whether the target already held the volume, so a
+	// failed copy never cleans up someone else's replica.
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeStatus",
+		"src:8080 VolumeMarkReadonly",
+		"dst:8080 ReadVolumeFileStatus",
+		"dst:8080 VolumeCopy",
+		"dst:8080 VolumeTailReceiver",
+		"src:8080 ReadVolumeFileStatus",
+		"dst:8080 ReadVolumeFileStatus",
+		"src:8080 VolumeDelete",
+	})
+
+	copyReq := cluster.copyReqs[0]
+	if copyReq.DiskType != "ssd" || copyReq.IoBytePerSecond != 42 || copyReq.SourceDataNode != string(srcAddr) {
+		t.Errorf("copy request not propagated: %+v", copyReq)
+	}
+	deleteReq := cluster.deleteReqs[0]
+	if deleteReq.OnlyEmpty || !deleteReq.KeepRemoteData {
+		t.Errorf("source delete must keep remote data and not be only-empty: %+v", deleteReq)
+	}
+}
+
+func TestLiveMoveVolumeVerifyMismatchKeepsSource(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "behind the source") {
+		t.Fatalf("expected target-behind error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted despite verification failure: %v", calls)
+		}
+	}
+	// The incomplete target copy is removed so the restored source stays the
+	// only replica, then the source is made writable again.
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeAbortKeepsSourceReadonlyWhenTargetCleanupFails(t *testing.T) {
+	// Restoring the source while the stale target stays mounted risks
+	// divergent replicas; the source stays readonly until a re-run recovers.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+	cluster.errs["dst:8080 VolumeDelete"] = errors.New("target unreachable")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected verification failure")
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("error does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("source made writable with the stale target still mounted: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeAlreadyReadonlyStaysReadonly(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("copy failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected copy failure")
+	}
+
+	// The hard mark is issued even for a readonly-reporting volume: that
+	// status also covers low-disk and readonly-but-can-delete states, which
+	// still accept needle deletes.
+	marked := false
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkReadonly" {
+			marked = true
+		}
+		// A volume that was readonly before the move (e.g. full) must stay so.
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatal("made an already-readonly volume writable after a failed move")
+		}
+	}
+	if !marked {
+		t.Fatalf("readonly-reporting source not hard-frozen: %v", cluster.callList())
+	}
+}
+
+func TestLiveMoveVolumePreFrozenAbortCleansTarget(t *testing.T) {
+	// tiering.run freezes replicas itself and thaws them after a failed move,
+	// so even a move that did not do the freezing must remove its target copy
+	// on abort — a stale mounted target plus a thawed source is divergence.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected verification failure")
+	}
+
+	cleaned := false
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeDelete" {
+			cleaned = true
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("thawed a source this move did not freeze: %v", cluster.callList())
+		}
+	}
+	if !cleaned {
+		t.Fatalf("incomplete target copy not cleaned up: %v", cluster.callList())
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorCleansMountedTarget(t *testing.T) {
+	// The server can finish the copy and mount the target even when the client
+	// loses the stream; the abort probes the target and cleans up the copy.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("stream lost")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected copy failure")
+	}
+
+	calls := cluster.callList()
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorUnknownTargetKeepsSourceReadonly(t *testing.T) {
+	// The pre-copy probe failed at the transport level, so whether the target
+	// held a replica before the move is unknown; the failed copy may still
+	// have mounted a complete copy there (the server can finish after the
+	// client loses the stream). Deleting it risks a healthy pre-existing
+	// replica, and reopening the source beside it risks two writable replicas
+	// taking divergent writes — so the source stays readonly.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.Unavailable, "connection refused")
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("stream lost")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected kept-readonly failure, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted the target with its prior state unknown: %v", calls)
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("reopened the source beside a possibly-mounted copy: %v", calls)
+		}
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorLeavesPreexistingTarget(t *testing.T) {
+	// A target that held the volume before the move is someone else's replica;
+	// a failed copy must not delete it.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("copy failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected kept-readonly failure, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted a pre-existing target replica after a failed copy: %v", calls)
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("reopened the source with a copy of unprovable origin on the target: %v", calls)
+		}
+	}
+}
+
+func TestLiveMoveVolumeTailErrorAborts(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "tail volume") {
+		t.Fatalf("expected tail error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted despite tail failure: %v", calls)
+		}
+	}
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeTailErrorToleratedForReadonlySource(t *testing.T) {
+	// A volume that was already readonly before the move was frozen for the
+	// whole copy, so a failed tail has nothing to deliver.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{IdleTimeout: time.Millisecond})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume on readonly source with failed tail: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not complete with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeTailFailureAbortsWhenSourceStillChanging(t *testing.T) {
+	// A tolerated tail failure substitutes a drain barrier: the source status
+	// must hold still across the idle window, or the move aborts.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.statusSeq[string(srcAddr)] = []*volume_server_pb.ReadVolumeFileStatusResponse{
+		volumeStatus(1000, 100, 10),
+		volumeStatus(1024, 116, 11), // a straggler landed between the two reads
+	}
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{IdleTimeout: time.Millisecond})
+	if err == nil || !strings.Contains(err.Error(), "still changing") {
+		t.Fatalf("expected still-changing abort, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted while still changing: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeSourceDeleteFailureKeepsBothReadonly(t *testing.T) {
+	// Past verification the target is complete and may hold new writes, and
+	// the failed delete may or may not have reached the source. Deleting the
+	// target or reopening the source could fork the volume; keep everything
+	// readonly for a re-run.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["src:8080 VolumeDelete"] = errors.New("delete timed out")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected source delete failure")
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("error does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted the verified target copy after a failed source delete: %v", cluster.callList())
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("restored source writability next to a mounted verified copy: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeRefusesRecopyOverCompleteTarget(t *testing.T) {
+	// A readonly source next to a target already holding a copy that is not
+	// behind it is the signature of a previous move whose source delete
+	// failed; recopying would tear down the possibly-authoritative target.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "already has a copy") {
+		t.Fatalf("expected recopy refusal, got: %v", err)
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("refusal does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" || call == "dst:8080 VolumeDelete" {
+			t.Fatalf("touched the possibly-authoritative target: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeReadonlySourceUnknownTargetRefused(t *testing.T) {
+	// With a readonly source, an unreachable target probe must fail closed:
+	// the target could hold the authoritative copy of a previous move.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.Unavailable, "connection refused")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected fail-closed refusal, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" {
+			t.Fatalf("copied over a target in unknown state: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumePreFrozenSourceMissingRustTargetProceeds(t *testing.T) {
+	// The Rust volume server answers a missing volume with codes.NotFound
+	// (the Go server uses a plain error, code Unknown). Both are definitive
+	// absence: with a pre-frozen source (tiering freezes before moving), a
+	// NotFound misclassified as "unknown" would refuse the move with
+	// ErrSourceKeptReadonly — and tiering callers then deliberately skip
+	// thawing, stranding every replica readonly after an ordinary move.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.NotFound, "not found volume id 7")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume to an empty Rust target: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not complete with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeReadonlySourceExistingTargetRefused(t *testing.T) {
+	// No client-side observation can prove an existing copy is a stale
+	// remnant rather than the authoritative copy of an unfinished move —
+	// compaction shrinks the authoritative copy below the stale source, and
+	// any snapshot can be invalidated by a write right after it. Even a
+	// target that reads as behind is refused.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 90, 9)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected fail-closed refusal, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" || call == "dst:8080 VolumeDelete" {
+			t.Fatalf("touched a possibly-authoritative target: %v", cluster.callList())
+		}
+	}
+}
+
+func TestCopyVolumeLeavesReadonlySourceUntouched(t *testing.T) {
+	// The copy is non-destructive, so a readonly-reporting source (possibly
+	// only transiently, e.g. low disk) is not hard-marked.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+
+	_, err := cluster.mover().CopyVolume(context.Background(), 7, srcAddr, dstAddr, "", 0, true, nil)
+	if err != nil {
+		t.Fatalf("CopyVolume: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkReadonly" || call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("readonly source touched by a non-destructive copy: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeTargetAheadCommits(t *testing.T) {
+	// The target serves writes while the move tails; a target that got ahead
+	// holds those acknowledged writes, and the move must commit so they stay
+	// on the surviving copy.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1100, 110, 11)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume with target ahead: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not commit with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeRejectsSameServer(t *testing.T) {
+	// The second target is the same server written with an explicit grpc port;
+	// the guard must see through the representation difference.
+	for _, target := range []pb.ServerAddress{srcAddr, pb.ServerAddress("src:8080.18080")} {
+		cluster := newFakeCluster()
+
+		err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, target, VolumeMoveOptions{})
+		if err == nil || !strings.Contains(err.Error(), "its own server") {
+			t.Fatalf("target %q: expected same-server rejection, got: %v", target, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Fatalf("target %q: RPCs issued for a rejected move: %v", target, cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeReadonlyMarkFailureRestores(t *testing.T) {
+	// The marking RPC can fail after the server applied the mark (e.g. its
+	// master notification failed), so a mark error must still restore.
+	cluster := newFakeCluster()
+	cluster.errs["src:8080 VolumeMarkReadonly"] = errors.New("master notification failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "mark volume") {
+		t.Fatalf("expected mark-readonly error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeMarkWritable" {
+		t.Fatalf("source writability not restored after failed readonly mark: %v", calls)
+	}
+}
+
+func TestReplicateVolumePropagatesThroughput(t *testing.T) {
+	cluster := newFakeCluster()
+
+	if err := cluster.mover().ReplicateVolume(context.Background(), 7, srcAddr, dstAddr, "ssd", 55, nil); err != nil {
+		t.Fatalf("ReplicateVolume: %v", err)
+	}
+	copyReq := cluster.copyReqs[0]
+	if copyReq.IoBytePerSecond != 55 || copyReq.DiskType != "ssd" {
+		t.Errorf("replicate request not propagated: %+v", copyReq)
+	}
+}
+
+func TestCopyVolumeRestoreWritable(t *testing.T) {
+	for _, restoreWritable := range []bool{true, false} {
+		cluster := newFakeCluster()
+		_, err := cluster.mover().CopyVolume(context.Background(), 7, srcAddr, dstAddr, "", 0, restoreWritable, nil)
+		if err != nil {
+			t.Fatalf("CopyVolume(restoreWritable=%v): %v", restoreWritable, err)
+		}
+		restored := false
+		for _, call := range cluster.callList() {
+			if call == "src:8080 VolumeMarkWritable" {
+				restored = true
+			}
+		}
+		if restored != restoreWritable {
+			t.Errorf("restoreWritable=%v but writability restored=%v", restoreWritable, restored)
+		}
+	}
+}