snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/operation/volume_move/ec_move.go

@@ -0,0 +1,215 @@
+package volume_move
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// EcShardMove is one planned relocation of mounted shards to another server.
+type EcShardMove struct {
+	VolumeId   needle.VolumeId
+	Collection string
+	ShardIds   []erasure_coding.ShardId
+	Source     pb.ServerAddress
+	Target     pb.ServerAddress
+	// TargetDisk picks the destination disk (0 lets the server pick).
+	TargetDisk uint32
+}
+
+// EcMoveOptions control MoveEcShards.
+type EcMoveOptions struct {
+	// IoBytePerSecond limits the shard copy rate; 0 falls back to the volume
+	// server's maintenance rate.
+	IoBytePerSecond int64
+	// Writer receives human-readable progress lines (nil discards them).
+	Writer io.Writer
+	// Progress, when set, receives percent/stage callbacks as the move advances.
+	Progress func(percent float64, stage string)
+}
+
+// MoveEcShards relocates mounted shards: copy+mount on the target, verify the
+// target registered them, then unmount+delete on the source. The verification
+// gates the destructive half — a copy/mount RPC can return OK while the shard
+// is not loadable on the target, and deleting the source then would lose it;
+// on a mismatch the source is kept so the caller can retry.
+func (m *Mover) MoveEcShards(ctx context.Context, move EcShardMove, opts EcMoveOptions) error {
+	writer := opts.Writer
+	if writer == nil {
+		writer = io.Discard
+	}
+	progress := opts.Progress
+	if progress == nil {
+		progress = func(float64, string) {}
+	}
+
+	// A same-server "move" cannot be expressed with these RPCs: the source
+	// delete is server-wide, so it would erase the just-copied shard. Removing
+	// a duplicate shard in place is RemoveEcShards. SameServer, not ==:
+	// "node:8080" and "node:8080.18080" are one server.
+	if SameServer(move.Source, move.Target) {
+		return fmt.Errorf("refusing EC shard move of volume %d shard(s) %v onto its own server %s: the source delete is server-wide", move.VolumeId, move.ShardIds, move.Source)
+	}
+
+	progress(10, fmt.Sprintf("copying EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
+	if err := m.CopyAndMountEcShards(ctx, move.VolumeId, move.Collection, move.ShardIds, move.Source, move.Target, move.TargetDisk, opts.IoBytePerSecond, writer); err != nil {
+		return err
+	}
+
+	progress(40, fmt.Sprintf("verifying EC shard(s) %d.%v on %s", move.VolumeId, move.ShardIds, move.Target))
+	if err := m.VerifyEcShards(ctx, move.VolumeId, move.Target, move.ShardIds); err != nil {
+		return err
+	}
+
+	progress(50, fmt.Sprintf("unmounting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
+	fmt.Fprintf(writer, "unmount %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
+	if err := m.UnmountEcShards(ctx, move.VolumeId, move.Source, move.ShardIds); err != nil {
+		return fmt.Errorf("unmount %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
+	}
+
+	progress(75, fmt.Sprintf("deleting EC shard(s) %d.%v from %s", move.VolumeId, move.ShardIds, move.Source))
+	fmt.Fprintf(writer, "delete %d.%v from %s\n", move.VolumeId, move.ShardIds, move.Source)
+	if err := m.DeleteEcShards(ctx, move.VolumeId, move.Collection, move.Source, move.ShardIds); err != nil {
+		return fmt.Errorf("delete %d.%v from %s: %v", move.VolumeId, move.ShardIds, move.Source, err)
+	}
+
+	progress(100, fmt.Sprintf("moved EC shard(s) %d.%v from %s to %s", move.VolumeId, move.ShardIds, move.Source, move.Target))
+	return nil
+}
+
+// CopyAndMountEcShards has the target copy the shards (with their .ecx/.ecj/
+// .vif/.ecsum sidecars) from source and mount them. A same-address call skips
+// the copy and just mounts — ec.encode uses that to bring freshly generated
+// shards online in place.
+func (m *Mover) CopyAndMountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, source, target pb.ServerAddress, targetDisk uint32, ioBytePerSecond int64, writer io.Writer) error {
+	if writer == nil {
+		writer = io.Discard
+	}
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if !SameServer(target, source) {
+		if err := checkDialable(source); err != nil {
+			return err
+		}
+	}
+	return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
+		if !SameServer(target, source) {
+			fmt.Fprintf(writer, "copy %d.%v %s => %s\n", volumeId, shardIds, source, target)
+			_, copyErr := client.VolumeEcShardsCopy(ctx, &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:        uint32(volumeId),
+				Collection:      collection,
+				ShardIds:        erasure_coding.ShardIdsToUint32(shardIds),
+				CopyEcxFile:     true,
+				CopyEcjFile:     true,
+				CopyVifFile:     true,
+				CopyEcsumFile:   true, // propagate the bitrot sidecar with the shards (no-op if the source has none)
+				SourceDataNode:  string(source),
+				DiskId:          targetDisk,
+				IoBytePerSecond: ioBytePerSecond,
+			})
+			if copyErr != nil {
+				return fmt.Errorf("copy %d.%v %s => %s: %v", volumeId, shardIds, source, target, copyErr)
+			}
+		}
+		fmt.Fprintf(writer, "mount %d.%v on %s\n", volumeId, shardIds, target)
+		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		if mountErr != nil {
+			return fmt.Errorf("mount %d.%v on %s: %v", volumeId, shardIds, target, mountErr)
+		}
+		return nil
+	})
+}
+
+// VerifyEcShards confirms server has every shard in shardIds registered for
+// the volume.
+func (m *Mover) VerifyEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, err := client.VolumeEcShardsInfo(ctx, &volume_server_pb.VolumeEcShardsInfoRequest{
+			VolumeId: uint32(volumeId),
+		})
+		if err != nil {
+			return fmt.Errorf("verify EC shard(s) on %s for volume %d: %v", server, volumeId, err)
+		}
+		var bits, zeroSized erasure_coding.ShardBits
+		for _, s := range resp.EcShardInfos {
+			if s.VolumeId != uint32(volumeId) || s.ShardId >= erasure_coding.MaxShardCount {
+				continue
+			}
+			// A zero-sized shard is residue of a failed operation, not a
+			// shard; counting it as present would let a broken copy pass
+			// verification and the source be deleted behind it.
+			if s.Size <= 0 {
+				zeroSized = zeroSized.Set(erasure_coding.ShardId(s.ShardId))
+				continue
+			}
+			bits = bits.Set(erasure_coding.ShardId(s.ShardId))
+		}
+		for _, sid := range shardIds {
+			if bits.Has(sid) {
+				continue
+			}
+			if zeroSized.Has(sid) {
+				return fmt.Errorf("%s has a zero-sized EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
+			}
+			return fmt.Errorf("%s missing EC shard %d.%d after copy/mount; keeping source", server, volumeId, sid)
+		}
+		return nil
+	})
+}
+
+// MountEcShards mounts shards already on server.
+func (m *Mover) MountEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, mountErr := client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return mountErr
+	})
+}
+
+// UnmountEcShards unmounts shards on server.
+func (m *Mover) UnmountEcShards(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, unmountErr := client.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+			VolumeId: uint32(volumeId),
+			ShardIds: erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return unmountErr
+	})
+}
+
+// DeleteEcShards deletes shards on server. The delete is server-wide: it
+// removes the shards from every disk of the server.
+func (m *Mover) DeleteEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, deleteErr := client.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+			ShardIds:   erasure_coding.ShardIdsToUint32(shardIds),
+		})
+		return deleteErr
+	})
+}
+
+// RemoveEcShards unmounts then deletes shards in place — the dedup path for a
+// shard that already has a copy elsewhere.
+func (m *Mover) RemoveEcShards(ctx context.Context, volumeId needle.VolumeId, collection string, server pb.ServerAddress, shardIds []erasure_coding.ShardId) error {
+	if err := m.UnmountEcShards(ctx, volumeId, server, shardIds); err != nil {
+		return fmt.Errorf("unmount %d.%v from %s: %v", volumeId, shardIds, server, err)
+	}
+	if err := m.DeleteEcShards(ctx, volumeId, collection, server, shardIds); err != nil {
+		return fmt.Errorf("delete %d.%v from %s: %v", volumeId, shardIds, server, err)
+	}
+	return nil
+}

weed/operation/volume_move/ec_move_test.go

@@ -0,0 +1,137 @@
+package volume_move
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+func ecMove(shardIds ...erasure_coding.ShardId) EcShardMove {
+	return EcShardMove{
+		VolumeId:   7,
+		Collection: "c1",
+		ShardIds:   shardIds,
+		Source:     srcAddr,
+		Target:     dstAddr,
+		TargetDisk: 2,
+	}
+}
+
+func dstShards(shardIds ...uint32) []*volume_server_pb.EcShardInfo {
+	var infos []*volume_server_pb.EcShardInfo
+	for _, sid := range shardIds {
+		infos = append(infos, &volume_server_pb.EcShardInfo{VolumeId: 7, ShardId: sid, Size: 1024})
+	}
+	return infos
+}
+
+func TestMoveEcShardsSequence(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3, 4)
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{IoBytePerSecond: 77})
+	if err != nil {
+		t.Fatalf("MoveEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"dst:8080 VolumeEcShardsCopy",
+		"dst:8080 VolumeEcShardsMount",
+		"dst:8080 VolumeEcShardsInfo",
+		"src:8080 VolumeEcShardsUnmount",
+		"src:8080 VolumeEcShardsDelete",
+	})
+
+	copyReq := cluster.ecCopyReqs[0]
+	if !copyReq.CopyEcxFile || !copyReq.CopyEcjFile || !copyReq.CopyVifFile || !copyReq.CopyEcsumFile {
+		t.Errorf("shard sidecars not all copied: %+v", copyReq)
+	}
+	if copyReq.DiskId != 2 || copyReq.SourceDataNode != string(srcAddr) || copyReq.Collection != "c1" || copyReq.IoBytePerSecond != 77 {
+		t.Errorf("copy request not propagated: %+v", copyReq)
+	}
+}
+
+func TestMoveEcShardsVerifyFailureKeepsSource(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3) // shard 4 didn't register
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "missing EC shard 7.4") {
+		t.Fatalf("expected missing-shard error, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeEcShardsUnmount" || call == "src:8080 VolumeEcShardsDelete" {
+			t.Fatalf("source touched despite verification failure: %v", cluster.callList())
+		}
+	}
+}
+
+func TestMoveEcShardsZeroSizedDestinationKeepsSource(t *testing.T) {
+	// A zero-sized shard on the destination is residue of a failed operation
+	// (seaweedfs issue 10730); verification must not count it as a delivered
+	// shard, or the source is deleted behind a broken copy.
+	cluster := newFakeCluster()
+	cluster.ecShards[string(dstAddr)] = dstShards(3, 4)
+	cluster.ecShards[string(dstAddr)][1].Size = 0 // shard 4 landed as an empty file
+
+	err := cluster.mover().MoveEcShards(context.Background(), ecMove(3, 4), EcMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "zero-sized EC shard 7.4") {
+		t.Fatalf("expected zero-sized-shard rejection, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeEcShardsUnmount" || call == "src:8080 VolumeEcShardsDelete" {
+			t.Fatalf("source touched despite zero-sized destination shard: %v", cluster.callList())
+		}
+	}
+}
+
+func TestMoveEcShardsRejectsSameServer(t *testing.T) {
+	// The second target is the same server written with an explicit grpc port;
+	// the guard must see through the representation difference.
+	for _, target := range []pb.ServerAddress{srcAddr, pb.ServerAddress("src:8080.18080")} {
+		cluster := newFakeCluster()
+		move := ecMove(3)
+		move.Target = target
+
+		err := cluster.mover().MoveEcShards(context.Background(), move, EcMoveOptions{})
+		if err == nil || !strings.Contains(err.Error(), "its own server") {
+			t.Fatalf("target %q: expected same-server rejection, got: %v", target, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Fatalf("target %q: RPCs issued for a rejected move: %v", target, cluster.callList())
+		}
+	}
+}
+
+func TestRemoveEcShards(t *testing.T) {
+	cluster := newFakeCluster()
+
+	err := cluster.mover().RemoveEcShards(context.Background(), 7, "c1", srcAddr, []erasure_coding.ShardId{3})
+	if err != nil {
+		t.Fatalf("RemoveEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeEcShardsUnmount",
+		"src:8080 VolumeEcShardsDelete",
+	})
+}
+
+func TestCopyAndMountEcShardsSameAddressMountsOnly(t *testing.T) {
+	cluster := newFakeCluster()
+
+	err := cluster.mover().CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, srcAddr, srcAddr, 0, 0, nil)
+	if err != nil {
+		t.Fatalf("CopyAndMountEcShards: %v", err)
+	}
+
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeEcShardsMount",
+	})
+}

weed/operation/volume_move/fake_client_test.go

@@ -0,0 +1,230 @@
+package volume_move
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// fakeCluster fakes the volume servers a move touches. It records every RPC as
+// "<server> <method>" so tests can assert the exact sequence, and lets tests
+// inject per-RPC errors and per-server state.
+type fakeCluster struct {
+	mu    sync.Mutex
+	calls []string
+
+	// errs fails an RPC, keyed "<server> <method>".
+	errs map[string]error
+	// readonly is each server's VolumeStatus.IsReadOnly answer.
+	readonly map[string]bool
+	// status is each server's ReadVolumeFileStatus answer.
+	status map[string]*volume_server_pb.ReadVolumeFileStatusResponse
+	// statusFailures fails a server's next N ReadVolumeFileStatus calls, e.g.
+	// to model a target that has no volume before the copy but one after. The
+	// failures use statusFailureErr, defaulting to a server-side (gRPC code
+	// Unknown) volume-not-found error.
+	statusFailures   map[string]int
+	statusFailureErr error
+	// statusSeq overrides a server's next ReadVolumeFileStatus answers in
+	// order, e.g. to model a source that changes between two reads.
+	statusSeq map[string][]*volume_server_pb.ReadVolumeFileStatusResponse
+	// ecShards is each server's VolumeEcShardsInfo answer.
+	ecShards map[string][]*volume_server_pb.EcShardInfo
+
+	lastAppendAtNs uint64
+
+	copyReqs   []*volume_server_pb.VolumeCopyRequest
+	deleteReqs []*volume_server_pb.VolumeDeleteRequest
+	ecCopyReqs []*volume_server_pb.VolumeEcShardsCopyRequest
+}
+
+func newFakeCluster() *fakeCluster {
+	return &fakeCluster{
+		errs:           make(map[string]error),
+		readonly:       make(map[string]bool),
+		status:         make(map[string]*volume_server_pb.ReadVolumeFileStatusResponse),
+		statusFailures: make(map[string]int),
+		statusSeq:      make(map[string][]*volume_server_pb.ReadVolumeFileStatusResponse),
+		ecShards:       make(map[string][]*volume_server_pb.EcShardInfo),
+	}
+}
+
+func (c *fakeCluster) mover() *Mover {
+	return NewMoverWithClientFunc(func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error {
+		return fn(&fakeClient{cluster: c, addr: string(addr)})
+	})
+}
+
+func (c *fakeCluster) record(addr, method string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls = append(c.calls, addr+" "+method)
+	return c.errs[addr+" "+method]
+}
+
+func (c *fakeCluster) callList() []string {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return append([]string(nil), c.calls...)
+}
+
+// fakeClient answers as the volume server at addr. Unused VolumeServerClient
+// methods panic via the embedded nil interface.
+type fakeClient struct {
+	volume_server_pb.VolumeServerClient
+	cluster *fakeCluster
+	addr    string
+}
+
+func (f *fakeClient) VolumeStatus(ctx context.Context, req *volume_server_pb.VolumeStatusRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeStatusResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeStatus"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeStatusResponse{IsReadOnly: f.cluster.readonly[f.addr]}, nil
+}
+
+func (f *fakeClient) VolumeMarkReadonly(ctx context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeMarkReadonly"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
+}
+
+func (f *fakeClient) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeMarkWritableResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeMarkWritable"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeMarkWritableResponse{}, nil
+}
+
+func (f *fakeClient) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
+	if err := f.cluster.record(f.addr, "ReadVolumeFileStatus"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	failures := f.cluster.statusFailures[f.addr]
+	if failures > 0 {
+		f.cluster.statusFailures[f.addr] = failures - 1
+	}
+	failureErr := f.cluster.statusFailureErr
+	var seqResp *volume_server_pb.ReadVolumeFileStatusResponse
+	if failures == 0 {
+		if seq := f.cluster.statusSeq[f.addr]; len(seq) > 0 {
+			seqResp = seq[0]
+			f.cluster.statusSeq[f.addr] = seq[1:]
+		}
+	}
+	resp := f.cluster.status[f.addr]
+	f.cluster.mu.Unlock()
+	if failures > 0 {
+		if failureErr != nil {
+			return nil, failureErr
+		}
+		return nil, status.Error(codes.Unknown, fmt.Sprintf("not found volume id %d", req.VolumeId))
+	}
+	if seqResp != nil {
+		return seqResp, nil
+	}
+	if resp == nil {
+		return nil, status.Error(codes.Unknown, fmt.Sprintf("not found volume id %d", req.VolumeId))
+	}
+	return resp, nil
+}
+
+func (f *fakeClient) VolumeCopy(ctx context.Context, req *volume_server_pb.VolumeCopyRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[volume_server_pb.VolumeCopyResponse], error) {
+	if err := f.cluster.record(f.addr, "VolumeCopy"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.copyReqs = append(f.cluster.copyReqs, req)
+	lastAppendAtNs := f.cluster.lastAppendAtNs
+	f.cluster.mu.Unlock()
+	return &fakeCopyStream{resps: []*volume_server_pb.VolumeCopyResponse{
+		{ProcessedBytes: 1024},
+		{LastAppendAtNs: lastAppendAtNs},
+	}}, nil
+}
+
+func (f *fakeClient) VolumeTailReceiver(ctx context.Context, req *volume_server_pb.VolumeTailReceiverRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeTailReceiverResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeTailReceiver"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeTailReceiverResponse{}, nil
+}
+
+func (f *fakeClient) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeDeleteResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeDelete"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.deleteReqs = append(f.cluster.deleteReqs, req)
+	f.cluster.mu.Unlock()
+	return &volume_server_pb.VolumeDeleteResponse{}, nil
+}
+
+func (f *fakeClient) VolumeConfigure(ctx context.Context, req *volume_server_pb.VolumeConfigureRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeConfigureResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeConfigure"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeConfigureResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsCopy"); err != nil {
+		return nil, err
+	}
+	f.cluster.mu.Lock()
+	f.cluster.ecCopyReqs = append(f.cluster.ecCopyReqs, req)
+	f.cluster.mu.Unlock()
+	return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsMount(ctx context.Context, req *volume_server_pb.VolumeEcShardsMountRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsMountResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsMount"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsUnmount"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsDelete"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
+}
+
+func (f *fakeClient) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest, opts ...grpc.CallOption) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
+	if err := f.cluster.record(f.addr, "VolumeEcShardsInfo"); err != nil {
+		return nil, err
+	}
+	return &volume_server_pb.VolumeEcShardsInfoResponse{EcShardInfos: f.cluster.ecShards[f.addr]}, nil
+}
+
+// fakeCopyStream feeds the canned VolumeCopy responses, then io.EOF.
+type fakeCopyStream struct {
+	grpc.ClientStream
+	resps []*volume_server_pb.VolumeCopyResponse
+}
+
+func (s *fakeCopyStream) Recv() (*volume_server_pb.VolumeCopyResponse, error) {
+	if len(s.resps) == 0 {
+		return nil, io.EOF
+	}
+	resp := s.resps[0]
+	s.resps = s.resps[1:]
+	return resp, nil
+}

weed/operation/volume_move/mover.go

@@ -0,0 +1,113 @@
+// Package volume_move implements the volume and EC shard move sequences shared
+// by the interactive shell commands (volume.move, volume.balance, ec.balance,
+// volume.tier.move, ...) and the maintenance workers (balance, ec_balance,
+// volume_tiering). The RPCs go through an injectable ClientFunc so every
+// sequence can be unit tested against a fake volume server client.
+package volume_move
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+// ClientFunc runs fn with a client for the volume server at addr. It matches
+// operation.WithVolumeServerClient, which production movers use to dial real
+// servers; tests substitute a fake client.
+type ClientFunc func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error
+
+// Mover executes volume and EC shard moves against volume servers.
+type Mover struct {
+	withClient ClientFunc
+}
+
+func NewMover(grpcDialOption grpc.DialOption) *Mover {
+	return &Mover{withClient: func(streamingMode bool, addr pb.ServerAddress, fn func(client volume_server_pb.VolumeServerClient) error) error {
+		// Validated here, the single point every mover RPC dials through:
+		// handing a malformed address (an unvalidated -source/-target flag)
+		// to the dialer would abort the whole process instead of failing the
+		// move.
+		if err := checkDialable(addr); err != nil {
+			return err
+		}
+		return operation.WithVolumeServerClient(streamingMode, addr, grpcDialOption, fn)
+	}}
+}
+
+// checkDialable rejects an address whose grpc normalization would abort the
+// process: for the "host:port" form, ServerAddress.ToGrpcAddress falls back
+// to a parser that calls glog.Fatalf when the port is not numeric. Everything
+// else either normalizes cleanly or fails at dial time as an ordinary error.
+// It also guards the source addresses embedded in copy/tail requests: the
+// receiving volume server dials those through the same fatal parser, so an
+// unchecked malformed source would terminate the destination server.
+func checkDialable(addr pb.ServerAddress) error {
+	s := string(addr)
+	colon := strings.LastIndex(s, ":")
+	if colon < 0 || colon+1 >= len(s) {
+		return nil // no port part; handed to the dialer untouched
+	}
+	ports := s[colon+1:]
+	if dot := strings.LastIndex(ports, "."); dot >= 0 {
+		// "port.grpcPort": different dial paths canonicalize a half-malformed
+		// form differently (the method splices the grpc part in unparsed; the
+		// string helper falls back to the http part + 10000), so a bad
+		// component could dial an unintended server or reach the fatal
+		// parser. Require both components numeric.
+		if _, err := strconv.ParseUint(ports[:dot], 10, 64); err != nil {
+			return fmt.Errorf("invalid volume server address %q: port %q is not a number", s, ports[:dot])
+		}
+		if _, err := strconv.ParseUint(ports[dot+1:], 10, 64); err != nil {
+			return fmt.Errorf("invalid volume server address %q: grpc port %q is not a number", s, ports[dot+1:])
+		}
+		return nil
+	}
+	if _, err := strconv.ParseUint(ports, 10, 64); err != nil {
+		return fmt.Errorf("invalid volume server address %q: port %q is not a number", s, ports)
+	}
+	return nil
+}
+
+// NewMoverWithClientFunc builds a Mover on a custom transport.
+func NewMoverWithClientFunc(withClient ClientFunc) *Mover {
+	return &Mover{withClient: withClient}
+}
+
+// SameServer reports whether two addresses name the same volume server. The
+// gRPC endpoint identifies the server process — each has exactly one — so
+// "node:8080" and "node:8080.18080" compare equal, while two servers sharing
+// a degenerate HTTP address (e.g. port 0 in test harnesses) stay distinct.
+// The normalization is non-fatal, unlike ServerAddress.ToGrpcAddress, whose
+// parser exits the process on a malformed port; anything unparsable compares
+// as its literal self.
+func SameServer(a, b pb.ServerAddress) bool {
+	return grpcEndpoint(string(a)) == grpcEndpoint(string(b))
+}
+
+// grpcEndpoint mirrors ServerAddress.ToGrpcAddress ("host:port" gets the
+// +10000 grpc port; "host:port.grpcPort" names it explicitly) but returns a
+// malformed address unchanged instead of exiting.
+func grpcEndpoint(addr string) string {
+	colon := strings.LastIndex(addr, ":")
+	if colon < 0 || colon+1 >= len(addr) {
+		return addr
+	}
+	host, ports := addr[:colon], addr[colon+1:]
+	if dot := strings.LastIndex(ports, "."); dot >= 0 {
+		if grpcPort, err := strconv.Atoi(ports[dot+1:]); err == nil {
+			return util.JoinHostPort(host, grpcPort)
+		}
+		return addr
+	}
+	httpPort, err := strconv.Atoi(ports)
+	if err != nil {
+		return addr
+	}
+	return util.JoinHostPort(host, httpPort+10000)
+}

weed/operation/volume_move/volume_move.go

@@ -0,0 +1,503 @@
+package volume_move
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ErrSourceKeptReadonly marks a move failure whose recovery deliberately keeps
+// the source volume readonly — the target copy may hold data the source lacks
+// (an ambiguous source delete, or an undeletable stale copy). Callers that
+// froze the source themselves must not thaw it on this error.
+var ErrSourceKeptReadonly = errors.New("the source volume is deliberately kept readonly")
+
+// VolumeMoveOptions control LiveMoveVolume.
+type VolumeMoveOptions struct {
+	// DiskType changes the volume's disk type on the target ("" keeps it).
+	DiskType string
+	// IoBytePerSecond throttles the copy (0 = unlimited).
+	IoBytePerSecond int64
+	// IdleTimeout is how long the tail phase waits for in-flight requests to
+	// drain (default 5s).
+	IdleTimeout time.Duration
+	// Writer receives human-readable progress lines (nil discards them).
+	Writer io.Writer
+	// Progress, when set, receives percent/stage callbacks as the move advances.
+	Progress func(percent float64, stage string)
+}
+
+func (o *VolumeMoveOptions) fillDefaults() {
+	if o.IdleTimeout <= 0 {
+		o.IdleTimeout = 5 * time.Second
+	}
+	if o.Writer == nil {
+		o.Writer = io.Discard
+	}
+	if o.Progress == nil {
+		o.Progress = func(float64, string) {}
+	}
+}
+
+// LiveMoveVolume moves one volume between volume servers while it keeps serving
+// reads: freeze the source (readonly), copy, tail to drain in-flight requests,
+// verify the target matches the source, then delete the source. A failure
+// before the source delete restores the source's writability if this move is
+// what froze it.
+func (m *Mover) LiveMoveVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, opts VolumeMoveOptions) (err error) {
+	opts.fillDefaults()
+
+	// VolumeCopy tears down any existing copy on the target first, so a
+	// same-server "move" would delete the volume and then fail to read it.
+	// SameServer, not ==: "node:8080" and "node:8080.18080" are one server.
+	if SameServer(source, target) {
+		return fmt.Errorf("refusing to move volume %d onto its own server %s", volumeId, source)
+	}
+
+	opts.Progress(10, fmt.Sprintf("marking volume %d readonly on %s", volumeId, source))
+	sourceWasWritable, err := m.ensureVolumeReadonly(ctx, volumeId, source, true)
+	// The source stays authoritative until its delete succeeds; on any earlier
+	// failure undo the freeze this move added and clean up the target copy.
+	// Installed before the error check: the marking RPC itself can fail after
+	// the server applied the mark (e.g. its master notification failed).
+	var copyStarted, copyCompleted, sourceDeleteStarted bool
+	var targetHadVolume, targetStateKnown bool
+	defer func() {
+		if err != nil {
+			if sourceDeleteStarted {
+				// Past verification: the target is complete and may already hold
+				// new writes, and the delete may or may not have reached the
+				// source. Deleting the target or reopening the source could fork
+				// the volume; keep the source readonly.
+				fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: the source delete failed with the verified copy on %s mounted\n", volumeId, source, target)
+				glog.Warningf("volume %d is left readonly on %s: the source delete failed with the verified copy on %s mounted", volumeId, source, target)
+				err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+				return
+			}
+			// The cleanup runs on its own deadline so an abort via cancelled
+			// context still cleans up.
+			cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+			defer cleanupCancel()
+			cleanupTarget := copyCompleted
+			if copyStarted && !copyCompleted {
+				// The server can finish the copy and mount the target even when
+				// the client loses the stream, so probe rather than assume.
+				exists, known := m.probeVolume(cleanupCtx, volumeId, target)
+				switch {
+				case known && !exists:
+					// nothing mounted on the target; just undo the freeze below
+				case known && exists && targetStateKnown && !targetHadVolume:
+					// the failed copy created it; remove it so the source stays
+					// the only replica
+					cleanupTarget = true
+				default:
+					// A copy may sit mounted on the target and its provenance
+					// cannot be proven: the prior state is unknown, the target
+					// held a replica before the move, or the probe failed.
+					// Deleting it risks someone else's replica; reopening the
+					// source beside it risks two writable replicas taking
+					// divergent writes. Keep the source readonly.
+					fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: a copy may exist on %s but its origin cannot be determined; delete one side explicitly, then re-run the move\n", volumeId, source, target)
+					glog.Warningf("volume %d is left readonly on %s: a copy may exist on %s but its origin cannot be determined; delete one side explicitly, then re-run the move", volumeId, source, target)
+					err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+					return
+				}
+			}
+			if cleanupTarget {
+				// The target copy may be missing tailed entries; remove it so
+				// the source stays the only replica.
+				if dErr := m.DeleteVolume(cleanupCtx, volumeId, target, false, true); dErr != nil {
+					// Restoring the source while the stale target stays mounted
+					// risks divergent replicas; keep the source readonly. A
+					// re-run refuses while the copy exists, so name the fix.
+					fmt.Fprintf(opts.Writer, "volume %d is left readonly on %s: failed to delete the incomplete copy on %s: %v; delete that copy, then re-run the move\n", volumeId, source, target, dErr)
+					glog.Warningf("volume %d is left readonly on %s: failed to delete the incomplete copy on %s: %v; delete that copy, then re-run the move", volumeId, source, target, dErr)
+					err = fmt.Errorf("%w: %w", ErrSourceKeptReadonly, err)
+					return
+				}
+			}
+			if sourceWasWritable {
+				m.restoreVolumeWritable(volumeId, source)
+			} else {
+				// The hard freeze added on top of a readonly-reporting status is
+				// kept: it is indistinguishable from an operator's mark. It is
+				// not persisted, so a volume server restart clears it.
+				fmt.Fprintf(opts.Writer, "volume %d on %s keeps the readonly mark the move added; volume.mark -writable clears it if the prior readonly state was transient\n", volumeId, source)
+			}
+		}
+	}()
+	if err != nil {
+		return fmt.Errorf("mark volume %d readonly on %s: %v", volumeId, source, err)
+	}
+
+	targetHadVolume, targetStateKnown = m.probeVolume(ctx, volumeId, target)
+	if !sourceWasWritable {
+		// A readonly source next to an existing copy is the signature of a
+		// previous move whose source delete failed — that copy may be the
+		// authoritative one, serving writes the source never saw, and the
+		// copy below would tear it down. No client-side observation proves
+		// otherwise: compaction revision is shared by ordinary replicas,
+		// aggregate sizes shrink under compaction, and any snapshot can be
+		// invalidated by a write right after it. Fail closed on any existing
+		// copy or unknown state; the operator deletes one side explicitly.
+		if !targetStateKnown {
+			err = fmt.Errorf("%w: cannot determine whether %s already holds a copy of volume %d; refusing to overwrite it while the source is readonly", ErrSourceKeptReadonly, target, volumeId)
+			return err
+		}
+		if targetHadVolume {
+			err = fmt.Errorf("%w: volume %d already has a copy on %s while the source on %s is readonly — likely a previous move that did not finish; delete the source to keep that copy, or delete the copy before re-running the move", ErrSourceKeptReadonly, volumeId, target, source)
+			return err
+		}
+	}
+
+	opts.Progress(20, fmt.Sprintf("copying volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "copying volume %d from %s to %s\n", volumeId, source, target)
+	copyStarted = true
+	lastAppendAtNs, err := m.copyVolumeData(ctx, volumeId, source, target, opts.DiskType, opts.IoBytePerSecond, opts.Writer)
+	if err != nil {
+		return fmt.Errorf("copy volume %d from %s to %s: %v", volumeId, source, target, err)
+	}
+	copyCompleted = true
+
+	opts.Progress(70, fmt.Sprintf("tailing volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "tailing volume %d from %s to %s\n", volumeId, source, target)
+	tailFailed := false
+	if tailErr := m.TailVolume(ctx, volumeId, source, target, lastAppendAtNs, opts.IdleTimeout); tailErr != nil {
+		// A tail failure is tolerable only when the volume was already readonly
+		// before this move began: frozen for the whole copy, it should have no
+		// in-flight writes for the tail to drain — and the stability check
+		// below still proves it. A volume this move froze can have stragglers
+		// admitted just before the freeze that only the tail delivers, so
+		// losing the tail there must abort the move.
+		if sourceWasWritable {
+			return fmt.Errorf("tail volume %d from %s to %s: %v", volumeId, source, target, tailErr)
+		}
+		tailFailed = true
+		fmt.Fprintf(opts.Writer, "tail volume %d from %s to %s: %v\n", volumeId, source, target, tailErr)
+		glog.Warningf("tail volume %d from %s to %s: %v", volumeId, source, target, tailErr)
+	}
+
+	// Verify before the point of no return: the source is deleted only when the
+	// target holds at least everything the source does. The source status is
+	// read here, after the tail — a write in flight when the source was frozen
+	// can still land after an earlier read, and a stale snapshot would wave
+	// through an incomplete target.
+	opts.Progress(85, fmt.Sprintf("verifying volume %d on %s", volumeId, target))
+	sourceStatus, err := m.ReadVolumeFileStatus(ctx, volumeId, source)
+	if err != nil {
+		return fmt.Errorf("read volume %d status on %s: %v", volumeId, source, err)
+	}
+	if tailFailed {
+		// A successful tail proves the source held still for the idle window; a
+		// tolerated tail failure proved nothing, so substitute the same drain
+		// barrier here: the source status must not change across the window.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(opts.IdleTimeout):
+		}
+		secondStatus, statusErr := m.ReadVolumeFileStatus(ctx, volumeId, source)
+		if statusErr != nil {
+			return fmt.Errorf("re-read volume %d status on %s: %v", volumeId, source, statusErr)
+		}
+		if secondStatus.DatFileSize != sourceStatus.DatFileSize || secondStatus.IdxFileSize != sourceStatus.IdxFileSize || secondStatus.FileCount != sourceStatus.FileCount {
+			return fmt.Errorf("volume %d on %s is still changing after a failed tail; aborting the move", volumeId, source)
+		}
+		sourceStatus = secondStatus
+	}
+	targetStatus, err := m.ReadVolumeFileStatus(ctx, volumeId, target)
+	if err != nil {
+		return fmt.Errorf("verify volume %d on target %s before deleting source: %v", volumeId, target, err)
+	}
+	if err = verifyTargetNotBehind(volumeId, sourceStatus, targetStatus); err != nil {
+		return err
+	}
+	if targetStatus.DatFileSize > sourceStatus.DatFileSize || targetStatus.IdxFileSize > sourceStatus.IdxFileSize || targetStatus.FileCount > sourceStatus.FileCount {
+		// The target announced itself writable when the copy mounted it, so
+		// clients may have written to it during the tail. Those writes live on
+		// the surviving copy; committing keeps them.
+		fmt.Fprintf(opts.Writer, "volume %d on %s has writes beyond the source; they stay with the moved volume\n", volumeId, target)
+	}
+
+	opts.Progress(90, fmt.Sprintf("deleting volume %d from %s", volumeId, source))
+	fmt.Fprintf(opts.Writer, "deleting volume %d from %s\n", volumeId, source)
+	sourceDeleteStarted = true
+	if err = m.DeleteVolume(ctx, volumeId, source, false, true); err != nil {
+		return fmt.Errorf("delete volume %d from %s: %v", volumeId, source, err)
+	}
+
+	opts.Progress(100, fmt.Sprintf("moved volume %d from %s to %s", volumeId, source, target))
+	fmt.Fprintf(opts.Writer, "moved volume %d from %s to %s\n", volumeId, source, target)
+	return nil
+}
+
+// verifyTargetNotBehind fails when the target holds less than the (frozen)
+// source — the copy or tail missed data and deleting the source would lose it.
+// A target that is ahead is not an error: the target serves writes during the
+// tail, and those belong to the copy that survives the move.
+func verifyTargetNotBehind(volumeId needle.VolumeId, source, target *volume_server_pb.ReadVolumeFileStatusResponse) error {
+	if target.DatFileSize < source.DatFileSize {
+		return fmt.Errorf("volume %d target is behind the source: .dat %d < %d bytes", volumeId, target.DatFileSize, source.DatFileSize)
+	}
+	if target.IdxFileSize < source.IdxFileSize {
+		return fmt.Errorf("volume %d target is behind the source: .idx %d < %d bytes", volumeId, target.IdxFileSize, source.IdxFileSize)
+	}
+	if target.FileCount < source.FileCount {
+		return fmt.Errorf("volume %d target is behind the source: %d < %d files", volumeId, target.FileCount, source.FileCount)
+	}
+	return nil
+}
+
+// CopyVolume freezes the volume on source, copies it to target, and reports the
+// stamp of the last entry copied. restoreWritable also restores the source's
+// writability on success, for copies that leave the source serving; a failed
+// copy always undoes the freeze this call added.
+func (m *Mover) CopyVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, restoreWritable bool, writer io.Writer) (lastAppendAtNs uint64, err error) {
+	if writer == nil {
+		writer = io.Discard
+	}
+	if SameServer(source, target) {
+		return 0, fmt.Errorf("refusing to copy volume %d onto its own server %s", volumeId, source)
+	}
+	// The copy is non-destructive, so a readonly-reporting source is taken as
+	// is (force=false): missed concurrent deletes only make the new replica
+	// trivially stale, and hard-marking here could pin a transiently readonly
+	// (e.g. low-disk) source readonly with nothing to gate on it.
+	sourceWasWritable, err := m.ensureVolumeReadonly(ctx, volumeId, source, false)
+	defer func() {
+		if sourceWasWritable && (err != nil || restoreWritable) {
+			m.restoreVolumeWritable(volumeId, source)
+		}
+	}()
+	if err != nil {
+		return 0, err
+	}
+	return m.copyVolumeData(ctx, volumeId, source, target, diskType, ioBytePerSecond, writer)
+}
+
+// ensureVolumeReadonly freezes the volume on server and reports whether it was
+// writable beforehand — only then may a failure path undo the freeze, since a
+// volume that was already readonly (e.g. full, or operator-set) must stay so.
+// With force, the mark is issued even when the status already reports
+// readonly: that answer also covers transient low-disk state and the
+// readonly-but-can-delete flag, and neither blocks needle deletes, so only the
+// hard mark makes the volume immutable — required before deleting the source
+// of a move. Without force, a readonly-reporting volume is left untouched.
+func (m *Mover) ensureVolumeReadonly(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, force bool) (wasWritable bool, err error) {
+	err = m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, statusErr := client.VolumeStatus(ctx, &volume_server_pb.VolumeStatusRequest{
+			VolumeId: uint32(volumeId),
+		})
+		if statusErr != nil {
+			return statusErr
+		}
+		wasWritable = !resp.IsReadOnly
+		if !wasWritable && !force {
+			return nil
+		}
+		_, readonlyErr := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+			VolumeId: uint32(volumeId),
+			Persist:  false,
+		})
+		return readonlyErr
+	})
+	return
+}
+
+// probeVolume reports whether server currently has the volume, and whether the
+// answer is trustworthy. The Go server answers a missing volume with a plain
+// error (gRPC code Unknown — the only way this RPC fails there); the Rust
+// server answers with codes.NotFound. Both mean the server responded and the
+// volume is absent. A transport-level failure means the state is unknown, and
+// callers deciding to delete must treat unknown as hands-off.
+func (m *Mover) probeVolume(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress) (exists bool, known bool) {
+	_, err := m.ReadVolumeFileStatus(ctx, volumeId, server)
+	if err == nil {
+		return true, true
+	}
+	if s, ok := status.FromError(err); ok && (s.Code() == codes.Unknown || s.Code() == codes.NotFound) {
+		return false, true
+	}
+	return false, false
+}
+
+// restoreVolumeWritable undoes a readonly mark after a failed move. It runs on
+// its own deadline so an abort via cancelled context still restores the source,
+// and only logs a failure — the caller is already returning the move error.
+func (m *Mover) restoreVolumeWritable(volumeId needle.VolumeId, server pb.ServerAddress) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if err := m.MarkVolumeWritable(ctx, volumeId, server, true, false); err != nil {
+		glog.Warningf("restore volume %d writable on %s: %v", volumeId, server, err)
+	}
+}
+
+// copyVolumeData streams a VolumeCopy on the target, which pulls the volume
+// from source and mounts it, and returns the stamp of the last entry copied
+// for the tail phase.
+func (m *Mover) copyVolumeData(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, writer io.Writer) (lastAppendAtNs uint64, err error) {
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err = checkDialable(source); err != nil {
+		return 0, err
+	}
+	err = m.withClient(true, target, func(client volume_server_pb.VolumeServerClient) error {
+		stream, replicateErr := client.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+			VolumeId:        uint32(volumeId),
+			SourceDataNode:  string(source),
+			DiskType:        diskType,
+			IoBytePerSecond: ioBytePerSecond,
+		})
+		if replicateErr != nil {
+			return replicateErr
+		}
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				}
+				return recvErr
+			}
+			if resp.LastAppendAtNs != 0 {
+				lastAppendAtNs = resp.LastAppendAtNs
+			} else {
+				fmt.Fprintf(writer, "%s => %s volume %d processed %s\n", source, target, volumeId, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
+			}
+		}
+		return nil
+	})
+	return
+}
+
+// TailVolume has the target follow the source's appends since sinceNs until the
+// source stays idle for idleTimeout, draining requests in flight when the move
+// froze the source.
+func (m *Mover) TailVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, sinceNs uint64, idleTimeout time.Duration) error {
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err := checkDialable(source); err != nil {
+		return err
+	}
+	return m.withClient(true, target, func(client volume_server_pb.VolumeServerClient) error {
+		_, replicateErr := client.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
+			VolumeId:           uint32(volumeId),
+			SinceNs:            sinceNs,
+			IdleTimeoutSeconds: uint32(idleTimeout.Seconds()),
+			SourceVolumeServer: string(source),
+		})
+		return replicateErr
+	})
+}
+
+// ReadVolumeFileStatus reads the volume's file sizes and needle count on server.
+func (m *Mover) ReadVolumeFileStatus(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress) (resp *volume_server_pb.ReadVolumeFileStatusResponse, err error) {
+	err = m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		var statusErr error
+		resp, statusErr = client.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{
+			VolumeId: uint32(volumeId),
+		})
+		return statusErr
+	})
+	return
+}
+
+// DeleteVolume removes the volume from server. When keepRemoteData is true, the
+// cloud-tier object backing the volume is left intact — used on the source side
+// of a move where another server is taking over the same .vif.
+func (m *Mover) DeleteVolume(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		_, deleteErr := client.VolumeDelete(ctx, &volume_server_pb.VolumeDeleteRequest{
+			VolumeId:       uint32(volumeId),
+			OnlyEmpty:      onlyEmpty,
+			KeepRemoteData: keepRemoteData,
+		})
+		return deleteErr
+	})
+}
+
+// MarkVolumeWritable marks the volume writable (or readonly when writable is
+// false, persisted per persist) on server.
+func (m *Mover) MarkVolumeWritable(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, writable, persist bool) (err error) {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		if writable {
+			_, err = client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{
+				VolumeId: uint32(volumeId),
+			})
+		} else {
+			_, err = client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
+				VolumeId: uint32(volumeId),
+				Persist:  persist,
+			})
+		}
+		return err
+	})
+}
+
+// ReplicateVolume copies a volume from source to target without touching the
+// source — the replica-creation half of a move. ioBytePerSecond limits the
+// copy rate; 0 falls back to the volume server's maintenance rate.
+func (m *Mover) ReplicateVolume(ctx context.Context, volumeId needle.VolumeId, source, target pb.ServerAddress, diskType string, ioBytePerSecond int64, writer io.Writer) error {
+	if writer == nil {
+		writer = io.Discard
+	}
+	if SameServer(source, target) {
+		return fmt.Errorf("refusing to replicate volume %d onto its own server %s", volumeId, source)
+	}
+	// The target dials the embedded source itself; a malformed one would
+	// abort the target server, not this client.
+	if err := checkDialable(source); err != nil {
+		return err
+	}
+	return m.withClient(false, target, func(client volume_server_pb.VolumeServerClient) error {
+		stream, replicateErr := client.VolumeCopy(ctx, &volume_server_pb.VolumeCopyRequest{
+			VolumeId:        uint32(volumeId),
+			SourceDataNode:  string(source),
+			DiskType:        diskType,
+			IoBytePerSecond: ioBytePerSecond,
+		})
+		if replicateErr != nil {
+			return replicateErr
+		}
+		for {
+			resp, recvErr := stream.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				}
+				return recvErr
+			}
+			if resp.ProcessedBytes > 0 {
+				fmt.Fprintf(writer, "volume %d processed %s bytes\n", volumeId, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
+			}
+		}
+		return nil
+	})
+}
+
+// ConfigureVolumeReplication sets the replication setting on the volume at server.
+func (m *Mover) ConfigureVolumeReplication(ctx context.Context, volumeId needle.VolumeId, server pb.ServerAddress, replication string) error {
+	return m.withClient(false, server, func(client volume_server_pb.VolumeServerClient) error {
+		resp, configureErr := client.VolumeConfigure(ctx, &volume_server_pb.VolumeConfigureRequest{
+			VolumeId:    uint32(volumeId),
+			Replication: replication,
+		})
+		if configureErr != nil {
+			return configureErr
+		}
+		if resp.Error != "" {
+			return errors.New(resp.Error)
+		}
+		return nil
+	})
+}

weed/operation/volume_move/volume_move_test.go

@@ -0,0 +1,633 @@
+package volume_move
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	srcAddr = pb.ServerAddress("src:8080")
+	dstAddr = pb.ServerAddress("dst:8080")
+)
+
+func volumeStatus(datSize, idxSize, fileCount uint64) *volume_server_pb.ReadVolumeFileStatusResponse {
+	return &volume_server_pb.ReadVolumeFileStatusResponse{
+		DatFileSize: datSize,
+		IdxFileSize: idxSize,
+		FileCount:   fileCount,
+	}
+}
+
+func TestSameServer(t *testing.T) {
+	if !SameServer("node:8080", "node:8080.18080") {
+		t.Error("one server written with and without an explicit grpc port compared unequal")
+	}
+	// Test harnesses run several servers on HTTP port 0 with distinct grpc
+	// ports; the grpc endpoint is the server's identity.
+	if SameServer("127.0.0.1:0.36825", "127.0.0.1:0.38609") {
+		t.Error("two servers sharing a degenerate HTTP address compared equal")
+	}
+	// Malformed addresses (an unvalidated -source/-target flag) must compare
+	// without touching the fatal ToGrpcAddress parser.
+	if !SameServer("node:abc", "node:abc") {
+		t.Error("identical malformed addresses compared unequal")
+	}
+	if SameServer("node:abc", "node:8080") {
+		t.Error("malformed and well-formed addresses compared equal")
+	}
+	if SameServer("", "node:8080") {
+		t.Error("empty and well-formed addresses compared equal")
+	}
+}
+
+func TestCheckDialable(t *testing.T) {
+	// Rejected: the class the fatal parser aborts on, plus dotted forms with
+	// a non-numeric component — those canonicalize differently across dial
+	// paths ("node:8080.bad" dials as node:bad on one and node:18080 on the
+	// other, silently aliasing another server).
+	for _, addr := range []pb.ServerAddress{"node:abc", "host.domain.com:x8080", "[::1]:abc",
+		"node:8080.bad", "node:bad.8080", "node:.bad", "node:8080."} {
+		if err := checkDialable(addr); err == nil {
+			t.Errorf("%q accepted although it is unsafe to dial", addr)
+		}
+	}
+	// Accepted: normalized cleanly, or handed to the dialer untouched where
+	// a bad address fails as an ordinary error.
+	for _, addr := range []pb.ServerAddress{"node:8080", "node:8080.18080", "127.0.0.1:0.36825", "", "node", "node:"} {
+		if err := checkDialable(addr); err != nil {
+			t.Errorf("%q rejected: %v", addr, err)
+		}
+	}
+}
+
+func TestEmbeddedSourceAddressValidated(t *testing.T) {
+	// The target dials the source embedded in copy/tail requests through the
+	// same fatal parser; the client must reject a malformed source before
+	// issuing any RPC.
+	bad := pb.ServerAddress("node:abc")
+
+	ops := map[string]func(m *Mover) error{
+		"ReplicateVolume": func(m *Mover) error {
+			return m.ReplicateVolume(context.Background(), 7, bad, dstAddr, "", 0, nil)
+		},
+		"TailVolume": func(m *Mover) error {
+			return m.TailVolume(context.Background(), 7, bad, dstAddr, 0, time.Second)
+		},
+		"CopyAndMountEcShards": func(m *Mover) error {
+			return m.CopyAndMountEcShards(context.Background(), 7, "c1", []erasure_coding.ShardId{3}, bad, dstAddr, 0, 0, nil)
+		},
+		"MoveEcShards": func(m *Mover) error {
+			move := ecMove(3)
+			move.Source = bad
+			return m.MoveEcShards(context.Background(), move, EcMoveOptions{})
+		},
+	}
+	for name, op := range ops {
+		cluster := newFakeCluster()
+		err := op(cluster.mover())
+		if err == nil || !strings.Contains(err.Error(), "invalid volume server address") {
+			t.Errorf("%s: expected invalid-address error, got: %v", name, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Errorf("%s: RPCs issued with a malformed embedded source: %v", name, cluster.callList())
+		}
+	}
+}
+
+func TestNewMoverRejectsMalformedAddressWithoutDialing(t *testing.T) {
+	// The production dial path must fail the move, not the process, when a
+	// caller passes an unvalidated malformed address.
+	mover := NewMover(grpc.WithTransportCredentials(insecure.NewCredentials()))
+	err := mover.LiveMoveVolume(context.Background(), 7, "node:abc", "node:8080", VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "invalid volume server address") {
+		t.Fatalf("expected invalid-address error, got: %v", err)
+	}
+}
+
+func assertCalls(t *testing.T, got, want []string) {
+	t.Helper()
+	if fmt.Sprint(got) != fmt.Sprint(want) {
+		t.Fatalf("RPC sequence mismatch:\n got:  %v\n want: %v", got, want)
+	}
+}
+
+func TestLiveMoveVolumeSequence(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.lastAppendAtNs = 12345
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{
+		DiskType:        "ssd",
+		IoBytePerSecond: 42,
+	})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume: %v", err)
+	}
+
+	// The source status read must come after the tail: a write in flight when
+	// the source was frozen can land after an earlier read, and verifying
+	// against that stale snapshot would miss a missed tail. The first target
+	// status read probes whether the target already held the volume, so a
+	// failed copy never cleans up someone else's replica.
+	assertCalls(t, cluster.callList(), []string{
+		"src:8080 VolumeStatus",
+		"src:8080 VolumeMarkReadonly",
+		"dst:8080 ReadVolumeFileStatus",
+		"dst:8080 VolumeCopy",
+		"dst:8080 VolumeTailReceiver",
+		"src:8080 ReadVolumeFileStatus",
+		"dst:8080 ReadVolumeFileStatus",
+		"src:8080 VolumeDelete",
+	})
+
+	copyReq := cluster.copyReqs[0]
+	if copyReq.DiskType != "ssd" || copyReq.IoBytePerSecond != 42 || copyReq.SourceDataNode != string(srcAddr) {
+		t.Errorf("copy request not propagated: %+v", copyReq)
+	}
+	deleteReq := cluster.deleteReqs[0]
+	if deleteReq.OnlyEmpty || !deleteReq.KeepRemoteData {
+		t.Errorf("source delete must keep remote data and not be only-empty: %+v", deleteReq)
+	}
+}
+
+func TestLiveMoveVolumeVerifyMismatchKeepsSource(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "behind the source") {
+		t.Fatalf("expected target-behind error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted despite verification failure: %v", calls)
+		}
+	}
+	// The incomplete target copy is removed so the restored source stays the
+	// only replica, then the source is made writable again.
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeAbortKeepsSourceReadonlyWhenTargetCleanupFails(t *testing.T) {
+	// Restoring the source while the stale target stays mounted risks
+	// divergent replicas; the source stays readonly until a re-run recovers.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+	cluster.errs["dst:8080 VolumeDelete"] = errors.New("target unreachable")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected verification failure")
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("error does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("source made writable with the stale target still mounted: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeAlreadyReadonlyStaysReadonly(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("copy failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected copy failure")
+	}
+
+	// The hard mark is issued even for a readonly-reporting volume: that
+	// status also covers low-disk and readonly-but-can-delete states, which
+	// still accept needle deletes.
+	marked := false
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkReadonly" {
+			marked = true
+		}
+		// A volume that was readonly before the move (e.g. full) must stay so.
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatal("made an already-readonly volume writable after a failed move")
+		}
+	}
+	if !marked {
+		t.Fatalf("readonly-reporting source not hard-frozen: %v", cluster.callList())
+	}
+}
+
+func TestLiveMoveVolumePreFrozenAbortCleansTarget(t *testing.T) {
+	// tiering.run freezes replicas itself and thaws them after a failed move,
+	// so even a move that did not do the freezing must remove its target copy
+	// on abort — a stale mounted target plus a thawed source is divergence.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected verification failure")
+	}
+
+	cleaned := false
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeDelete" {
+			cleaned = true
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("thawed a source this move did not freeze: %v", cluster.callList())
+		}
+	}
+	if !cleaned {
+		t.Fatalf("incomplete target copy not cleaned up: %v", cluster.callList())
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorCleansMountedTarget(t *testing.T) {
+	// The server can finish the copy and mount the target even when the client
+	// loses the stream; the abort probes the target and cleans up the copy.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("stream lost")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected copy failure")
+	}
+
+	calls := cluster.callList()
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorUnknownTargetKeepsSourceReadonly(t *testing.T) {
+	// The pre-copy probe failed at the transport level, so whether the target
+	// held a replica before the move is unknown; the failed copy may still
+	// have mounted a complete copy there (the server can finish after the
+	// client loses the stream). Deleting it risks a healthy pre-existing
+	// replica, and reopening the source beside it risks two writable replicas
+	// taking divergent writes — so the source stays readonly.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.Unavailable, "connection refused")
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("stream lost")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected kept-readonly failure, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted the target with its prior state unknown: %v", calls)
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("reopened the source beside a possibly-mounted copy: %v", calls)
+		}
+	}
+}
+
+func TestLiveMoveVolumeCopyErrorLeavesPreexistingTarget(t *testing.T) {
+	// A target that held the volume before the move is someone else's replica;
+	// a failed copy must not delete it.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeCopy"] = errors.New("copy failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected kept-readonly failure, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted a pre-existing target replica after a failed copy: %v", calls)
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("reopened the source with a copy of unprovable origin on the target: %v", calls)
+		}
+	}
+}
+
+func TestLiveMoveVolumeTailErrorAborts(t *testing.T) {
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "tail volume") {
+		t.Fatalf("expected tail error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	for _, call := range calls {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted despite tail failure: %v", calls)
+		}
+	}
+	if fmt.Sprint(calls[len(calls)-2:]) != fmt.Sprint([]string{"dst:8080 VolumeDelete", "src:8080 VolumeMarkWritable"}) {
+		t.Fatalf("expected target cleanup then source restore, got: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeTailErrorToleratedForReadonlySource(t *testing.T) {
+	// A volume that was already readonly before the move was frozen for the
+	// whole copy, so a failed tail has nothing to deliver.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{IdleTimeout: time.Millisecond})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume on readonly source with failed tail: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not complete with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeTailFailureAbortsWhenSourceStillChanging(t *testing.T) {
+	// A tolerated tail failure substitutes a drain barrier: the source status
+	// must hold still across the idle window, or the move aborts.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1 // no volume on the target before the copy
+	cluster.statusSeq[string(srcAddr)] = []*volume_server_pb.ReadVolumeFileStatusResponse{
+		volumeStatus(1000, 100, 10),
+		volumeStatus(1024, 116, 11), // a straggler landed between the two reads
+	}
+	cluster.errs["dst:8080 VolumeTailReceiver"] = errors.New("tail failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{IdleTimeout: time.Millisecond})
+	if err == nil || !strings.Contains(err.Error(), "still changing") {
+		t.Fatalf("expected still-changing abort, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeDelete" {
+			t.Fatalf("source deleted while still changing: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeSourceDeleteFailureKeepsBothReadonly(t *testing.T) {
+	// Past verification the target is complete and may hold new writes, and
+	// the failed delete may or may not have reached the source. Deleting the
+	// target or reopening the source could fork the volume; keep everything
+	// readonly for a re-run.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.errs["src:8080 VolumeDelete"] = errors.New("delete timed out")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil {
+		t.Fatal("expected source delete failure")
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("error does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeDelete" {
+			t.Fatalf("deleted the verified target copy after a failed source delete: %v", cluster.callList())
+		}
+		if call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("restored source writability next to a mounted verified copy: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeRefusesRecopyOverCompleteTarget(t *testing.T) {
+	// A readonly source next to a target already holding a copy that is not
+	// behind it is the signature of a previous move whose source delete
+	// failed; recopying would tear down the possibly-authoritative target.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "already has a copy") {
+		t.Fatalf("expected recopy refusal, got: %v", err)
+	}
+	if !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("refusal does not mark the source as deliberately kept readonly: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" || call == "dst:8080 VolumeDelete" {
+			t.Fatalf("touched the possibly-authoritative target: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeReadonlySourceUnknownTargetRefused(t *testing.T) {
+	// With a readonly source, an unreachable target probe must fail closed:
+	// the target could hold the authoritative copy of a previous move.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.Unavailable, "connection refused")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected fail-closed refusal, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" {
+			t.Fatalf("copied over a target in unknown state: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumePreFrozenSourceMissingRustTargetProceeds(t *testing.T) {
+	// The Rust volume server answers a missing volume with codes.NotFound
+	// (the Go server uses a plain error, code Unknown). Both are definitive
+	// absence: with a pre-frozen source (tiering freezes before moving), a
+	// NotFound misclassified as "unknown" would refuse the move with
+	// ErrSourceKeptReadonly — and tiering callers then deliberately skip
+	// thawing, stranding every replica readonly after an ordinary move.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1000, 100, 10)
+	cluster.statusFailures[string(dstAddr)] = 1
+	cluster.statusFailureErr = status.Error(codes.NotFound, "not found volume id 7")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume to an empty Rust target: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not complete with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeReadonlySourceExistingTargetRefused(t *testing.T) {
+	// No client-side observation can prove an existing copy is a stale
+	// remnant rather than the authoritative copy of an unfinished move —
+	// compaction shrinks the authoritative copy below the stale source, and
+	// any snapshot can be invalidated by a write right after it. Even a
+	// target that reads as behind is refused.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(900, 90, 9)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !errors.Is(err, ErrSourceKeptReadonly) {
+		t.Fatalf("expected fail-closed refusal, got: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "dst:8080 VolumeCopy" || call == "dst:8080 VolumeDelete" {
+			t.Fatalf("touched a possibly-authoritative target: %v", cluster.callList())
+		}
+	}
+}
+
+func TestCopyVolumeLeavesReadonlySourceUntouched(t *testing.T) {
+	// The copy is non-destructive, so a readonly-reporting source (possibly
+	// only transiently, e.g. low disk) is not hard-marked.
+	cluster := newFakeCluster()
+	cluster.readonly[string(srcAddr)] = true
+
+	_, err := cluster.mover().CopyVolume(context.Background(), 7, srcAddr, dstAddr, "", 0, true, nil)
+	if err != nil {
+		t.Fatalf("CopyVolume: %v", err)
+	}
+
+	for _, call := range cluster.callList() {
+		if call == "src:8080 VolumeMarkReadonly" || call == "src:8080 VolumeMarkWritable" {
+			t.Fatalf("readonly source touched by a non-destructive copy: %v", cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeTargetAheadCommits(t *testing.T) {
+	// The target serves writes while the move tails; a target that got ahead
+	// holds those acknowledged writes, and the move must commit so they stay
+	// on the surviving copy.
+	cluster := newFakeCluster()
+	cluster.status[string(srcAddr)] = volumeStatus(1000, 100, 10)
+	cluster.status[string(dstAddr)] = volumeStatus(1100, 110, 11)
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err != nil {
+		t.Fatalf("LiveMoveVolume with target ahead: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeDelete" {
+		t.Fatalf("move did not commit with source delete: %v", calls)
+	}
+}
+
+func TestLiveMoveVolumeRejectsSameServer(t *testing.T) {
+	// The second target is the same server written with an explicit grpc port;
+	// the guard must see through the representation difference.
+	for _, target := range []pb.ServerAddress{srcAddr, pb.ServerAddress("src:8080.18080")} {
+		cluster := newFakeCluster()
+
+		err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, target, VolumeMoveOptions{})
+		if err == nil || !strings.Contains(err.Error(), "its own server") {
+			t.Fatalf("target %q: expected same-server rejection, got: %v", target, err)
+		}
+		if len(cluster.callList()) != 0 {
+			t.Fatalf("target %q: RPCs issued for a rejected move: %v", target, cluster.callList())
+		}
+	}
+}
+
+func TestLiveMoveVolumeReadonlyMarkFailureRestores(t *testing.T) {
+	// The marking RPC can fail after the server applied the mark (e.g. its
+	// master notification failed), so a mark error must still restore.
+	cluster := newFakeCluster()
+	cluster.errs["src:8080 VolumeMarkReadonly"] = errors.New("master notification failed")
+
+	err := cluster.mover().LiveMoveVolume(context.Background(), 7, srcAddr, dstAddr, VolumeMoveOptions{})
+	if err == nil || !strings.Contains(err.Error(), "mark volume") {
+		t.Fatalf("expected mark-readonly error, got: %v", err)
+	}
+
+	calls := cluster.callList()
+	if calls[len(calls)-1] != "src:8080 VolumeMarkWritable" {
+		t.Fatalf("source writability not restored after failed readonly mark: %v", calls)
+	}
+}
+
+func TestReplicateVolumePropagatesThroughput(t *testing.T) {
+	cluster := newFakeCluster()
+
+	if err := cluster.mover().ReplicateVolume(context.Background(), 7, srcAddr, dstAddr, "ssd", 55, nil); err != nil {
+		t.Fatalf("ReplicateVolume: %v", err)
+	}
+	copyReq := cluster.copyReqs[0]
+	if copyReq.IoBytePerSecond != 55 || copyReq.DiskType != "ssd" {
+		t.Errorf("replicate request not propagated: %+v", copyReq)
+	}
+}
+
+func TestCopyVolumeRestoreWritable(t *testing.T) {
+	for _, restoreWritable := range []bool{true, false} {
+		cluster := newFakeCluster()
+		_, err := cluster.mover().CopyVolume(context.Background(), 7, srcAddr, dstAddr, "", 0, restoreWritable, nil)
+		if err != nil {
+			t.Fatalf("CopyVolume(restoreWritable=%v): %v", restoreWritable, err)
+		}
+		restored := false
+		for _, call := range cluster.callList() {
+			if call == "src:8080 VolumeMarkWritable" {
+				restored = true
+			}
+		}
+		if restored != restoreWritable {
+			t.Errorf("restoreWritable=%v but writability restored=%v", restoreWritable, restored)
+		}
+	}
+}