snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/server/master_grpc_server_collection_test.go

@@ -0,0 +1,287 @@
+package weed_server
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/topology"
+)
+
+// recordingVolumeServer reports what each DeleteCollection arrived carrying.
+// failWith answers an error at once; hang holds the call until the caller gives
+// up or the test releases it, the way a server that is still reachable but
+// wedged behaves.
+type recordingVolumeServer struct {
+	volume_server_pb.UnimplementedVolumeServerServer
+
+	hang     bool
+	failWith error
+
+	calls       chan deleteCollectionCall
+	release     chan struct{}
+	releaseOnce sync.Once
+}
+
+type deleteCollectionCall struct {
+	collection string
+	// budget is the time the RPC arrived with, or 0 when it carried no deadline.
+	budget time.Duration
+}
+
+func newRecordingVolumeServer() *recordingVolumeServer {
+	return &recordingVolumeServer{
+		calls:   make(chan deleteCollectionCall, 8),
+		release: make(chan struct{}),
+	}
+}
+
+func (s *recordingVolumeServer) DeleteCollection(ctx context.Context, req *volume_server_pb.DeleteCollectionRequest) (*volume_server_pb.DeleteCollectionResponse, error) {
+	call := deleteCollectionCall{collection: req.Collection}
+	if deadline, ok := ctx.Deadline(); ok {
+		call.budget = time.Until(deadline)
+	}
+	s.calls <- call
+
+	if s.failWith != nil {
+		return nil, s.failWith
+	}
+	if s.hang {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-s.release:
+		}
+	}
+	return &volume_server_pb.DeleteCollectionResponse{}, nil
+}
+
+// Release unblocks every held DeleteCollection. Safe to call more than once.
+func (s *recordingVolumeServer) Release() {
+	s.releaseOnce.Do(func() { close(s.release) })
+}
+
+// addVolumeServer publishes stub on a fresh listener and registers it as a data
+// node of topo. Only the grpc port is dialed; the http port just has to stay
+// distinct per server so the topology does not treat two nodes as one address.
+func addVolumeServer(t *testing.T, topo *topology.Topology, id string, stub *recordingVolumeServer) *topology.DataNode {
+	t.Helper()
+	t.Cleanup(stub.Release)
+	grpcPort := serveGrpc(t, func(s *grpc.Server) {
+		volume_server_pb.RegisterVolumeServerServer(s, stub)
+	})
+	return topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
+		GetOrCreateDataNode("127.0.0.1", grpcPort-10000, grpcPort, "", id, map[string]uint32{"": 10})
+}
+
+func newCollectionTestMaster(topo *topology.Topology) *MasterServer {
+	return &MasterServer{
+		Topo:           topo,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+}
+
+// awaitCall waits for one DeleteCollection to reach the stub.
+func awaitCall(t *testing.T, stub *recordingVolumeServer, what string) deleteCollectionCall {
+	t.Helper()
+	select {
+	case call := <-stub.calls:
+		return call
+	case <-time.After(15 * time.Second):
+		t.Fatalf("%s: the volume server never received DeleteCollection", what)
+		return deleteCollectionCall{}
+	}
+}
+
+// A caller with no deadline of its own must still not wait forever: the master
+// has to bound each RPC, or one wedged volume server holds the fan-out open with
+// nothing to end it.
+func TestDoDeleteNormalCollectionBoundsEachVolumeServerRPC(t *testing.T) {
+	const collection = "bucket-a"
+
+	stub := newRecordingVolumeServer()
+	stub.hang = true
+
+	topo := topology.NewTopology("test", nil, 32*1024, 5, false)
+	dn := addVolumeServer(t, topo, "vs1", stub)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+	}, dn)
+	ms := newCollectionTestMaster(topo)
+
+	done := make(chan error, 1)
+	go func() { done <- ms.doDeleteNormalCollection(context.Background(), collection) }()
+
+	call := awaitCall(t, stub, "bounded RPC")
+	if call.budget <= 0 {
+		t.Fatal("DeleteCollection reached the volume server with no deadline: a wedged server holds the fan-out open with nothing to end it")
+	}
+	if call.budget > deleteCollectionTimeout {
+		t.Errorf("DeleteCollection budget = %v, want at most deleteCollectionTimeout %v", call.budget, deleteCollectionTimeout)
+	}
+
+	stub.Release()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Errorf("doDeleteNormalCollection returned %v, want nil once the volume server answers", err)
+		}
+	case <-time.After(15 * time.Second):
+		t.Fatal("doDeleteNormalCollection did not return after the volume server answered")
+	}
+}
+
+// The fan-out must outlive the caller. A collection can span many servers and
+// the deletes are sequential, so letting an inbound cancellation end the loop
+// would abandon a large delete part-done, with volumes left behind and no
+// request still running to come back for them.
+func TestDeleteCollectionOutlivesTheCallersCancellation(t *testing.T) {
+	const collection = "bucket-b"
+
+	normal := newRecordingVolumeServer()
+	ec := newRecordingVolumeServer()
+
+	topo := topology.NewTopology("test", nil, 32*1024, 5, false)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+	}, addVolumeServer(t, topo, "vs-normal", normal))
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: collection, EcIndexBits: 0x1f},
+	}, addVolumeServer(t, topo, "vs-ec", ec))
+	ms := newCollectionTestMaster(topo)
+
+	// The caller is already gone by the time the delete starts.
+	expired, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	if err := ms.deleteCollection(expired, collection); err != nil {
+		t.Fatalf("deleteCollection returned %v; a cancelled caller must not fail the fan-out", err)
+	}
+
+	// The EC pass runs second, so it is the one most likely to find the caller
+	// already gone.
+	for _, tc := range []struct {
+		stub *recordingVolumeServer
+		what string
+	}{{normal, "normal pass"}, {ec, "ec pass"}} {
+		call := awaitCall(t, tc.stub, "cancelled caller, "+tc.what)
+		if call.collection != collection {
+			t.Errorf("%s: server was told to delete %q, want %q", tc.what, call.collection, collection)
+		}
+		if call.budget <= 0 {
+			t.Errorf("%s: DeleteCollection arrived with no deadline; each RPC should still be bounded", tc.what)
+		}
+	}
+
+	if _, found := topo.FindCollection(collection); found {
+		t.Error("the collection was left in the topology; the delete did not run to completion")
+	}
+}
+
+// ListVolumeServers reports a node once per replica it holds. DeleteCollection
+// removes the whole collection from the server it reaches, so the master must
+// send it once per server, not once per replica.
+func TestDoDeleteNormalCollectionSendsOneRPCPerVolumeServer(t *testing.T) {
+	const collection = "bucket-c"
+
+	stub := newRecordingVolumeServer()
+
+	topo := topology.NewTopology("test", nil, 32*1024, 5, false)
+	dn := addVolumeServer(t, topo, "vs1", stub)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+		{Id: 2, Collection: collection, Version: 3},
+		{Id: 3, Collection: collection, Version: 3},
+	}, dn)
+
+	collectionInTopology, found := topo.FindCollection(collection)
+	if !found {
+		t.Fatalf("test setup: collection %s was not registered", collection)
+	}
+	if listed := len(collectionInTopology.ListVolumeServers()); listed < 2 {
+		t.Fatalf("test setup: the one node is listed %d times, expected once per replica", listed)
+	}
+
+	ms := newCollectionTestMaster(topo)
+	if err := ms.doDeleteNormalCollection(context.Background(), collection); err != nil {
+		t.Fatalf("doDeleteNormalCollection: %v", err)
+	}
+
+	awaitCall(t, stub, "one RPC per server")
+	select {
+	case extra := <-stub.calls:
+		t.Errorf("the same server was told to delete %q more than once", extra.collection)
+	default:
+	}
+}
+
+// One server refusing must not leave the collection on every server after it in
+// the list: returning at the first failure meant a single node that was down
+// stranded the rest, and the failure is reported either way.
+func TestDoDeleteNormalCollectionKeepsGoingPastAFailingServer(t *testing.T) {
+	const collection = "bucket-d"
+
+	failing := newRecordingVolumeServer()
+	failing.failWith = errors.New("volume server is out of disk")
+	healthy := newRecordingVolumeServer()
+
+	topo := topology.NewTopology("test", nil, 32*1024, 5, false)
+	failingNode := addVolumeServer(t, topo, "vs-failing", failing)
+	healthyNode := addVolumeServer(t, topo, "vs-healthy", healthy)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+	}, failingNode)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+	}, healthyNode)
+
+	ms := newCollectionTestMaster(topo)
+	if err := ms.doDeleteNormalCollection(context.Background(), collection); err == nil {
+		t.Fatal("doDeleteNormalCollection reported success while a server refused")
+	}
+
+	awaitCall(t, failing, "failing server")
+	awaitCall(t, healthy, "server behind the failing one")
+
+	if _, found := topo.FindCollection(collection); !found {
+		t.Error("the collection was dropped from the topology despite failing; nothing would come back for the rest")
+	}
+}
+
+// A collection can hold normal volumes and EC shards at once. Returning after a
+// failed normal pass left the EC shards in place with no request left to come
+// back for them, so both passes have to run.
+func TestDeleteCollectionRunsTheEcPassWhenTheNormalPassFails(t *testing.T) {
+	const collection = "bucket-e"
+
+	normal := newRecordingVolumeServer()
+	normal.failWith = errors.New("volume server is out of disk")
+	ec := newRecordingVolumeServer()
+
+	topo := topology.NewTopology("test", nil, 32*1024, 5, false)
+	topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{
+		{Id: 1, Collection: collection, Version: 3},
+	}, addVolumeServer(t, topo, "vs-normal", normal))
+	topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
+		{Id: 9, Collection: collection, EcIndexBits: 0x1f},
+	}, addVolumeServer(t, topo, "vs-ec", ec))
+
+	ms := newCollectionTestMaster(topo)
+	if err := ms.deleteCollection(context.Background(), collection); err == nil {
+		t.Fatal("deleteCollection reported success while the normal pass failed")
+	}
+
+	// The failure is reported, and the EC shards are cleaned up anyway.
+	awaitCall(t, ec, "ec pass after a failed normal pass")
+
+	if _, found := topo.FindCollection(collection); !found {
+		t.Error("the normal collection was dropped from the topology despite failing; nothing would retry it")
+	}
+}