snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/balancer/clustermanager/balancerstateaggregator.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+)
+
+type subBalancerState struct {
+	state balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
+}
+
+func (s *subBalancerState) String() string {
+	return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
+}
+
+type balancerStateAggregator struct {
+	cc     balancer.ClientConn
+	logger *grpclog.PrefixLogger
+	csEval *balancer.ConnectivityStateEvaluator
+
+	mu sync.Mutex
+	// This field is used to ensure that no updates are forwarded to the parent
+	// CC once the aggregator is closed. A closed sub-balancer could still send
+	// pickers to this aggregator.
+	closed bool
+	// Map from child policy name to last reported state.
+	idToPickerState map[string]*subBalancerState
+	// Set when UpdateState call propagation is paused.
+	pauseUpdateState bool
+	// Set when UpdateState call propagation is paused and an UpdateState call
+	// is suppressed.
+	needUpdateStateOnResume bool
+}
+
+func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator {
+	return &balancerStateAggregator{
+		cc:              cc,
+		logger:          logger,
+		csEval:          &balancer.ConnectivityStateEvaluator{},
+		idToPickerState: make(map[string]*subBalancerState),
+	}
+}
+
+func (bsa *balancerStateAggregator) close() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.closed = true
+}
+
+// add adds a sub-balancer in CONNECTING state.
+//
+// This is called when there's a new child.
+func (bsa *balancerStateAggregator) add(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+
+	bsa.idToPickerState[id] = &subBalancerState{
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
+	}
+	bsa.csEval.RecordTransition(connectivity.Shutdown, connectivity.Connecting)
+	bsa.buildAndUpdateLocked()
+}
+
+// remove removes the sub-balancer state. Future updates from this sub-balancer,
+// if any, will be ignored.
+//
+// This is called when a child is removed.
+func (bsa *balancerStateAggregator) remove(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	if _, ok := bsa.idToPickerState[id]; !ok {
+		return
+	}
+	// Setting the state of the deleted sub-balancer to Shutdown will get
+	// csEvltr to remove the previous state for any aggregated state
+	// evaluations. Transitions to and from connectivity.Shutdown are ignored
+	// by csEvltr.
+	bsa.csEval.RecordTransition(bsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown)
+	// Remove id and picker from picker map. This also results in future updates
+	// for this ID to be ignored.
+	delete(bsa.idToPickerState, id)
+	bsa.buildAndUpdateLocked()
+}
+
+// pauseStateUpdates causes UpdateState calls to not propagate to the parent
+// ClientConn.  The last state will be remembered and propagated when
+// ResumeStateUpdates is called.
+func (bsa *balancerStateAggregator) pauseStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = true
+	bsa.needUpdateStateOnResume = false
+}
+
+// resumeStateUpdates will resume propagating UpdateState calls to the parent,
+// and call UpdateState on the parent if any UpdateState call was suppressed.
+func (bsa *balancerStateAggregator) resumeStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = false
+	if bsa.needUpdateStateOnResume {
+		bsa.cc.UpdateState(bsa.buildLocked())
+	}
+}
+
+// UpdateState is called to report a balancer state change from sub-balancer.
+// It's usually called by the balancer group.
+//
+// It calls parent ClientConn's UpdateState with the new aggregated state.
+func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
+	bsa.logger.Infof("State update from sub-balancer %q: %+v", id, state)
+
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	pickerSt, ok := bsa.idToPickerState[id]
+	if !ok {
+		// All state starts with an entry in pickStateMap. If ID is not in map,
+		// it's either removed, or never existed.
+		return
+	}
+	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		bsa.csEval.RecordTransition(pickerSt.stateToAggregate, state.ConnectivityState)
+		pickerSt.stateToAggregate = state.ConnectivityState
+	}
+	pickerSt.state = state
+	bsa.buildAndUpdateLocked()
+}
+
+// buildAndUpdateLocked combines the sub-state from each sub-balancer into one
+// state, and sends a picker update to the parent ClientConn.
+func (bsa *balancerStateAggregator) buildAndUpdateLocked() {
+	if bsa.closed {
+		return
+	}
+	if bsa.pauseUpdateState {
+		// If updates are paused, do not call UpdateState, but remember that we
+		// need to call it when they are resumed.
+		bsa.needUpdateStateOnResume = true
+		return
+	}
+	bsa.cc.UpdateState(bsa.buildLocked())
+}
+
+// buildLocked combines sub-states into one.
+func (bsa *balancerStateAggregator) buildLocked() balancer.State {
+	// The picker's return error might not be consistent with the
+	// aggregatedState. Because for this LB policy, we want to always build
+	// picker with all sub-pickers (not only ready sub-pickers), so even if the
+	// overall state is Ready, pick for certain RPCs can behave like Connecting
+	// or TransientFailure.
+	bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState)
+	return balancer.State{
+		ConnectivityState: bsa.csEval.CurrentState(),
+		Picker:            newPickerGroup(bsa.idToPickerState),
+	}
+}

internal/xds/balancer/clustermanager/clustermanager.go

@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clustermanager implements the cluster manager LB policy for xds.
+package clustermanager
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/balancergroup"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const balancerName = "xds_cluster_manager_experimental"
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	b := &bal{}
+	b.logger = prefixLogger(b)
+	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
+	b.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               opts,
+		StateAggregator:         b.stateAggregator,
+		Logger:                  b.logger,
+		SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies
+	})
+	b.logger.Infof("Created")
+	return b
+}
+
+func (bb) Name() string {
+	return balancerName
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type bal struct {
+	logger          *internalgrpclog.PrefixLogger
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *balancerStateAggregator
+
+	children map[string]childConfig
+}
+
+func (b *bal) setErrorPickerForChild(childName string, err error) {
+	b.stateAggregator.UpdateState(childName, balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	})
+}
+
+func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) error {
+	endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints)
+
+	// Remove sub-balancers that are not in the new list from the aggregator and
+	// balancergroup.
+	for name := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			b.stateAggregator.remove(name)
+			b.bg.Remove(name)
+		}
+	}
+
+	var retErr error
+	for childName, childCfg := range newConfig.Children {
+		lbCfg := childCfg.ChildPolicy.Config
+		if _, ok := b.children[childName]; !ok {
+			// Add new sub-balancers to the aggregator and balancergroup.
+			b.stateAggregator.add(childName)
+			b.bg.Add(childName, balancer.Get(childCfg.ChildPolicy.Name))
+		} else {
+			// If the child policy type has changed for existing sub-balancers,
+			// parse the new config and send down the config update to the
+			// balancergroup, which will take care of gracefully switching the
+			// child over to the new policy.
+			//
+			// If we run into errors here, we need to ensure that RPCs to this
+			// child fail, while RPCs to other children with good configs
+			// continue to succeed.
+			newPolicyName, oldPolicyName := childCfg.ChildPolicy.Name, b.children[childName].ChildPolicy.Name
+			if newPolicyName != oldPolicyName {
+				var err error
+				var cfgJSON []byte
+				cfgJSON, err = childCfg.ChildPolicy.MarshalJSON()
+				if err != nil {
+					retErr = fmt.Errorf("failed to JSON marshal load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+				// This overwrites lbCfg to be in the format expected by the
+				// gracefulswitch balancer. So, when this config is pushed to
+				// the child (below), it will result in a graceful switch to the
+				// new child policy.
+				lbCfg, err = balancergroup.ParseConfig(cfgJSON)
+				if err != nil {
+					retErr = fmt.Errorf("failed to parse load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+			}
+		}
+
+		if err := b.bg.UpdateClientConnState(childName, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Endpoints:     endpointsSplit[childName],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: lbCfg,
+		}); err != nil {
+			retErr = fmt.Errorf("failed to push new configuration %v to child %q: %v", childCfg.ChildPolicy.Config, childName, err)
+			b.setErrorPickerForChild(childName, retErr)
+		}
+
+		// Picker update is sent to the parent ClientConn only after the
+		// new child policy returns a picker. So, there is no need to
+		// set needUpdateStateOnResume to true here.
+	}
+
+	b.children = newConfig.Children
+
+	// If multiple sub-balancers run into errors, we will return only the last
+	// one, which is still good enough, since the grpc channel will anyways
+	// return this error as balancer.ErrBadResolver to the name resolver,
+	// resulting in re-resolution attempts.
+	return retErr
+
+	// Adding or removing a sub-balancer will result in the
+	// needUpdateStateOnResume bit to true which results in a picker update once
+	// resumeStateUpdates() is called.
+}
+
+func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error {
+	if b.logger.V(2) {
+		b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
+	}
+
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	b.stateAggregator.pauseStateUpdates()
+	defer b.stateAggregator.resumeStateUpdates()
+	return b.updateChildren(s, newConfig)
+}
+
+func (b *bal) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *bal) Close() {
+	b.stateAggregator.close()
+	b.bg.Close()
+	b.logger.Infof("Shutdown")
+}
+
+func (b *bal) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+const prefix = "[xds-cluster-manager-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *bal) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clustermanager/clustermanager_test.go

@@ -0,0 +1,775 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+	testBackendAddrsCount   = 12
+)
+
+var testBackendAddrStrs []string
+
+func init() {
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+}
+
+func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) {
+	t.Helper()
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p.Pick(info)
+		if fmt.Sprint(err) != fmt.Sprint(wantErr) {
+			t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr)
+		}
+		if gotSCSt.SubConn != wantSC {
+			t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC)
+		}
+	}
+}
+
+func (s) TestClusterPicks(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestConfigUpdateAddCluster covers the cases the balancer receives config
+// update with extra clusters.
+func (s) TestConfigUpdateAddCluster(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with different routes, and different actions. Expect a
+	// new subconn and a picker update.
+	configJSON2 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_3":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], BalancerAttributes: nil})
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[2]}}, []string{"cds:cluster_3"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect exactly one new subconn.
+	addrs := <-cc.NewSubConnAddrsCh
+	sc := <-cc.NewSubConnCh
+	// Clear the attributes before adding to map.
+	addrs[0].BalancerAttributes = nil
+	m1[addrs[0]] = sc
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Should have no more newSubConn.
+	select {
+	case <-time.After(time.Millisecond * 500):
+	case <-cc.NewSubConnCh:
+		addrs := <-cc.NewSubConnAddrsCh
+		t.Fatalf("unexpected NewSubConn with address %v", addrs)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_3"),
+			},
+			wantSC: m1[wantAddrs[2]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives
+// config update with no clusters. Pick should fail with details in error.
+func (s) TestRoutingConfigUpdateDeleteAll(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with no clusters.
+	configJSON2 := `{}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect two removed subconns.
+	for range wantAddrs {
+		select {
+		case <-time.After(time.Millisecond * 500):
+			t.Fatalf("timeout waiting for remove subconn")
+		case <-cc.ShutdownSubConnCh:
+		}
+	}
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(ctx, "cds:notacluster")})
+		if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() {
+			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`)
+		}
+	}
+
+	// Resend the previous config with clusters
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m2 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m2[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p3 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m2[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m2[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+func (s) TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) {
+	const (
+		userAgent          = "ua"
+		defaultTestTimeout = 1 * time.Second
+	)
+
+	// Setup the stub balancer such that we can read the build options passed to
+	// it in the UpdateClientConnState method.
+	ccsCh := testutils.NewChannel()
+	bOpts := balancer.BuildOptions{
+		DialCreds:       insecure.NewCredentials(),
+		CustomUserAgent: userAgent,
+	}
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			if !cmp.Equal(bd.BuildOptions, bOpts) {
+				err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
+				ccsCh.Send(err)
+				return err
+			}
+			ccsCh.Send(nil)
+			return nil
+		},
+	})
+
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, bOpts)
+
+	configJSON1 := fmt.Sprintf(`{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"%s":""}] }
+}
+}`, t.Name())
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := ccsCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timed out waiting for UpdateClientConnState result: %v", err)
+	}
+	if v != nil {
+		t.Fatal(v)
+	}
+}
+
+const initIdleBalancerName = "test-init-idle-balancer"
+
+var errTestInitIdle = fmt.Errorf("init Idle balancer error 0")
+
+func init() {
+	stub.Register(initIdleBalancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
+			sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(state balancer.SubConnState) {
+					err := fmt.Errorf("wrong picker error")
+					if state.ConnectivityState == connectivity.Idle {
+						err = errTestInitIdle
+					}
+					bd.ClientConn.UpdateState(balancer.State{
+						ConnectivityState: state.ConnectivityState,
+						Picker:            &testutils.TestConstPicker{Err: err},
+					})
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+}
+
+// TestInitialIdle covers the case that if the child reports Idle, the overall
+// state will be Idle.
+func (s) TestInitialIdle(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"test-init-idle-balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		sc := <-cc.NewSubConnCh
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
+	}
+
+	if state1 := <-cc.NewStateCh; state1 != connectivity.Idle {
+		t.Fatalf("Received aggregated state: %v, want Idle", state1)
+	}
+}
+
+// TestClusterGracefulSwitch tests the graceful switch functionality for a child
+// of the cluster_manager. The test performs the following steps:
+//  1. Configures a single child for the cluster_manager with a round_robin LB
+//     policy. Ensures the child sends a picker update when the subchannel moves
+//     to READY.
+//  2. Updates the exiting child with a different LB policy (pick_first), and a
+//     different endpoint. This should trigger the graceful switch process, and
+//     cluster_manager should not send a picker update until the child has
+//     completed the graceful switch process.
+//  3. The child should complete the graceful switch process once it receives an
+//     update that moves its subchannel to READY. At this point, the cluster_manager
+//     should send a picker update reflecting the new pick_first balancer's picker.
+func (s) TestClusterGracefulSwitch(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	// Push round_robin config with one endpoint to the cluster_manager.
+	rrConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"round_robin": {}}] }
+		}
+	}`
+	rrConfig, err := parser.ParseConfig([]byte(rrConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse round_robin config: %v", err)
+	}
+	rrEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "rr-backend"}}}, []string{"csp:cluster"})
+	ccs := balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{rrEndpoint}},
+		BalancerConfig: rrConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send round_robin config update to cluster_manager: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Wait for a subchannel to be created.
+	var sc1 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the round_robin LB policy")
+	case sc1 = <-cc.NewSubConnCh:
+	}
+
+	// Update the subchannel's state to READY, and ensure we get a picker update.
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	var p1 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p1 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	pi := balancer.PickInfo{Ctx: SetPickedCluster(ctx, "csp:cluster")}
+	testPick(t, p1, pi, sc1, nil)
+
+	// Update the config to use pick_first, and a different endpoint.
+	pfConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"pick_first": {}}] }
+		}
+	}`
+	pfConfig, err := parser.ParseConfig([]byte(pfConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse pick_first config: %v", err)
+	}
+	pfEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "pf-backend"}}}, []string{"csp:cluster"})
+	ccs = balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{pfEndpoint}},
+		BalancerConfig: pfConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send pick_first config update to cluster_manager: %v", err)
+	}
+
+	// Wait for a subchannel to be created.
+	var sc2 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the pick_first LB policy")
+	case sc2 = <-cc.NewSubConnCh:
+	}
+
+	// Move the pick_first LB policy's subchannel to CONNECTING. This will cause
+	// the pick_first LB policy to send a connecting picker. But this should not
+	// result in the cluster_manager sending a picker update to the channel as
+	// the graceful switch process is not yet complete.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("Received picker from cluster_manager when none expected during graceful switch process")
+	case <-sCtx.Done():
+	}
+
+	// Move the pick_first LB policy's subchannel to READY. This will cause the
+	// pick_first LB policy to send a ready picker and complete the graceful
+	// switch process.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	var p2 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p2 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	testPick(t, p2, pi, sc2, nil)
+
+	// Completion of the graceful switch process should result in the old
+	// subchannel being removed.
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannels belonging to old child to be deleted")
+	case rsc := <-cc.ShutdownSubConnCh:
+		if rsc != sc1 {
+			t.Fatalf("Unexpected subchannel %v being shutdown, want %v", rsc, sc1)
+		}
+	}
+}
+
+// tcc wraps a testutils.TestClientConn but stores all state transitions in a
+// slice.
+type tcc struct {
+	*testutils.BalancerClientConn
+	states []balancer.State
+}
+
+func (t *tcc) UpdateState(bs balancer.State) {
+	t.states = append(t.states, bs)
+	t.BalancerClientConn.UpdateState(bs)
+}
+
+func (s) TestUpdateStatePauses(t *testing.T) {
+	cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)}
+
+	balFuncs := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil})
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil})
+			return nil
+		},
+	}
+	stub.Register("update_state_balancer", balFuncs)
+
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"update_state_balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that the only state update is the second one called by the child.
+	if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready {
+		t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states)
+	}
+}

internal/xds/balancer/clustermanager/config.go

@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type childConfig struct {
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+// lbConfig is the balancer config for xds routing policy.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Children map[string]childConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	cfg := &lbConfig{}
+	if err := json.Unmarshal(c, cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}

internal/xds/balancer/clustermanager/config_test.go

@@ -0,0 +1,144 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer"
+)
+
+const (
+	testJSONConfig = `{
+      "children":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "weighted:cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1" : {
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2" : {
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }
+            }
+          }]
+        },
+        "weighted:cluster_1_cluster_3_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1": {
+                  "weight":99,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_3": {
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+                }
+              }
+            }
+          }]
+        }
+      }
+}
+`
+
+	cdsName = "cds_experimental"
+	wtName  = "weighted_target_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON1  = `{
+	"targets": {
+	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
+	} }`
+	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
+	wtConfigJSON2 = `{
+    "targets": {
+      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
+    } }`
+	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
+)
+
+func (s) Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Children: map[string]childConfig{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig1},
+					},
+					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig2},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", d)
+			}
+		})
+	}
+}

internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go

@@ -0,0 +1,342 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
+	"github.com/google/uuid"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/xds" // Register the xDS name resolver and related LB policies.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+func makeEmptyCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("EmptyCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func makeUnaryCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("UnaryCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Configure client side xDS resources on the management server.
+	const (
+		serviceName     = "my-service-client-side-xds"
+		routeConfigName = "route-" + serviceName
+		clusterName1    = "cluster1-" + serviceName
+		clusterName2    = "cluster2-" + serviceName
+		endpointsName1  = "endpoints1-" + serviceName
+		endpointsName2  = "endpoints2-" + serviceName
+		endpointsName3  = "endpoints3-" + serviceName
+	)
+	// A single Listener resource pointing to the following Route
+	// configuration:
+	//   - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	//   - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+			},
+		}},
+	}}
+	// Two cluster resources corresponding to the ones mentioned in the above
+	// route configuration resource. These are configured with round_robin as
+	// their endpoint picking policy.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelNone),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+	}
+	// Spin up two test backends, one for each cluster below.
+	server1 := stubserver.StartTestService(t, nil)
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	// Two endpoints resources, each with one backend from above.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2.
+	if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a wrapped pickfirst LB policy. When the endpoint picking policy on
+	// the cluster resource is changed to pickfirst, this will allow us to
+	// verify that load balancing configuration is pushed to it.
+	pfBuilder := balancer.Get(pickfirst.Name)
+	internal.BalancerUnregister(pfBuilder.Name())
+
+	lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1)
+	stub.Register(pfBuilder.Name(), stub.BalancerFuncs{
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return pfBuilder.(balancer.ConfigParser).ParseConfig(lbCfg)
+		},
+		Init: func(bd *stub.BalancerData) {
+			bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			select {
+			case lbCfgCh <- ccs.BalancerConfig:
+			default:
+			}
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+	})
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to pickfirst. The endpoints resource is also changed here
+	// to ensure that we can verify that the new child policy
+	cluster2 := &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{
+							ShuffleAddressList: true,
+						}),
+					},
+				},
+			},
+		},
+	}
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+	endpoints3 := e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)})
+	resources.Clusters = append(resources.Clusters, cluster2)
+	resources.Endpoints = append(resources.Endpoints, endpoints3)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for configuration to be pushed to the new pickfirst child policy")
+	case <-lbCfgCh:
+	}
+
+	// Ensure RPCs are still succeeding.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2, and the
+	// new endpoints resource.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server3.Address); err == nil {
+			break
+		}
+		t.Log(err)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to be routed to the new endpoints resource")
+	}
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to an unsupported LB policy. This should result in
+	// failure of RPCs to that cluster.
+	cluster2 = &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						// The type not registered in gRPC Policy registry.
+						TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+							TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist",
+							Value:   &structpb.Struct{},
+						}),
+					},
+				},
+			},
+		},
+	}
+	resources.Clusters[1] = cluster2
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// At this point, RPCs to cluster1 should continue to succeed, while RPCs to
+	// cluster2 should start to fail.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it starts to fail.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		_, err := client.UnaryCall(ctx, &testpb.SimpleRequest{})
+		got := status.Code(err)
+		if got == codes.Unavailable {
+			break
+		}
+		t.Logf("UnaryCall() returned code: %v, want %v", got, codes.Unavailable)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to start failing")
+	}
+
+	// Channel should still be READY.
+	if got, want := cc.GetState(), connectivity.Ready; got != want {
+		t.Fatalf("grpc.ClientConn in state %v, want %v", got, want)
+	}
+}

internal/xds/balancer/clustermanager/picker.go

@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// pickerGroup contains a list of pickers. If the picker isn't ready, the pick
+// will be queued.
+type pickerGroup struct {
+	pickers map[string]balancer.Picker
+}
+
+func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup {
+	pickers := make(map[string]balancer.Picker)
+	for id, st := range idToPickerState {
+		pickers[id] = st.state.Picker
+	}
+	return &pickerGroup{
+		pickers: pickers,
+	}
+}
+
+func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	cluster := PickedCluster(info.Ctx)
+	if p := pg.pickers[cluster]; p != nil {
+		return p.Pick(info)
+	}
+	return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster)
+}
+
+type clusterKey struct{}
+
+// PickedCluster returns the cluster name stored in the context.
+func PickedCluster(ctx context.Context) string {
+	cluster, _ := ctx.Value(clusterKey{}).(string)
+	return cluster
+}
+
+// SetPickedCluster adds the selected cluster to the context for the
+// xds_cluster_manager LB policy to pick.
+func SetPickedCluster(ctx context.Context, cluster string) context.Context {
+	return context.WithValue(ctx, clusterKey{}, cluster)
+}