snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/balancer/balancer.go

@@ -0,0 +1,30 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package balancer installs all the xds balancers.
+package balancer
+
+import (
+	_ "google.golang.org/grpc/balancer/leastrequest"                  // Register the least_request_experimental balancer
+	_ "google.golang.org/grpc/balancer/weightedtarget"                // Register the weighted_target balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer"      // Register the CDS balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/clusterimpl"      // Register the xds_cluster_impl balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/clustermanager"   // Register the xds_cluster_manager balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // Register the outlier_detection balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/priority"         // Register the priority balancer
+)

internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go

@@ -0,0 +1,932 @@
+/*
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/balancer/pickfirst"
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter.
+	_ "google.golang.org/grpc/internal/xds/resolver"          // Register the xds resolver
+)
+
+// makeAggregateClusterResource returns an aggregate cluster resource with the
+// given name and list of child names.
+func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeAggregate,
+		ChildNames:  childNames,
+	})
+}
+
+// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the
+// given name and given DNS host and port.
+func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeLogicalDNS,
+		DNSHostName: dnsHost,
+		DNSPort:     dnsPort,
+	})
+}
+
+// verifyDNSResolution verifies that the DNS resolver is started for the expected target.
+func verifyDNSResolution(ctx context.Context, t *testing.T, dnsTargetCh chan resolver.Target, dnsR *manual.Resolver, host string, port uint32) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for DNS watch")
+	case target := <-dnsTargetCh:
+		addr := fmt.Sprintf("%s:%d", host, port)
+		if target.Endpoint() != addr {
+			t.Fatalf("DNS resolution started for target %q, want %q", target.Endpoint(), addr)
+		}
+		dnsR.UpdateState(resolver.State{
+			Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: addr}}}},
+		})
+	}
+}
+
+// Tests the case where the cluster resource requested is a leaf cluster. The
+// management server sends two updates for the same leaf cluster resource. The
+// test verifies that the load balancing configuration pushed to the priority LB
+// policy contains the expected discovery mechanism corresponding to the leaf
+// cluster, on both occasions.
+func (s) TestAggregateClusterSuccess_LeafNode(t *testing.T) {
+	tests := []struct {
+		name                  string
+		firstClusterResource  *v3clusterpb.Cluster
+		secondClusterResource *v3clusterpb.Cluster
+		wantFirstChildCfg     serviceconfig.LoadBalancingConfig
+		wantSecondChildCfg    serviceconfig.LoadBalancingConfig
+	}{
+		{
+			name:                  "eds",
+			firstClusterResource:  e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone),
+			secondClusterResource: e2e.DefaultCluster(clusterName, serviceName+"-new", e2e.SecurityLevelNone),
+			wantFirstChildCfg: &priority.LBConfig{
+				Children: map[string]*priority.Child{
+					"priority-0-0": {
+						Config:                     createPriorityConfig(clusterName),
+						IgnoreReresolutionRequests: true,
+					},
+				},
+				Priorities: []string{"priority-0-0"},
+			},
+			wantSecondChildCfg: &priority.LBConfig{
+				Children: map[string]*priority.Child{
+					"priority-1-0": {
+						Config:                     createPriorityConfig(clusterName),
+						IgnoreReresolutionRequests: true,
+					},
+				},
+				Priorities: []string{"priority-1-0"},
+			},
+		},
+		{
+			name:                  "dns",
+			firstClusterResource:  makeLogicalDNSClusterResource(clusterName, "dns_host", uint32(port)),
+			secondClusterResource: makeLogicalDNSClusterResource(clusterName, "dns_host_new", uint32(port)),
+			wantFirstChildCfg: &priority.LBConfig{
+				Children:   map[string]*priority.Child{"priority-0": {Config: createPriorityConfig(clusterName)}},
+				Priorities: []string{"priority-0"},
+			},
+			wantSecondChildCfg: &priority.LBConfig{
+				Children:   map[string]*priority.Child{"priority-1": {Config: createPriorityConfig(clusterName)}},
+				Priorities: []string{"priority-1"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+			mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+			// Push the first cluster resource through the management server and
+			// verify the configuration pushed to the child policy.
+			resources := e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+				Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+				Clusters:       []*v3clusterpb.Cluster{test.firstClusterResource},
+				Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+				SkipValidation: true,
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+			if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantFirstChildCfg); err != nil {
+				t.Fatal(err)
+			}
+
+			// Push the second cluster resource through the management server and
+			// verify the configuration pushed to the child policy.
+			resources.Clusters[0] = test.secondClusterResource
+			resources.Endpoints[0] = e2e.DefaultEndpoint(serviceName+"-new", host, []uint32{port})
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+			if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantSecondChildCfg); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// Tests the case where the cluster resource requested is an aggregate cluster
+// root pointing to two child clusters, one of type EDS and the other of type
+// LogicalDNS. The test verifies that load balancing configuration is pushed to
+// the priority LB policy only when all child clusters are resolved and it also
+// verifies that the pushed configuration contains the expected priority config.
+// The test then updates the aggregate cluster to point to two child clusters,
+// the same leaf cluster of type EDS and a different leaf cluster of type
+// LogicalDNS and verifies that the load balancing configuration pushed to the
+// priority LB policy contains the expected config.
+func (s) TestAggregateClusterSuccess_ThenUpdateChildClusters(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with the aggregate cluster resource
+	// pointing to two child clusters, one EDS and one LogicalDNS. Include the
+	// resource corresponding to the EDS cluster here, but don't include
+	// resource corresponding to the LogicalDNS cluster yet.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the LogicalDNS cluster in the management server. This
+	// should result in configuration being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	const dnsClusterNameNew = dnsClusterName + "-new"
+	const dnsHostNameNew = dnsHostName + "-new"
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterNameNew}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+			makeLogicalDNSClusterResource(dnsClusterNameNew, dnsHostNameNew, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostNameNew, dnsPort)
+
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-2": {Config: createPriorityConfig(dnsClusterNameNew)},
+		},
+		Priorities: []string{"priority-0-0", "priority-2"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where the cluster resource requested is an aggregate cluster
+// root pointing to two child clusters, one of type EDS and the other of type
+// LogicalDNS. The test verifies that the load balancing configuration pushed to
+// the priority LB policy contains the discovery mechanisms for both child
+// clusters. The test then updates the root cluster resource requested by the
+// cds LB policy to a leaf cluster of type EDS and verifies the load balancing
+// configuration pushed to the priority LB policy contains a single discovery
+// mechanism.
+func (s) TestAggregateClusterSuccess_ThenChangeRootToEDS(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with the aggregate cluster resource
+	// pointing to two child clusters.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	// Since the service name of the EDS cluster remains same, same priority name
+	// is used.
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where a requested cluster resource switches between being a
+// leaf and an aggregate cluster pointing to an EDS and LogicalDNS child
+// cluster. In each of these cases, the test verifies that the load balancing
+// configuration pushed to the priority LB policy contains the expected config.
+func (s) TestAggregatedClusterSuccess_SwitchBetweenLeafAndAggregate(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Start off with the requested cluster being a leaf EDS cluster.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch the requested cluster to be an aggregate cluster pointing to two
+	// child clusters.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch the cluster back to a leaf EDS cluster.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where an aggregate cluster exceeds the maximum depth,
+// which is 16. Verifies that the channel moves to TRANSIENT_FAILURE, and the
+// error is propagated to RPC callers. The test then modifies the graph to no
+// longer exceed maximum depth, but be at the maximum allowed depth, and
+// verifies that an RPC can be made successfully.
+func (s) TestAggregatedClusterFailure_ExceedsMaxStackDepth(t *testing.T) {
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}),
+			makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}),
+			makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}),
+			makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}),
+			makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}),
+			makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}),
+			makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}),
+			makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}),
+			makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}),
+			makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}),
+			makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}),
+			makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}),
+			makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}),
+			makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}),
+			makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}),
+			makeAggregateClusterResource(clusterName+"-15", []string{clusterName + "-16"}),
+			e2e.DefaultCluster(clusterName+"-16", serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	const wantErr = "aggregate cluster graph exceeds max depth"
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if code := status.Code(err); code != codes.Unavailable {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", code, codes.Unavailable)
+	}
+	if err != nil && !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want err containing: %v", err, wantErr)
+	}
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	// Update the aggregate cluster resource to no longer exceed max depth, and
+	// be at the maximum depth allowed.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}),
+			makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}),
+			makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}),
+			makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}),
+			makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}),
+			makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}),
+			makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}),
+			makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}),
+			makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}),
+			makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}),
+			makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}),
+			makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}),
+			makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}),
+			makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}),
+			makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}),
+			e2e.DefaultCluster(clusterName+"-15", serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests a diamond shaped aggregate cluster (A->[B,C]; B->D; C->D). Verifies
+// that the load balancing configuration pushed to the priority LB
+// policy specifies cluster D only once. Also verifies that configuration is
+// pushed only after all child clusters are resolved.
+func (s) TestAggregatedClusterSuccess_DiamondDependency(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with an aggregate cluster resource having
+	// a diamond dependency pattern, (A->[B,C]; B->D; C->D). Includes resources
+	// for cluster A, B and D, but don't include the resource for cluster C yet.
+	// This will help us verify that no configuration is pushed to the child
+	// policy until the whole cluster graph is resolved.
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+		clusterNameD = clusterName + "-D"
+	)
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameD}),
+			e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the resource for cluster C in the management server,
+	// thereby completing the cluster graph. This should result in configuration
+	// being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, makeAggregateClusterResource(clusterNameC, []string{clusterNameD}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameD),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where the aggregate cluster graph contains duplicates (A->[B,
+// C]; B->[C, D]). Verifies that the load balancing configuration pushed to the
+// priority LB policy does not contain duplicates, and that the priority
+// corresponding to cluster C is higher than that for cluster D. Also verifies
+// that the configuration is pushed only after all child clusters are resolved.
+func (s) TestAggregatedClusterSuccess_IgnoreDups(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with an aggregate cluster resource that
+	// has duplicates in the graph, (A->[B, C]; B->[C, D]). Include resources
+	// for clusters A, B and D, but don't configure the resource for cluster C
+	// yet. This will help us verify that no configuration is pushed to the
+	// child policy until the whole cluster graph is resolved.
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+		clusterNameD = clusterName + "-D"
+	)
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameC, clusterNameD}),
+			e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the resource for cluster C in the management server,
+	// thereby completing the cluster graph. This should result in configuration
+	// being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterNameC, edsClusterName, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{1234}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameC),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1-0": {
+				Config:                     createPriorityConfig(clusterNameD),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0", "priority-1-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph has a node that has
+// child node of itself. The case for this is A -> A, and since there is no base
+// cluster (EDS or Logical DNS), no configuration should be pushed to the child
+// policy.  The channel is expected to move to TRANSIENT_FAILURE and RPCs are
+// expected to fail with code UNAVAILABLE and an error message specifying that
+// the aggregate cluster graph has no leaf clusters.  Then the test updates A -> B,
+// where B is a leaf EDS cluster. Verifies that configuration is pushed to the
+// child policy and that an RPC can be successfully made.
+func (s) TestAggregatedCluster_NodeChildOfItself(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+	)
+	// Configure the management server with an aggregate cluster resource whose
+	// child is itself.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{makeAggregateClusterResource(clusterNameA, []string{clusterNameA})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Verify that the RPC fails with expected code.
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode)
+	}
+	const wantErr = "aggregate cluster graph has no leaf clusters"
+	if !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want error containing %s", err, wantErr)
+	}
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	// Update the aggregate cluster to point to a leaf EDS cluster.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			e2e.DefaultCluster(clusterNameB, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the configuration pushed to the child policy.
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameB),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph contains a cycle and
+// contains no leaf clusters. The case used here is [A -> B, B -> A]. As there
+// are no leaf clusters in this graph, no configuration should be pushed to the
+// child policy. The channel is expected to move to TRANSIENT_FAILURE and RPCs
+// are expected to fail with code UNAVAILABLE and an error message specifying
+// that the aggregate cluster graph has no leaf clusters.
+func (s) TestAggregatedCluster_CycleWithNoLeafNode(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+	)
+	// Configure the management server with an aggregate cluster resource graph
+	// that contains a cycle and no leaf clusters.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameA}),
+		},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Verify that the RPC fails with expected code.
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode)
+	}
+	const wantErr = "aggregate cluster graph has no leaf clusters"
+	if !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want %s", err, wantErr)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph contains a cycle and
+// also contains a leaf cluster. The case used here is [A -> B, B -> A, C]. As
+// there is a leaf cluster in this graph , configuration should be pushed to the
+// child policy and RPCs should get routed to that leaf cluster.
+func (s) TestAggregatedCluster_CycleWithLeafNode(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+	)
+	// Configure the management server with an aggregate cluster resource graph
+	// that contains a cycle, but also contains a leaf cluster.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameA, clusterNameC}),
+			e2e.DefaultCluster(clusterNameC, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the configuration pushed to the child policy.
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameC),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the scenario where the cluster tree changes, and verifies that the
+// watchers for the cds balancer are updated accordingly. That is the cluster
+// removed from the tree no longer has a watcher and the new cluster added has a
+// new watcher.
+func (s) TestWatchers(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cdsResourceRequestedCh := make(chan []string, 1)
+	onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+		if req.GetTypeUrl() == version.V3ClusterURL {
+			if len(req.GetResourceNames()) > 0 {
+				select {
+				case cdsResourceRequestedCh <- req.GetResourceNames():
+				case <-ctx.Done():
+				}
+			}
+		}
+		return nil
+	}
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, onStreamReq)
+
+	const (
+		clusterA = clusterName
+		clusterB = clusterName + "-B"
+		clusterC = clusterName + "-C"
+		clusterD = clusterName + "-D"
+	)
+
+	// Initial CDS resources: A -> B, C
+	initialResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterA)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterA, []string{clusterB, clusterC}),
+		},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, initialResources); err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+	wantNames := []string{clusterA, clusterB, clusterC}
+	if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the CDS resources to remove cluster C and add cluster D.
+	initialResources.Clusters = []*v3clusterpb.Cluster{
+		makeAggregateClusterResource(clusterA, []string{clusterB, clusterD}),
+	}
+	if err := mgmtServer.Update(ctx, initialResources); err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+	wantNames = []string{clusterA, clusterB, clusterD}
+	if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/balancer/cdsbalancer/cdsbalancer.go

@@ -0,0 +1,472 @@
+/*
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package cdsbalancer implements a balancer to handle CDS responses.
+package cdsbalancer
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/nop"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsdepmgr"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const cdsName = "cds_experimental"
+
+var (
+	// newChildBalancer is a helper function to build a new priority balancer
+	// and will be overridden in unittests.
+	newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
+		builder := balancer.Get(priority.Name)
+		if builder == nil {
+			return nil, fmt.Errorf("xds: no balancer builder with name %v", priority.Name)
+		}
+		// We directly pass the parent clientConn to the underlying priority
+		// balancer because the cdsBalancer does not deal with subConns.
+		return builder.Build(cc, opts), nil
+	}
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+// bb implements the balancer.Builder interface to help build a cdsBalancer.
+// It also implements the balancer.ConfigParser interface to help parse the
+// JSON service config, to be passed to the cdsBalancer.
+type bb struct{}
+
+// Build creates a new CDS balancer with the ClientConn.
+func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	builder := balancer.Get(priority.Name)
+	if builder == nil {
+		// Shouldn't happen, registered through imported Priority builder. Still,
+		// defensive programming.
+		logger.Errorf("%q LB policy is needed but not registered", priority.Name)
+		return nop.NewBalancer(cc, fmt.Errorf("%q LB policy is needed but not registered", priority.Name))
+	}
+	parser, ok := builder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported Priority builder has this method.
+		logger.Errorf("%q LB policy does not implement a config parser", priority.Name)
+		return nop.NewBalancer(cc, fmt.Errorf("%q LB policy does not implement a config parser", priority.Name))
+	}
+
+	b := &cdsBalancer{
+		bOpts:             opts,
+		childConfigParser: parser,
+		clusterConfigs:    make(map[string]*xdsresource.ClusterResult),
+		priorityConfigs:   make(map[string]*priorityConfig),
+		cc:                cc,
+	}
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	return b
+}
+
+// Name returns the name of balancers built by this builder.
+func (bb) Name() string {
+	return cdsName
+}
+
+// lbConfig represents the loadBalancingConfig section of the service config
+// for the cdsBalancer.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	ClusterName string `json:"cluster"`
+	IsDynamic   bool   `json:"isDynamic"`
+}
+
+// ParseConfig parses the JSON load balancer config provided into an
+// internal form or returns an error if the config is invalid.
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, fmt.Errorf("xds: unable to unmarshal lbconfig: %s, error: %v", string(c), err)
+	}
+	return &cfg, nil
+}
+
+// cdsBalancer implements a CDS based LB policy. It instantiates a
+// cluster_resolver balancer to further resolve the serviceName received from
+// CDS, into localities and endpoints. Implements the balancer.Balancer
+// interface which is exposed to gRPC and implements the balancer.ClientConn
+// interface which is exposed to the cluster_resolver balancer.
+type cdsBalancer struct {
+	// The following fields are initialized at build time and are either
+	// read-only after that or provide their own synchronization, and therefore
+	// do not need to be guarded by a mutex.
+	cc                balancer.ClientConn   // ClientConn interface passed to child LB.
+	bOpts             balancer.BuildOptions // BuildOptions passed to child LB.
+	childConfigParser balancer.ConfigParser // Config parser for cluster_resolver LB policy.
+	logger            *grpclog.PrefixLogger // Prefix logger for all logging.
+
+	// All fields below are accessed only from methods implementing the
+	// balancer.Balancer interface. Since gRPC guarantees that these methods are
+	// never invoked concurrently, no additional synchronization is required to
+	// protect access to these fields.
+	xdsClient         xdsclient.XDSClient
+	childLB           balancer.Balancer                     // Child policy, built upon resolution of the cluster graph.
+	clusterConfigs    map[string]*xdsresource.ClusterResult // Cluster name to the last received result for that cluster.
+	priorityConfigs   map[string]*priorityConfig            // Hostname to priority config for that leaf cluster.
+	lbCfg             *lbConfig                             // Current load balancing configuration.
+	priorities        []*priorityConfig                     // List of priorities in the order.
+	unsubscribe       func()                                // For dynamic cluster unsubscription.
+	isSubscribed      bool                                  // True if a dynamic cluster has been subscribed to.
+	clusterSubscriber xdsdepmgr.ClusterSubscriber           // To subscribe to dynamic cluster resource.
+	xdsLBPolicy       internalserviceconfig.BalancerConfig  // Stores the locality and endpoint picking policy.
+	attributes        *attributes.Attributes                // Attributes from resolver state.
+	serviceConfig     *serviceconfig.ParseResult
+	// Each new leaf cluster needs a child name generator to reuse child policy
+	// names. But to make sure the names across leaf clusters doesn't conflict,
+	// we need a seq ID. This ID is incremented for each new cluster.
+	childNameGeneratorSeqID uint64
+}
+
+// UpdateClientConnState receives the serviceConfig, xdsConfig,
+// ClusterSubscriber and the xdsClient object from the xdsResolver. If an error
+// is encountered, the parent (clustermanager) sets the corresponding cluster’s
+// picker to transient_failure. Otherwise, the received configuration is
+// processed and forwarded to the appropriate child policy.
+func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error {
+	if b.xdsClient == nil {
+		c := xdsclient.FromResolverState(state.ResolverState)
+		if c == nil {
+			b.logger.Warningf("Received balancer config with no xDS client")
+			return balancer.ErrBadResolverState
+		}
+		b.xdsClient = c
+	}
+	b.logger.Infof("Received balancer config update: %s", pretty.ToJSON(state.BalancerConfig))
+
+	xdsConfig := xdsresource.XDSConfigFromResolverState(state.ResolverState)
+	if xdsConfig == nil {
+		b.logger.Warningf("Received balancer config with no xDS config")
+		return balancer.ErrBadResolverState
+	}
+	b.clusterConfigs = xdsConfig.Clusters
+	b.clusterSubscriber = xdsdepmgr.XDSClusterSubscriberFromResolverState(state.ResolverState)
+	if b.clusterSubscriber == nil {
+		b.logger.Warningf("Received balancer config with no cluster subscriber")
+		return balancer.ErrBadResolverState
+	}
+	// The errors checked here should ideally never happen because the
+	// ServiceConfig in this case is prepared by the xdsResolver and is not
+	// something that is received on the wire.
+	lbCfg, ok := state.BalancerConfig.(*lbConfig)
+	if !ok {
+		b.logger.Warningf("Received unexpected balancer config type: %T", state.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+	if lbCfg.ClusterName == "" {
+		b.logger.Warningf("Received balancer config with no cluster name")
+		return balancer.ErrBadResolverState
+	}
+
+	b.lbCfg = lbCfg
+	b.serviceConfig = state.ResolverState.ServiceConfig
+	b.attributes = state.ResolverState.Attributes
+	return b.handleXDSConfigUpdate()
+}
+
+// handleXDSConfigUpdate processes the XDSConfig update from the xDS resolver.
+func (b *cdsBalancer) handleXDSConfigUpdate() error {
+	clusterName := b.lbCfg.ClusterName
+
+	// If the cluster is dynamic and we dont have a subscription yet, create
+	// one.
+	if b.lbCfg.IsDynamic && !b.isSubscribed {
+		b.unsubscribe = b.clusterSubscriber.SubscribeToCluster(clusterName)
+		b.isSubscribed = true
+		return nil
+	}
+
+	clusterUpdate, ok := b.clusterConfigs[clusterName]
+	if !ok {
+		// If the cluster is missing from the config, check if it is dynamic.
+		// For dynamic clusters, the xDS config may be updated before the
+		// corresponding cluster resource is received. This should never occur
+		// for static clusters.
+		if b.lbCfg.IsDynamic {
+			return nil
+		}
+		return b.annotateErrorWithNodeID(fmt.Errorf("did not find the cluster %q in XDSConfig", clusterName))
+	}
+	// If the cluster resource has an error, return the error.
+	if clusterUpdate.Err != nil {
+		return clusterUpdate.Err
+	}
+	return b.handleClusterUpdate()
+}
+
+// handleClusterUpdate handles a good XDSConfig update from the xDS resolver.
+// Builds the child policy config and pushes it down.
+func (b *cdsBalancer) handleClusterUpdate() error {
+	clusterName := b.lbCfg.ClusterName
+	clusterConfig := b.clusterConfigs[clusterName].Config
+
+	var newPriorities []*priorityConfig
+	switch clusterConfig.Cluster.ClusterType {
+	case xdsresource.ClusterTypeEDS, xdsresource.ClusterTypeLogicalDNS:
+		p := b.updatePriorityConfig(clusterName, &clusterConfig)
+		newPriorities = append(newPriorities, p)
+	case xdsresource.ClusterTypeAggregate:
+		for _, leaf := range clusterConfig.AggregateConfig.LeafClusters {
+			leafCluster := b.clusterConfigs[leaf]
+			// Update priority config for leaf clusters.
+			p := b.updatePriorityConfig(leaf, &leafCluster.Config)
+			newPriorities = append(newPriorities, p)
+		}
+	}
+	b.priorities = newPriorities
+
+	if err := b.updateOutlierDetection(); err != nil {
+		return b.annotateErrorWithNodeID(fmt.Errorf("failed to correctly update Outlier Detection config %v", err))
+	}
+
+	// The LB policy is configured by the root cluster.
+	if err := json.Unmarshal(clusterConfig.Cluster.LBPolicy, &b.xdsLBPolicy); err != nil {
+		return b.annotateErrorWithNodeID(fmt.Errorf("error unmarshalling xDS LB Policy: %v", err))
+	}
+	if err := b.updateChildConfig(); err != nil {
+		return b.annotateErrorWithNodeID(err)
+	}
+	return nil
+}
+
+// updateChildConfig builds child policy configuration using endpoint addresses
+// returned from the XDSConfig and child policy configuration.
+//
+// A child policy is created if one doesn't already exist. The newly built
+// configuration is then pushed to the child policy.
+func (b *cdsBalancer) updateChildConfig() error {
+	if b.childLB == nil {
+		childLB, err := newChildBalancer(b.cc, b.bOpts)
+		if err != nil {
+			return fmt.Errorf("failed to create child policy of type %s: %v", priority.Name, err)
+		}
+		b.childLB = childLB
+	}
+
+	childCfgBytes, endpoints, err := buildPriorityConfigJSON(b.priorities, &b.xdsLBPolicy)
+	if err != nil {
+		return fmt.Errorf("failed to build child policy config: %v", err)
+	}
+	childCfg, err := b.childConfigParser.ParseConfig(childCfgBytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse child policy config. This should never happen because the config was generated: %v", err)
+	}
+	if b.logger.V(2) {
+		b.logger.Infof("Built child policy config: %s", pretty.ToJSON(childCfg))
+	}
+
+	for i := range endpoints {
+		for j := range endpoints[i].Addresses {
+			addr := endpoints[i].Addresses[j]
+			addr.BalancerAttributes = endpoints[i].Attributes
+			// BalancerAttributes are used for the following:
+			// * Authority Override.
+			// * grpc.lb.backend_service metric label propagation.
+			// See https://github.com/grpc/grpc-go/issues/6472
+			endpoints[i].Addresses[j] = addr
+		}
+	}
+	if err := b.childLB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Endpoints:     endpoints,
+			ServiceConfig: b.serviceConfig,
+			Attributes:    b.attributes,
+		},
+		BalancerConfig: childCfg,
+	}); err != nil {
+		return fmt.Errorf("failed to push config to child policy: %v", err)
+	}
+	return nil
+}
+
+// updatePriorityConfig updates the priority configuration for the specified EDS
+// or DNS cluster, creating it if it does not already exist.
+func (b *cdsBalancer) updatePriorityConfig(clusterName string, clusterConfig *xdsresource.ClusterConfig) *priorityConfig {
+	name := hostName(clusterName, *clusterConfig.Cluster)
+	pc, ok := b.priorityConfigs[name]
+	if !ok {
+		pc = &priorityConfig{
+			childNameGen: newNameGenerator(b.childNameGeneratorSeqID),
+		}
+		b.priorityConfigs[name] = pc
+		// Increment the seq ID for the next new cluster. This is done to make
+		// sure that the child policy names generated for different clusters
+		// don't conflict with each other.
+		b.childNameGeneratorSeqID++
+	}
+	pc.clusterConfig = clusterConfig
+	return pc
+}
+
+// updateOutlierDetection updates Outlier Detection config for all priorities.
+func (b *cdsBalancer) updateOutlierDetection() error {
+	odBuilder := balancer.Get(outlierdetection.Name)
+	if odBuilder == nil {
+		// Shouldn't happen, registered through imported Outlier Detection,
+		// defensive programming.
+		return fmt.Errorf("%q LB policy is needed but not registered", outlierdetection.Name)
+	}
+
+	odParser, ok := odBuilder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported Outlier Detection builder has this method.
+		return fmt.Errorf("%q LB policy does not implement a config parser", outlierdetection.Name)
+	}
+
+	for _, p := range b.priorities {
+		// Update Outlier Detection Config.
+		odJSON := p.clusterConfig.Cluster.OutlierDetection
+		if odJSON == nil {
+			odJSON = json.RawMessage(`{}`)
+		}
+
+		lbCfg, err := odParser.ParseConfig(odJSON)
+		if err != nil {
+			return fmt.Errorf("error parsing Outlier Detection config %v: %v", odJSON, err)
+		}
+
+		odCfg, ok := lbCfg.(*outlierdetection.LBConfig)
+		if !ok {
+			// Shouldn't happen, Parser built at build time with Outlier
+			// Detection builder pulled from gRPC LB Registry.
+			return fmt.Errorf("config parser for Outlier Detection returned config with unexpected type %T: %v", lbCfg, lbCfg)
+		}
+		p.outlierDetection = *odCfg
+	}
+	return nil
+}
+
+// ResolverError handles errors reported by the xdsResolver.
+func (b *cdsBalancer) ResolverError(err error) {
+	// Missing Listener or RouteConfiguration on the management server
+	// results in a 'resource not found' error from the xDS resolver. In
+	// these cases, we should report transient failure.
+	if xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound {
+		b.closeChildPolicyAndReportTF(err)
+		return
+	}
+	var root string
+	if b.lbCfg != nil {
+		root = b.lbCfg.ClusterName
+	}
+	b.onClusterError(root, err)
+}
+
+// UpdateSubConnState handles subConn updates from gRPC.
+func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+// closeChildPolicyAndReportTF closes the child policy, if it exists, and
+// updates the connectivity state of the channel to TransientFailure with an
+// error picker.
+func (b *cdsBalancer) closeChildPolicyAndReportTF(err error) {
+	if b.childLB != nil {
+		b.childLB.Close()
+		b.childLB = nil
+	}
+	b.cc.UpdateState(balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	})
+}
+
+// Close closes the child policy, unsubscribes to the dynamic cluster, and
+// closes the cdsBalancer.
+func (b *cdsBalancer) Close() {
+	if b.childLB != nil {
+		b.childLB.Close()
+		b.childLB = nil
+	}
+	if b.unsubscribe != nil {
+		b.unsubscribe()
+	}
+	b.logger.Infof("Shutdown")
+}
+
+func (b *cdsBalancer) ExitIdle() {
+	if b.childLB == nil {
+		b.logger.Warningf("Received ExitIdle with no child policy")
+		return
+	}
+	// This implementation assumes the child balancer supports
+	// ExitIdle (but still checks for the interface's existence to
+	// avoid a panic if not). If the child does not, no subconns
+	// will be connected.
+	b.childLB.ExitIdle()
+}
+
+// Node ID needs to be manually added to errors generated in the following
+// scenarios:
+//   - resource-does-not-exist: since the xDS watch API uses a separate callback
+//     instead of returning an error value. TODO(gRFC A88): Once A88 is
+//     implemented, the xDS client will be able to add the node ID to
+//     resource-does-not-exist errors as well, and we can get rid of this
+//     special handling.
+//   - received a good update from the xDS client, but the update either contains
+//     an invalid security configuration or contains invalid aggragate cluster
+//     config.
+func (b *cdsBalancer) annotateErrorWithNodeID(err error) error {
+	nodeID := b.xdsClient.BootstrapConfig().Node().GetId()
+	return fmt.Errorf("[xDS node id: %v]: %w", nodeID, err)
+}
+
+// onClusterAmbientError handles an ambient error, if a childLB already has a
+// good update, it should continue using that.
+func (b *cdsBalancer) onClusterAmbientError(name string, err error) {
+	b.logger.Warningf("Cluster resource %q received ambient error update: %v", name, err)
+
+	if xdsresource.ErrType(err) != xdsresource.ErrorTypeConnection && b.childLB != nil {
+		// Connection errors will be sent to the child balancers directly.
+		// There's no need to forward them.
+		b.childLB.ResolverError(err)
+	}
+}
+
+// onClusterResourceError handles errors to stop using the previously seen
+// resource. Propagates the error down to the child policy if one exists, and
+// puts the channel in TRANSIENT_FAILURE.
+func (b *cdsBalancer) onClusterResourceError(name string, err error) {
+	b.logger.Warningf("CDS watch for resource %q reported resource error", name)
+	b.closeChildPolicyAndReportTF(err)
+}
+
+func (b *cdsBalancer) onClusterError(name string, err error) {
+	if b.childLB != nil {
+		b.onClusterAmbientError(name, err)
+	} else {
+		b.onClusterResourceError(name, err)
+	}
+}

internal/xds/balancer/cdsbalancer/configbuilder.go

@@ -0,0 +1,375 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cdsbalancer
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"slices"
+
+	"google.golang.org/grpc/experimental/balancer/hostname"
+	"google.golang.org/grpc/experimental/balancer/weight"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/proxyattributes"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+)
+
+const million = 1000000
+
+// priorityConfig is config for one priority. For example, if there's an EDS and
+// a DNS, the priority list will be [priorityConfig{EDS}, priorityConfig{DNS}].
+//
+// Each priorityConfig corresponds to one leaf cluster retrieved from XDSConfig
+// for the top-level cluster.
+type priorityConfig struct {
+	// clusterConfig has the cluster update as well as EDS or DNS endpoints
+	// depending on the leaf cluster type.
+	clusterConfig *xdsresource.ClusterConfig
+	// outlierDetection is the Outlier Detection LB configuration for this
+	// priority.
+	outlierDetection outlierdetection.LBConfig
+	// Each leaf cluster has a name generator so that the child policies can
+	// reuse names between updates (EDS updates for example).
+	childNameGen *nameGenerator
+}
+
+// hostName returns the name of the host for the given cluster.
+//
+// For EDS, it's the EDSServiceName (or ClusterName if empty).
+// For DNS, it's the DNSHostName.
+func hostName(clusterName string, update xdsresource.ClusterUpdate) string {
+	switch update.ClusterType {
+	case xdsresource.ClusterTypeEDS:
+		if update.EDSServiceName != "" {
+			return update.EDSServiceName
+		}
+		return clusterName
+	case xdsresource.ClusterTypeLogicalDNS:
+		return update.DNSHostName
+	default:
+		return ""
+	}
+}
+
+// buildPriorityConfigJSON builds balancer config for the passed in
+// priorities.
+//
+// The built tree of balancers (see test for the output struct).
+//
+//	          ┌────────┐
+//	          │priority│
+//	          └┬──────┬┘
+//	           │      │
+//	┌──────────▼─┐  ┌─▼──────────┐
+//	│cluster_impl│  │cluster_impl│
+//	└──────┬─────┘  └─────┬──────┘
+//	       │              │
+//	┌──────▼─────┐  ┌─────▼──────┐
+//	│xDSLBPolicy │  │xDSLBPolicy │ (Locality and Endpoint picking layer)
+//	└────────────┘  └────────────┘
+func buildPriorityConfigJSON(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]byte, []resolver.Endpoint, error) {
+	pc, endpoints, err := buildPriorityConfig(priorities, xdsLBPolicy)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to build priority config: %v", err)
+	}
+	ret, err := json.Marshal(pc)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to marshal built priority config struct into json: %v", err)
+	}
+	return ret, endpoints, nil
+}
+
+func buildPriorityConfig(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*priority.LBConfig, []resolver.Endpoint, error) {
+	var (
+		retConfig    = &priority.LBConfig{Children: make(map[string]*priority.Child)}
+		retEndpoints []resolver.Endpoint
+	)
+	for _, p := range priorities {
+		clusterUpdate := p.clusterConfig.Cluster
+		switch clusterUpdate.ClusterType {
+		case xdsresource.ClusterTypeEDS:
+			names, configs, endpoints, err := buildClusterImplConfigForEDS(p.childNameGen, p.clusterConfig, xdsLBPolicy)
+			if err != nil {
+				return nil, nil, err
+			}
+			retConfig.Priorities = append(retConfig.Priorities, names...)
+			retEndpoints = append(retEndpoints, endpoints...)
+			odCfgs := convertClusterImplMapToOutlierDetection(configs, p.outlierDetection)
+			for n, c := range odCfgs {
+				retConfig.Children[n] = &priority.Child{
+					Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: c},
+					// Ignore all re-resolution from EDS children.
+					IgnoreReresolutionRequests: true,
+				}
+			}
+			continue
+		case xdsresource.ClusterTypeLogicalDNS:
+			name, config, endpoints := buildClusterImplConfigForDNS(p.childNameGen, p.clusterConfig, xdsLBPolicy)
+			retConfig.Priorities = append(retConfig.Priorities, name)
+			retEndpoints = append(retEndpoints, endpoints...)
+			odCfg := makeClusterImplOutlierDetectionChild(config, p.outlierDetection)
+			retConfig.Children[name] = &priority.Child{
+				Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: odCfg},
+				// Not ignore re-resolution from DNS children, they will trigger
+				// DNS to re-resolve.
+				IgnoreReresolutionRequests: false,
+			}
+			continue
+		}
+	}
+	return retConfig, retEndpoints, nil
+}
+
+func convertClusterImplMapToOutlierDetection(ciCfgs map[string]*clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) map[string]*outlierdetection.LBConfig {
+	odCfgs := make(map[string]*outlierdetection.LBConfig, len(ciCfgs))
+	for n, c := range ciCfgs {
+		odCfgs[n] = makeClusterImplOutlierDetectionChild(c, odCfg)
+	}
+	return odCfgs
+}
+
+func makeClusterImplOutlierDetectionChild(ciCfg *clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) *outlierdetection.LBConfig {
+	odCfgRet := odCfg
+	odCfgRet.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: ciCfg}
+	return &odCfgRet
+}
+
+func buildClusterImplConfigForDNS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (string, *clusterimpl.LBConfig, []resolver.Endpoint) {
+	pName := fmt.Sprintf("priority-%v", g.prefix)
+	clusterUpdate := config.Cluster
+	lbconfig := &clusterimpl.LBConfig{
+		Cluster:     clusterUpdate.ClusterName,
+		ChildPolicy: xdsLBPolicy,
+	}
+	endpoints := config.EndpointConfig.DNSEndpoints.Endpoints
+	if len(endpoints) == 0 {
+		return pName, lbconfig, nil
+	}
+	var retEndpoint resolver.Endpoint
+	for _, e := range endpoints {
+		// LOGICAL_DNS requires all resolved addresses to be grouped into a
+		// single logical endpoint. We iterate over the input endpoints and
+		// aggregate their addresses into a new endpoint variable.
+		retEndpoint.Addresses = append(retEndpoint.Addresses, e.Addresses...)
+	}
+	// Even though localities are not a thing for the LOGICAL_DNS cluster and
+	// its endpoint(s), we add an empty locality attribute here to ensure that
+	// LB policies that rely on locality information (like weighted_target)
+	// continue to work.
+	localityStr := xdsinternal.LocalityString(clients.Locality{})
+	retEndpoint = hostname.Set(hierarchy.SetInEndpoint(retEndpoint, []string{pName, localityStr}), clusterUpdate.DNSHostName)
+	// Set the locality weight to 1. This is required because the child policy
+	// like weighted_target which relies on locality weights to distribute
+	// traffic. These policies may drop traffic if the weight is 0.
+	retEndpoint = wrrlocality.SetAddrInfo(retEndpoint, wrrlocality.AddrInfo{LocalityWeight: 1})
+	return pName, lbconfig, []resolver.Endpoint{retEndpoint}
+}
+
+// buildClusterImplConfigForEDS returns a list of cluster_impl configs, one for
+// each priority, sorted by priority, and the addresses for each priority (with
+// hierarchy attributes set).
+//
+// For example, if there are two priorities, the returned values will be
+// - ["p0", "p1"]
+// - map{"p0":p0_config, "p1":p1_config}
+// - [p0_address_0, p0_address_1, p1_address_0, p1_address_1]
+//   - p0 addresses' hierarchy attributes are set to p0
+func buildClusterImplConfigForEDS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]string, map[string]*clusterimpl.LBConfig, []resolver.Endpoint, error) {
+	edsUpdate := config.EndpointConfig.EDSUpdate
+
+	// Localities of length 0 is triggered by an NACK or resource-not-found
+	// error before update, or an empty localities list in an update. In either
+	// case want to create a priority, and send down empty address list, causing
+	// TF for that priority. "If any discovery mechanism instance experiences an
+	// error retrieving data, and it has not previously reported any results, it
+	// should report a result that is a single priority with no endpoints." -
+	// A37
+	priorities := [][]xdsresource.Locality{{}}
+	if len(edsUpdate.Localities) != 0 {
+		priorities = groupLocalitiesByPriority(edsUpdate.Localities)
+	}
+	retNames := g.generate(priorities)
+	retConfigs := make(map[string]*clusterimpl.LBConfig, len(retNames))
+	var retEndpoints []resolver.Endpoint
+	for i, pName := range retNames {
+		priorityLocalities := priorities[i]
+		cfg, endpoints, err := priorityLocalitiesToClusterImpl(priorityLocalities, pName, *config.Cluster, xdsLBPolicy)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		retConfigs[pName] = cfg
+		retEndpoints = append(retEndpoints, endpoints...)
+	}
+	return retNames, retConfigs, retEndpoints, nil
+}
+
+// groupLocalitiesByPriority returns the localities grouped by priority.
+//
+// The returned list is sorted from higher priority to lower. Each item in the
+// list is a group of localities.
+//
+// For example, for L0-p0, L1-p0, L2-p1, results will be
+// - [[L0, L1], [L2]]
+func groupLocalitiesByPriority(localities []xdsresource.Locality) [][]xdsresource.Locality {
+	priorities := make(map[int][]xdsresource.Locality)
+	for _, locality := range localities {
+		priority := int(locality.Priority)
+		priorities[priority] = append(priorities[priority], locality)
+	}
+	// Sort the priorities based on the int value, deduplicate, and then turn
+	// the sorted list into a string list. This will be child names, in priority
+	// order.
+	priorityIntSlice := slices.Sorted(maps.Keys(priorities))
+	ret := make([][]xdsresource.Locality, 0, len(priorityIntSlice))
+	for _, p := range priorityIntSlice {
+		ret = append(ret, priorities[p])
+	}
+	return ret
+}
+
+// priorityLocalitiesToClusterImpl takes a list of localities (with the same
+// priority), and generates a cluster impl policy config, and a list of
+// addresses with their path hierarchy set to [priority-name, locality-name], so
+// priority and the xDS LB Policy know which child policy each address is for.
+func priorityLocalitiesToClusterImpl(localities []xdsresource.Locality, priorityName string, clusterUpdate xdsresource.ClusterUpdate, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*clusterimpl.LBConfig, []resolver.Endpoint, error) {
+	var retEndpoints []resolver.Endpoint
+
+	// Compute the sum of locality weights to normalize locality weights. The
+	// xDS client guarantees that the sum of locality weights (within a
+	// priority) will not exceed MaxUint32.
+	var localityWeightSum uint32
+	for _, locality := range localities {
+		localityWeightSum += locality.Weight
+	}
+
+	for _, locality := range localities {
+		// Compute the sum of endpoint weights to normalize endpoint weights.
+		// The xDS client does not currently guarantee that the sum of endpoint
+		// weights (within a locality) will not exceed MaxUint32. TODO(i/8862):
+		// Once the xDS client guarantees that the sum of endpoint weights does
+		// not exceed MaxUint32, we can change the type of this variable from
+		// uint64 to uint32.
+		var endpointWeightSum uint64
+		for _, endpoint := range locality.Endpoints {
+			endpointWeightSum += uint64(endpoint.Weight)
+		}
+
+		localityStr := xdsinternal.LocalityString(locality.ID)
+		for _, endpoint := range locality.Endpoints {
+			// Filter out all "unhealthy" endpoints (unknown and healthy are
+			// both considered to be healthy:
+			// https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus).
+			if endpoint.HealthStatus != xdsresource.EndpointHealthStatusHealthy && endpoint.HealthStatus != xdsresource.EndpointHealthStatusUnknown {
+				continue
+			}
+
+			// Create a copy of endpoint.ResolverEndpoint to avoid race between
+			// the xDS Client (which owns this shared object in its cache) and
+			// the Cluster Resolver (which is trying to modify attributes).
+			resolverEndpoint := endpoint.ResolverEndpoint
+			resolverEndpoint.Addresses = slices.Clone(endpoint.ResolverEndpoint.Addresses)
+
+			if clusterUpdate.IsHTTP11ProxyEnabled {
+				var proxyAddrStr string
+				if val, ok := endpoint.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok {
+					proxyAddrStr = val.Address
+				} else if val, ok := locality.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok {
+					proxyAddrStr = val.Address
+				}
+				if proxyAddrStr != "" {
+					for idx, addr := range resolverEndpoint.Addresses {
+						connectAddr := addr.Addr
+						addr.Addr = proxyAddrStr
+						resolverEndpoint.Addresses[idx] = proxyattributes.Set(addr, proxyattributes.Options{
+							ConnectAddr: connectAddr,
+						})
+					}
+				}
+			}
+
+			resolverEndpoint = hierarchy.SetInEndpoint(resolverEndpoint, []string{priorityName, localityStr})
+			resolverEndpoint = xdsinternal.SetLocalityIDInEndpoint(resolverEndpoint, locality.ID)
+			// "To provide the xds_wrr_locality load balancer information about
+			// locality weights received from EDS, the cluster resolver will
+			// populate a new locality weight attribute for each address The
+			// attribute will have the weight (as an integer) of the locality
+			// the address is part of." - A52
+			resolverEndpoint = wrrlocality.SetAddrInfo(resolverEndpoint, wrrlocality.AddrInfo{LocalityWeight: locality.Weight})
+
+			if envconfig.PickFirstWeightedShuffling {
+				normalizedLocalityWeight := fractionToFixedPoint(uint64(locality.Weight), uint64(localityWeightSum))
+				normalizedEndpointWeight := fractionToFixedPoint(uint64(endpoint.Weight), endpointWeightSum)
+				endpointWeight := fixedPointMultiply(normalizedEndpointWeight, normalizedLocalityWeight)
+				if endpointWeight == 0 {
+					endpointWeight = 1
+				}
+				resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: endpointWeight})
+			} else {
+				resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: locality.Weight * endpoint.Weight})
+			}
+			retEndpoints = append(retEndpoints, resolverEndpoint)
+		}
+	}
+	return &clusterimpl.LBConfig{
+		Cluster:     clusterUpdate.ClusterName,
+		ChildPolicy: xdsLBPolicy,
+	}, retEndpoints, nil
+}
+
+// fixedPointFractionalBits is the number of bits used for the fractional part
+// of normalized endpoint and locality weights.
+//
+// We use the UQ1.31 fixed-point format (Unsigned, 1 integer bit, 31 fractional bits).
+// This allows representing values in the range [0.0, 2.0) with a precision
+// of 2^-31.
+//
+// Bit Layout:
+// [ 31 ] [ 30 ................. 0 ]
+//
+//	|              |
+//	|              +--- Fractional Part (31 bits)
+//	+------------------ Integer Part (1 bit)
+//
+// See gRFC A113 for more details.
+const fixedPointFractionalBits = 31
+
+// fractionToFixedPoint converts a fraction represented by numerator and
+// denominator to a fixed-point number between 0 and 1 represented as a uint32.
+//
+// The xDS client guarantees that the sum of locality weights (within a
+// priority) will not exceed MaxUint32. TODO(i/8862): Once the xDS client
+// guarantees that the sum of endpoint weights does not exceed MaxUint32, we can
+// change the types of this function's arguments from uint64 to uint32.
+func fractionToFixedPoint(numerator, denominator uint64) uint32 {
+	return uint32(uint64(numerator) << fixedPointFractionalBits / uint64(denominator))
+}
+
+func fixedPointMultiply(a, b uint32) uint32 {
+	return uint32((uint64(a) * uint64(b)) >> fixedPointFractionalBits)
+}

internal/xds/balancer/cdsbalancer/configbuilder_childname.go

@@ -0,0 +1,111 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+)
+
+// nameGenerator generates a child name for a list of priorities (each priority
+// is a list of localities).
+//
+// The purpose of this generator is to reuse names between updates. So the
+// struct keeps state between generate() calls, and a later generate() might
+// return names returned by the previous call.
+type nameGenerator struct {
+	prevLocalitiesToChildNames map[clients.Locality]string // locality to child name mapping generated for the previous update
+	prevChildNames             []string                    // prioritized list of child names generated for the previous update
+	prefix                     uint64
+	nextID                     uint64
+}
+
+func newNameGenerator(prefix uint64) *nameGenerator {
+	return &nameGenerator{prefix: prefix}
+}
+
+// generate returns a list of names for the given list of priorities.
+//
+// Each priority is a list of localities. The name for the priority is picked as
+// - for each locality in this priority, if it exists in the existing names,
+// this priority will reuse the name
+// - if no reusable name is found for this priority, a new name is generated
+//
+// For example:
+// - update 1: [[L1], [L2], [L3]] --> ["0", "1", "2"]
+// - update 2: [[L1], [L2], [L3]] --> ["0", "1", "2"]
+// - update 3: [[L1, L2], [L3]] --> ["0", "2"]   (Two priorities were merged)
+// - update 4: [[L1], [L4]] --> ["0", "3",]      (A priority was split, and a new priority was added)
+func (ng *nameGenerator) generate(priorities [][]xdsresource.Locality) []string {
+	ret := make([]string, len(priorities))
+	usedNames := make(map[string]bool)
+	newNames := make(map[clients.Locality]string)
+
+	// Pass 1: Same priority index match.
+	for i, priority := range priorities {
+		if i >= len(ng.prevChildNames) {
+			continue
+		}
+		targetName := ng.prevChildNames[i]
+		for _, locality := range priority {
+			if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok && name == targetName {
+				ret[i] = targetName
+				usedNames[targetName] = true
+				break
+			}
+		}
+	}
+
+	// Pass 2: Greedy reuse.
+	for i, priority := range priorities {
+		if ret[i] != "" {
+			continue
+		}
+		for _, locality := range priority {
+			if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok {
+				if !usedNames[name] {
+					ret[i] = name
+					usedNames[name] = true
+					break
+				}
+			}
+		}
+	}
+
+	// Pass 3: New name.
+	for i, name := range ret {
+		if name == "" {
+			newID := fmt.Sprintf("priority-%d-%d", ng.prefix, ng.nextID)
+			ng.nextID++
+			ret[i] = newID
+			usedNames[newID] = true
+		}
+	}
+
+	// Update state.
+	for i, priority := range priorities {
+		for _, l := range priority {
+			newNames[l.ID] = ret[i]
+		}
+	}
+	ng.prevLocalitiesToChildNames = newNames
+	ng.prevChildNames = ret
+	return ret
+}

internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go

@@ -0,0 +1,311 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+)
+
+func (s) Test_nameGenerator_generate(t *testing.T) {
+	tests := []struct {
+		name   string
+		prefix uint64
+		steps  []struct {
+			input [][]xdsresource.Locality
+			want  []string
+		}
+	}{
+		{
+			name:   "init, two new priorities",
+			prefix: 3,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-3-0", "priority-3-1"},
+				},
+			},
+		},
+		{
+			name:   "one new priority",
+			prefix: 1,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+					},
+					want: []string{"priority-1-0"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-1-0", "priority-1-1"},
+				},
+			},
+		},
+		{
+			name:   "merge two priorities",
+			prefix: 4,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-4-0", "priority-4-1", "priority-4-2"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-4-0", "priority-4-2"},
+				},
+			},
+		},
+		{
+			name: "swap two priorities",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1", "priority-0-2"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-1", "priority-0-0", "priority-0-2"},
+				},
+			},
+		},
+		{
+			name: "split priority",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-2", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "priority index preference",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L0"}}},
+					},
+					want: []string{"priority-0-0"},
+				},
+			},
+		},
+		{
+			name: "merge partial",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "swap shift",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-1", "priority-0-0"},
+				},
+			},
+		},
+		{
+			name: "replace priority",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L5"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-2"},
+				},
+			},
+		},
+		{
+			name: "reordered merge",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "three-step shift stability",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-2", "priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ng := newNameGenerator(tt.prefix)
+			for i, step := range tt.steps {
+				got := ng.generate(step.input)
+				if diff := cmp.Diff(got, step.want); diff != "" {
+					t.Errorf("step %d: generate() = got: %v, want: %v, diff (-got +want): %s", i, got, step.want, diff)
+				}
+			}
+		})
+	}
+}

internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go

@@ -0,0 +1,420 @@
+/*
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the "cds_experimental" LB policy.
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router"    // Register the router filter
+	_ "google.golang.org/grpc/internal/xds/resolver"             // Register the xds resolver
+)
+
+// setupAndDial performs common setup across all tests
+//
+//   - creates an xDS client with the passed in bootstrap contents
+//   - creates a  xds resolver to be used
+//   - creates a ClientConn to talk to the test backends
+//
+// Returns a function to close the ClientConn and the xDS client.
+func setupAndDial(t *testing.T, bootstrapContents []byte) (*grpc.ClientConn, func()) {
+	t.Helper()
+	// Create an xDS resolver with the above bootstrap configuration.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("xDS resolver creation failed: %v", err)
+	}
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	return cc, func() {
+		cc.Close()
+	}
+}
+
+// TestErrorFromParentLB_ConnectionError tests the case where a connection error
+// is sent by the CDS LB Policy. The CDS LB policy sends a connection error when
+// the ADS stream to the management server breaks. The test verifies that there
+// is no perceivable effect because of this connection error, and that RPCs
+// continue to work (because the LB policies are expected to use previously
+// received xDS resources).
+func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) {
+	// Create a listener to be used by the management server. The test will
+	// close this listener to simulate ADS stream breakage.
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	// Start an xDS management server with the above restartable listener, and
+	// push a channel when the stream is closed.
+	streamClosedCh := make(chan struct{}, 1)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			select {
+			case streamClosedCh <- struct{}{}:
+			default:
+			}
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		NodeID:     nodeID,
+		DialTarget: serviceName,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	cc, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Close the listener and ensure that the ADS stream breaks.
+	lis.Close()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to close")
+	default:
+	}
+
+	// Ensure that RPCs continue to succeed for the next second.
+	for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+	}
+}
+
+// TestErrorFromParentLB_ResourceNotFound tests the case where a
+// resource-not-found error is received by the CDS LB policy for a cluster. The
+// resource-not-found error is received when the cluster resource associated
+// with these LB policies is removed by the management server. The test verifies
+// that the associated EDS is canceled and RPCs fail. It also ensures that when
+// the Cluster resource is added back, the EDS resource is re-requested and RPCs
+// being to succeed.
+func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) {
+	// Start an xDS management server that uses a couple of channels to
+	// notify the test about the following events:
+	// - an EDS requested with the expected resource name is requested
+	// - EDS resource is unrequested, i.e, an EDS request with no resource name
+	//   is received, which indicates that we are no longer interested in that
+	//   resource.
+	edsResourceRequestedCh := make(chan struct{}, 1)
+	edsResourceCanceledCh := make(chan struct{}, 1)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if req.GetTypeUrl() == version.V3EndpointsURL {
+				switch len(req.GetResourceNames()) {
+				case 0:
+					select {
+					case edsResourceCanceledCh <- struct{}{}:
+					default:
+					}
+				case 1:
+					if req.GetResourceNames()[0] == edsServiceName {
+						select {
+						case edsResourceRequestedCh <- struct{}{}:
+						default:
+						}
+					}
+				default:
+					t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames()))
+				}
+			}
+			return nil
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure cluster and endpoints resources in the management server.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	cc, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	// Wait for the EDS resource to be requested.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to be requested")
+	case <-edsResourceRequestedCh:
+	}
+
+	// Ensure that a successful RPC can be made.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	oldCluster := resources.Clusters
+	// Delete the cluster resource from the management server.
+	resources.Clusters = nil
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the EDS resource to be not requested anymore.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to not be requested")
+	case <-edsResourceCanceledCh:
+	}
+
+	// Ensure that RPCs start to fail with expected error.
+	wantErr := fmt.Sprintf("resource %q of type %q has been removed", clusterName, "ClusterResource")
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		_, err := client.EmptyCall(sCtx, &testpb.Empty{})
+		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
+			break
+		}
+		if err != nil {
+			t.Logf("EmptyCall failed: %v", err)
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("RPCs did not fail after removal of Cluster resource")
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Add the cluster resource back to the management server.
+	resources.Clusters = oldCluster
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the EDS resource to be requested again.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to be requested")
+	case <-edsResourceRequestedCh:
+	}
+
+	// Ensure that a successful RPC can be made.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil {
+			t.Logf("EmptyCall failed: %v", err)
+			continue
+		}
+		break
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("RPCs did not fail after removal of Cluster resource")
+	}
+}
+
+// Test verifies that when the received Cluster resource contains outlier
+// detection configuration, the LB config pushed to the priority policy contains
+// the appropriate configuration for the outlier detection LB policy.
+func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) {
+	// Unregister the priority balancer builder for the duration of this test,
+	// and register a policy under the same name that makes the LB config
+	// pushed to it available to the test.
+	priorityBuilder := balancer.Get(priority.Name)
+	internal.BalancerUnregister(priorityBuilder.Name())
+	lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1)
+	stub.Register(priority.Name, stub.BalancerFuncs{
+		Init: func(bd *stub.BalancerData) {
+			bd.ChildBalancer = priorityBuilder.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return priorityBuilder.(balancer.ConfigParser).ParseConfig(lbCfg)
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			select {
+			case lbCfgCh <- ccs.BalancerConfig:
+			default:
+			}
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+	})
+	defer balancer.Register(priorityBuilder)
+
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure cluster and endpoints resources in the management server.
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)
+	cluster.OutlierDetection = &v3clusterpb.OutlierDetection{
+		Interval:                 durationpb.New(10 * time.Second),
+		BaseEjectionTime:         durationpb.New(30 * time.Second),
+		MaxEjectionTime:          durationpb.New(300 * time.Second),
+		MaxEjectionPercent:       wrapperspb.UInt32(10),
+		SuccessRateStdevFactor:   wrapperspb.UInt32(2000),
+		EnforcingSuccessRate:     wrapperspb.UInt32(50),
+		SuccessRateMinimumHosts:  wrapperspb.UInt32(10),
+		SuccessRateRequestVolume: wrapperspb.UInt32(50),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{cluster},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	_, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	// The priority configuration generated should have Outlier Detection as a
+	// direct child due to Outlier Detection being turned on.
+	wantCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config: &iserviceconfig.BalancerConfig{
+					Name: outlierdetection.Name,
+					Config: &outlierdetection.LBConfig{
+						Interval:           iserviceconfig.Duration(10 * time.Second), // default interval
+						BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+						MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+						MaxEjectionPercent: 10,
+						SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+							StdevFactor:           2000,
+							EnforcementPercentage: 50,
+							MinimumHosts:          10,
+							RequestVolume:         50,
+						},
+						ChildPolicy: &iserviceconfig.BalancerConfig{
+							Name: clusterimpl.Name,
+							Config: &clusterimpl.LBConfig{
+								Cluster: clusterName,
+								ChildPolicy: &iserviceconfig.BalancerConfig{
+									Name: wrrlocality.Name,
+									Config: &wrrlocality.LBConfig{
+										ChildPolicy: &iserviceconfig.BalancerConfig{
+											Name: roundrobin.Name,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+
+	select {
+	case lbCfg := <-lbCfgCh:
+		gotCfg := lbCfg.(*priority.LBConfig)
+		if diff := cmp.Diff(wantCfg, gotCfg); diff != "" {
+			t.Fatalf("Child policy received unexpected diff in config (-want +got):\n%s", diff)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for child policy to receive its configuration")
+	}
+}

internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/priority" // Register priority LB policy.
+)
+
+// TestLogicalDNS_MultipleEndpoints tests the priority LB policy using a
+// LOGICAL_DNS discovery mechanism.
+//
+// The test verifies that multiple addresses returned by the DNS resolver are
+// grouped into a single endpoint (as per gRFC A61). Because the round_robin LB
+// policy sees only one endpoint, it should not rotate traffic between the
+// addresses. Instead, the single endpoint is picked, and connects to the first
+// address.
+func (s) TestLogicalDNS_MultipleEndpoints(t *testing.T) {
+	// Spin up a management server to receive xDS resources from.
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+	resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start backend servers which provide an implementation of the TestService.
+	server1 := stubserver.StartTestService(t, nil)
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+
+	// Register a manual resolver with the "dns" scheme to override DNS resolution.
+	// This global override is safe because connection to the xDS management
+	// server uses the passthrough scheme instead and therefore overriding
+	// the DNS resolver does not affect it in any way.
+	const dnsScheme = "dns"
+	dnsR := manual.NewBuilderWithScheme(dnsScheme)
+	originalDNS := resolver.Get("dns")
+	resolver.Register(dnsR)
+	t.Cleanup(func() { resolver.Register(originalDNS) })
+
+	// For LOGICAL_DNS, this updates the SINGLE endpoint to have 2 IPs.
+	dnsR.InitialState(resolver.State{
+		Endpoints: []resolver.Endpoint{{
+			Addresses: []resolver.Address{
+				{Addr: server1.Address},
+				{Addr: server2.Address},
+			}}},
+	})
+
+	const (
+		serviceName   = "test-xds-service"
+		clusterName   = "cluster-test-xds-service"
+		endpointsName = "endpoints-test-xds-service"
+		rdsName       = "route-test-xds-service"
+	)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+			ClusterName: clusterName,
+			ServiceName: endpointsName,
+			Type:        e2e.ClusterTypeLogicalDNS,
+			DNSHostName: "dns",
+			DNSPort:     uint32(8080),
+			Policy:      e2e.LoadBalancingPolicyRoundRobin,
+		})},
+		Endpoints: nil,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///"+serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("Failed to create new client for local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Ensure the RPC is routed to the first backend.
+	testClient := testgrpc.NewTestServiceClient(cc)
+	for i := 0; i < 10; i++ {
+		var peer peer.Peer
+		if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
+			t.Fatalf("RPC failed: %v", err)
+		}
+
+		if got, want := peer.Addr.String(), server1.Address; got != want {
+			t.Errorf("peer.Addr = %q, want = %q", got, want)
+		}
+	}
+}

internal/xds/balancer/cdsbalancer/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cdsbalancer
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[cds-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *cdsBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clusterimpl/balancer_test.go

@@ -0,0 +1,356 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/testutils/fakeclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultShortTestTimeout = 100 * time.Microsecond
+
+	testClusterName = "test-cluster"
+	testServiceName = "test-eds-service"
+)
+
+var (
+	testBackendEndpoints = []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}}
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func init() {
+	NewRandomWRR = testutils.NewTestWRR
+}
+
+// TestPickerUpdateAfterClose covers the case where a child policy sends a
+// picker update after the cluster_impl policy is closed. Because picker updates
+// are handled in the run() goroutine, which exits before Close() returns, we
+// expect the above picker update to be dropped.
+func (s) TestPickerUpdateAfterClose(t *testing.T) {
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a subConn state
+	// change).
+	closeCh := make(chan struct{})
+	const childPolicyName = "stubBalancer-TestPickerUpdateAfterClose"
+	stub.Register(childPolicyName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			// Create a subConn which will be used later on to test the race
+			// between StateListener() and Close().
+			sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(balancer.SubConnState) {
+					go func() {
+						// Wait for Close() to be called on the parent policy before
+						// sending the picker update.
+						<-closeCh
+						bd.ClientConn.UpdateState(balancer.State{
+							Picker: base.NewErrPicker(errors.New("dummy error picker")),
+						})
+					}()
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+
+	var maxRequest uint32 = 50
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+						MaxRequests:    &maxRequest,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: childPolicyName,
+			},
+		},
+	}); err != nil {
+		b.Close()
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Send a subConn state change to trigger a picker update. The stub balancer
+	// that we use as the child policy will not send a picker update until the
+	// parent policy is closed.
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	b.Close()
+	close(closeCh)
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("unexpected picker update after balancer is closed")
+	case <-time.After(defaultShortTestTimeout):
+	}
+}
+
+// TestClusterNameInAddressAttributes covers the case that cluster name is
+// attached to the subconn address attributes.
+func (s) TestClusterNameInAddressAttributes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendEndpoints[0].Addresses[0].Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	cn, ok := xdsinternal.GetXDSHandshakeClusterName(addrs1[0].Attributes)
+	if !ok || cn != testClusterName {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName)
+	}
+
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	const testClusterName2 = "test-cluster-2"
+	var addr2 = resolver.Address{Addr: "2.2.2.2"}
+	state2 := xdsclient.SetClient(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{addr2}}}}, xdsC)
+	state2 = xdsresource.SetXDSConfig(state2, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName2: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName2,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state2,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName2,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, addr2.Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	// New addresses should have the new cluster name.
+	cn2, ok := xdsinternal.GetXDSHandshakeClusterName(addrs2[0].Attributes)
+	if !ok || cn2 != testClusterName2 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
+	}
+}
+
+// Test verify that the case picker is updated synchronously on receipt of
+// configuration update.
+func (s) TestPickerUpdatedSynchronouslyOnConfigUpdate(t *testing.T) {
+	// Override the pickerUpdateHook to be notified that picker was updated.
+	pickerUpdated := make(chan struct{}, 1)
+	origNewPickerUpdated := pickerUpdateHook
+	pickerUpdateHook = func() {
+		pickerUpdated <- struct{}{}
+	}
+	defer func() { pickerUpdateHook = origNewPickerUpdated }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Override the clientConnUpdateHook to ensure client conn was updated.
+	clientConnUpdateDone := make(chan struct{}, 1)
+	origClientConnUpdateHook := clientConnUpdateHook
+	clientConnUpdateHook = func() {
+		// Verify that picker was updated before the completion of
+		// client conn update.
+		select {
+		case <-pickerUpdated:
+		case <-ctx.Done():
+			t.Fatal("Client conn update completed before picker update.")
+		}
+		clientConnUpdateDone <- struct{}{}
+	}
+	defer func() { clientConnUpdateHook = origClientConnUpdateHook }()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a resolver
+	// update).
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{
+				Picker: base.NewErrPicker(errors.New("dummy error picker")),
+			})
+			return nil
+		},
+	})
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: t.Name(),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("Unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	select {
+	case <-clientConnUpdateDone:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for client conn update to be completed.")
+	}
+}
+
+func (s) TestDropRequestsPerMillion(t *testing.T) {
+	tests := []struct {
+		name        string
+		numerator   uint32
+		denominator uint32
+		want        uint32
+	}{
+		{name: "zero", numerator: 0, denominator: million, want: 0},
+		{name: "five percent hundred", numerator: 5, denominator: 100, want: 50000},
+		// numerator*million overflows a uint32 once numerator reaches 4295 with
+		// the million denominator, which is only a 0.43% drop.
+		{name: "point four three percent million", numerator: 4295, denominator: million, want: 4295},
+		{name: "one percent million", numerator: 10000, denominator: million, want: 10000},
+		{name: "fifty percent million", numerator: 500000, denominator: million, want: 500000},
+		{name: "hundred percent million", numerator: million, denominator: million, want: million},
+		// A fraction above 100% is capped at a million.
+		{name: "over hundred percent", numerator: 150, denominator: 100, want: million},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := dropRequestsPerMillion(tt.numerator, tt.denominator); got != tt.want {
+				t.Errorf("dropRequestsPerMillion(%d, %d) = %d, want %d", tt.numerator, tt.denominator, got, tt.want)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/clusterimpl.go

@@ -0,0 +1,644 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clusterimpl implements the xds_cluster_impl balancing policy. It
+// handles the cluster features (e.g. circuit_breaking, RPC dropping).
+//
+// Note that it doesn't handle name resolution, which is done by policy
+// xds_cluster_resolver.
+package clusterimpl
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/weightedroundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/balancer/loadstore"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	// Name is the name of the cluster_impl balancer.
+	Name                   = "xds_cluster_impl_experimental"
+	defaultRequestCountMax = 1024
+	loadStoreStopTimeout   = 1 * time.Second
+)
+
+var (
+	// Below function is no-op in actual code, but can be overridden in
+	// tests to give tests visibility into exactly when certain events happen.
+	clientConnUpdateHook = func() {}
+	pickerUpdateHook     = func() {}
+	buildProvider        = buildProviderFunc
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &clusterImplBalancer{
+		ClientConn:      cc,
+		loadWrapper:     loadstore.NewWrapper(),
+		requestCountMax: defaultRequestCountMax,
+	}
+	b.xdsHIPtr.Store(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+	b.logger = prefixLogger(b)
+	b.child = gracefulswitch.NewBalancer(b, bOpts)
+	b.logger.Infof("Created")
+
+	var creds credentials.TransportCredentials
+	switch {
+	case bOpts.DialCreds != nil:
+		creds = bOpts.DialCreds
+	case bOpts.CredsBundle != nil:
+		creds = bOpts.CredsBundle.TransportCredentials()
+	}
+	if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+		b.xdsCredsInUse = true
+	}
+	b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse)
+	return b
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type clusterImplBalancer struct {
+	balancer.ClientConn
+
+	// The following fields are set at creation time, and are read-only after
+	// that, and therefore need not be protected by a mutex.
+	logger *grpclog.PrefixLogger
+	// TODO: #8366 -  Refactor usage of loadWrapper to easily plugin a test
+	// load reporter from tests.
+	loadWrapper *loadstore.Wrapper
+
+	// The following fields are only accessed from balancer API methods, which
+	// are guaranteed to be called serially by gRPC.
+	xdsClient        xdsclient.XDSClient     // Sent down in ResolverState attributes.
+	cancelLoadReport func(context.Context)   // To stop reporting load through the above xDS client.
+	edsServiceName   string                  // EDS service name to report load for.
+	lrsServer        *bootstrap.ServerConfig // Load reporting server configuration.
+	dropCategories   []DropConfig            // The categories for drops.
+	child            *gracefulswitch.Balancer
+	xdsHIPtr         atomic.Pointer[grpcsync.RefCounted[*xds.HandshakeInfo]] // Accessed atomically as it is shared between the balancer and the transport.
+	xdsCredsInUse    bool
+
+	// The active security configuration received from the control plane.
+	securityConfig *xdsresource.SecurityConfig
+
+	// The following fields are protected by mu, since they are accessed in
+	// balancer API methods and in methods called from the child policy.
+	mu                       sync.Mutex
+	clusterName              string                                      // The cluster name for credentials handshaking.
+	inhibitPickerUpdates     bool                                        // Inhibits state updates from child policy when processing an update from the parent.
+	pendingPickerUpdates     bool                                        // True if a picker update from the child policy was inhibited when processing an update from the parent.
+	childState               balancer.State                              // Most recent state update from the child policy.
+	drops                    []*dropper                                  // Drops implementation.
+	requestCounterCluster    string                                      // The cluster name for the request counter, from LB config.
+	requestCounterService    string                                      // The service name for the request counter, from LB config.
+	requestCountMax          uint32                                      // Max concurrent requests, from LB config.
+	requestCounter           *xdsclient.ClusterRequestsCounter           // Tracks total inflight requests for a given service.
+	telemetryLabels          map[string]string                           // Telemetry labels to set on picks, from LB config.
+	lrsReportEndpointMetrics *xdsresource.LRSReportEndpointMetricsConfig // LRS metrics to propagate.
+}
+
+// dropRequestsPerMillion scales a drop overload's numerator and denominator to
+// a number of requests to drop per million. numerator can be as large as a
+// million (a 100% drop), so the multiplication is done in uint64 to avoid
+// overflowing a uint32, and the result is capped at a million because a drop
+// ratio above 100% is treated as 100%.
+func dropRequestsPerMillion(numerator, denominator uint32) uint32 {
+	rpm := uint64(numerator) * million / uint64(denominator)
+	if rpm > million {
+		rpm = million
+	}
+	return uint32(rpm)
+}
+
+// handleClusterConfigLocked updates the internal state of the balancer with the
+// new cluster configuration. It returns true if a new picker needs to be
+// generated as a result of these changes. It must be called with b.mu held.
+func (b *clusterImplBalancer) handleClusterConfigLocked(clusterConfig xdsresource.ClusterConfig) bool {
+	clusterUpdate := clusterConfig.Cluster
+	var updatePicker bool
+
+	b.telemetryLabels = clusterUpdate.TelemetryLabels
+
+	if !b.lrsReportEndpointMetrics.Equal(clusterUpdate.LRSReportEndpointMetrics) {
+		b.lrsReportEndpointMetrics = clusterUpdate.LRSReportEndpointMetrics
+		updatePicker = true
+	}
+
+	var newDrops []DropConfig
+	if clusterUpdate.ClusterType == xdsresource.ClusterTypeEDS {
+		edsUpdate := clusterConfig.EndpointConfig.EDSUpdate
+		newDrops = make([]DropConfig, 0, len(edsUpdate.Drops))
+		for _, d := range edsUpdate.Drops {
+			newDrops = append(newDrops, DropConfig{
+				Category:           d.Category,
+				RequestsPerMillion: dropRequestsPerMillion(d.Numerator, d.Denominator),
+			})
+		}
+		if !slices.Equal(b.dropCategories, newDrops) {
+			b.dropCategories = newDrops
+			b.drops = make([]*dropper, 0, len(newDrops))
+			for _, c := range newDrops {
+				b.drops = append(b.drops, newDropper(c))
+			}
+			updatePicker = true
+		}
+	}
+
+	if b.requestCounterCluster != clusterUpdate.ClusterName || b.requestCounterService != clusterUpdate.EDSServiceName {
+		b.requestCounterCluster = clusterUpdate.ClusterName
+		b.requestCounterService = clusterUpdate.EDSServiceName
+		b.requestCounter = xdsclient.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName)
+		updatePicker = true
+	}
+	var newRequestCountMax uint32 = 1024
+	if clusterUpdate.MaxRequests != nil {
+		newRequestCountMax = *clusterUpdate.MaxRequests
+	}
+	if b.requestCountMax != newRequestCountMax {
+		b.requestCountMax = newRequestCountMax
+		updatePicker = true
+	}
+
+	return updatePicker
+}
+
+func (b *clusterImplBalancer) newPickerLocked() *picker {
+	return &picker{
+		drops:           b.drops,
+		s:               b.childState,
+		loadStore:       b.loadWrapper,
+		counter:         b.requestCounter,
+		countMax:        b.requestCountMax,
+		telemetryLabels: b.telemetryLabels,
+		clusterName:     b.clusterName,
+		metrics:         b.lrsReportEndpointMetrics,
+	}
+}
+
+// updateLoadStore checks the config for load store, and decides whether it
+// needs to restart the load reporting stream.
+func (b *clusterImplBalancer) updateLoadStore(clusterUpdate *xdsresource.ClusterUpdate) error {
+	var updateLoadClusterAndService bool
+
+	// ClusterName is different, restart. ClusterName is from ClusterName and
+	// EDSServiceName.
+	clusterName := b.getClusterName()
+	if clusterName != clusterUpdate.ClusterName {
+		updateLoadClusterAndService = true
+		b.setClusterName(clusterUpdate.ClusterName)
+		clusterName = clusterUpdate.ClusterName
+	}
+	if b.edsServiceName != clusterUpdate.EDSServiceName {
+		updateLoadClusterAndService = true
+		b.edsServiceName = clusterUpdate.EDSServiceName
+	}
+	if updateLoadClusterAndService {
+		// This updates the clusterName and serviceName that will be reported
+		// for the loads. The update here is too early, the perfect timing is
+		// when the picker is updated with the new connection. But from this
+		// balancer's point of view, it's impossible to tell.
+		//
+		// On the other hand, this will almost never happen. Each LRS policy
+		// shouldn't get updated config. The parent should do a graceful switch
+		// when the clusterName or serviceName is changed.
+		b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName)
+	}
+
+	var (
+		stopOldLoadReport  bool
+		startNewLoadReport bool
+	)
+
+	// Check if it's necessary to restart load report.
+	if b.lrsServer == nil {
+		if clusterUpdate.LRSServerConfig != nil {
+			// Old is nil, new is not nil, start new LRS.
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			startNewLoadReport = true
+		}
+		// Old is nil, new is nil, do nothing.
+	} else if clusterUpdate.LRSServerConfig == nil {
+		// Old is not nil, new is nil, stop old, don't start new.
+		b.lrsServer = nil
+		stopOldLoadReport = true
+	} else {
+		// Old is not nil, new is not nil, compare string values, if
+		// different, stop old and start new.
+		if !b.lrsServer.Equal(clusterUpdate.LRSServerConfig) {
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			stopOldLoadReport = true
+			startNewLoadReport = true
+		}
+	}
+
+	if stopOldLoadReport {
+		if b.cancelLoadReport != nil {
+			stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+			defer stopCancel()
+			b.cancelLoadReport(stopCtx)
+			b.cancelLoadReport = nil
+			if !startNewLoadReport {
+				// If a new LRS stream will be started later, no need to update
+				// it to nil here.
+				b.loadWrapper.UpdateLoadStore(nil)
+			}
+		}
+	}
+	if startNewLoadReport {
+		var loadStore *lrsclient.LoadStore
+		if b.xdsClient != nil {
+			loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(b.lrsServer)
+		}
+		b.loadWrapper.UpdateLoadStore(loadStore)
+	}
+
+	return nil
+}
+
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg := configs[instanceName]
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
+
+func (b *clusterImplBalancer) buildProviders(config *xdsresource.SecurityConfig) (certprovider.Provider, certprovider.Provider, error) {
+	cpc := b.xdsClient.BootstrapConfig().CertProviderConfigs()
+	var rootProvider certprovider.Provider
+	if config.UseSystemRootCerts {
+		rootProvider = systemRootCertsProvider{}
+	} else {
+		rp, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		rootProvider = rp
+	}
+
+	var identityProvider certprovider.Provider
+	if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" {
+		ip, err := buildProvider(cpc, name, cert, true, false)
+		if err != nil {
+			if rootProvider != nil {
+				rootProvider.Close()
+			}
+			return nil, nil, err
+		}
+		identityProvider = ip
+	}
+	return rootProvider, identityProvider, nil
+}
+
+// handleSecurityConfig processes the security configuration received from the
+// management server, creates appropriate certificate provider plugins, and
+// updates the HandshakeInfo which is added as an address attribute in
+// NewSubConn() calls.
+func (b *clusterImplBalancer) handleSecurityConfig(config *xdsresource.SecurityConfig) error {
+	// If xdsCredentials are not in use, i.e, the user did not want to get
+	// security configuration from an xDS server, we should not be acting on the
+	// received security config here. Doing so poses a security threat.
+	if !b.xdsCredsInUse || config.Equal(b.securityConfig) {
+		return nil
+	}
+
+	// Security config being nil is a valid case where the management server has
+	// not sent any security configuration. The xdsCredentials implementation
+	// handles this by delegating to its fallback credentials.
+	if config == nil {
+		b.securityConfig = nil
+		oldHI := b.xdsHIPtr.Swap(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+		if oldHI != nil {
+			oldHI.Decrement()
+		}
+		return nil
+	}
+
+	rootProvider, identityProvider, err := b.buildProviders(config)
+	if err != nil {
+		return err
+	}
+
+	newHI := xds.NewHandshakeInfo(rootProvider, identityProvider, config.SubjectAltNameMatchers, config.SNI, false, config.AutoSNISANValidation, config.UseAutoHostSNI)
+	b.securityConfig = config
+	oldHI := b.xdsHIPtr.Swap(newHI)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	return nil
+}
+
+func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	defer clientConnUpdateHook()
+
+	b.mu.Lock()
+	b.inhibitPickerUpdates = true
+	b.mu.Unlock()
+	if b.logger.V(2) {
+		b.logger.Infof("Received configuration: %s", pretty.ToJSON(s.BalancerConfig))
+	}
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	// Need to check for potential errors at the beginning of this function, so
+	// that on errors, we reject the whole config, instead of applying part of
+	// it.
+	bb := balancer.Get(newConfig.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("child policy %q not registered", newConfig.ChildPolicy.Name)
+	}
+
+	if b.xdsClient == nil {
+		c := xdsclient.FromResolverState(s.ResolverState)
+		if c == nil {
+			return balancer.ErrBadResolverState
+		}
+		b.xdsClient = c
+	}
+
+	xdsConfig := xdsresource.XDSConfigFromResolverState(s.ResolverState)
+	if xdsConfig == nil {
+		b.logger.Warningf("Received balancer config with no xDS config")
+		return balancer.ErrBadResolverState
+	}
+	clusterCfg := xdsConfig.Clusters[newConfig.Cluster]
+	clusterUpdate := clusterCfg.Config.Cluster
+	if err := b.handleSecurityConfig(clusterUpdate.SecurityCfg); err != nil {
+		// If the security config is invalid, for example, if the provider
+		// instance is not found in the bootstrap config, we need to put the
+		// channel in transient failure.
+		return fmt.Errorf("received Cluster resource that contains invalid security config: %v", err)
+
+	}
+	// Update load reporting config. This needs to be done before updating the
+	// child policy because we need the loadStore from the updated client to be
+	// passed to the ccWrapper, so that the next picker from the child policy
+	// will pick up the new loadStore.
+	if err := b.updateLoadStore(clusterUpdate); err != nil {
+		return err
+	}
+
+	// Build config for the gracefulswitch balancer. It is safe to ignore JSON
+	// marshaling errors here, since the config was already validated as part of
+	// ParseConfig().
+	cfg := []map[string]any{{newConfig.ChildPolicy.Name: newConfig.ChildPolicy.Config}}
+	cfgJSON, _ := json.Marshal(cfg)
+	parsedCfg, err := gracefulswitch.ParseConfig(cfgJSON)
+	if err != nil {
+		return err
+	}
+
+	// Addresses and sub-balancer config are sent to sub-balancer.
+	err = b.child.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  weightedroundrobin.SetBackendService(s.ResolverState, b.clusterName),
+		BalancerConfig: parsedCfg,
+	})
+
+	b.mu.Lock()
+	updatePicker := b.handleClusterConfigLocked(clusterCfg.Config)
+	// We want to send a picker update to the parent if one of the two
+	// conditions are met:
+	// - drop/request count config or LRS metrics config has changed *and* there
+	//   is already a picker from the child, or
+	// - there is a pending picker update from the child (and this covers the
+	//   case where the drop/request config has not changed, but the child sent
+	//   a picker update while we were still processing config from our parent).
+	if (updatePicker && b.childState.Picker != nil) || b.pendingPickerUpdates {
+		b.pendingPickerUpdates = false
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker:            b.newPickerLocked(),
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.mu.Unlock()
+	pickerUpdateHook()
+	return err
+}
+
+func (b *clusterImplBalancer) ResolverError(err error) {
+	b.child.ResolverError(err)
+}
+
+func (b *clusterImplBalancer) updateSubConnState(_ balancer.SubConn, s balancer.SubConnState, cb func(balancer.SubConnState)) {
+	// Trigger re-resolution when a SubConn turns transient failure. This is
+	// necessary for the LogicalDNS in cluster_resolver policy to re-resolve.
+	//
+	// Note that this happens not only for the addresses from DNS, but also for
+	// EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent
+	// knows). The parent priority policy is configured to ignore re-resolution
+	// signal from the EDS children.
+	if s.ConnectivityState == connectivity.TransientFailure {
+		b.ClientConn.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
+	if cb != nil {
+		cb(s)
+	}
+}
+
+func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)
+}
+
+func (b *clusterImplBalancer) Close() {
+	b.child.Close()
+	b.childState = balancer.State{}
+
+	if b.cancelLoadReport != nil {
+		stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+		defer stopCancel()
+		b.cancelLoadReport(stopCtx)
+		b.cancelLoadReport = nil
+	}
+	oldHI := b.xdsHIPtr.Swap(nil)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	b.securityConfig = nil
+	b.logger.Infof("Shutdown")
+}
+
+func (b *clusterImplBalancer) ExitIdle() {
+	b.child.ExitIdle()
+}
+
+// Override methods to accept updates from the child LB.
+
+func (b *clusterImplBalancer) UpdateState(state balancer.State) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Inhibit sending a picker update to our parent as part of handling new
+	// state from the child, if we are currently handling an update from our
+	// parent. Update the childState field regardless.
+	b.childState = state
+	if b.inhibitPickerUpdates {
+		b.pendingPickerUpdates = true
+		if b.logger.V(2) {
+			b.logger.Infof("Received a picker update from the child when processing an update from the parent")
+		}
+		return
+	}
+
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: state.ConnectivityState,
+		Picker:            b.newPickerLocked(),
+	})
+	pickerUpdateHook()
+}
+
+func (b *clusterImplBalancer) setClusterName(n string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.clusterName = n
+}
+
+func (b *clusterImplBalancer) getClusterName() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.clusterName
+}
+
+// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be
+// retrieved from the addresses when creating SubConn.
+//
+// All SubConns passed to the child policies are wrapped in this, so that the
+// picker can get the localityID from the picked SubConn, and do load reporting.
+//
+// After wrapping, all SubConns to and from the parent ClientConn (e.g. for
+// SubConn state update, update/remove SubConn) must be the original SubConns.
+// All SubConns to and from the child policy (NewSubConn, forwarding SubConn
+// state update) must be the wrapper. The balancer keeps a map from the original
+// SubConn to the wrapper for this purpose.
+type scWrapper struct {
+	balancer.SubConn
+
+	localityID clients.Locality
+
+	hostname string
+}
+
+func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	clusterName := b.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetXDSHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = xds.SetHandshakeInfo(newAddrs[i], &b.xdsHIPtr)
+
+		host := xdsresource.Hostname(addr)
+		// If the hostname contains a port, strip it. Per [RFC 6066, Section
+		// 3](https://www.rfc-editor.org/rfc/rfc6066.html#section-3), the SNI
+		// may only contain a qualified DNS hostname, which excludes port
+		// numbers.
+		h, _, err := net.SplitHostPort(host)
+		if err == nil {
+			host = h
+		}
+		// Store hostname in the address attributes, so that it can be used in
+		// the client handshake.
+		newAddrs[i] = xds.SetAddressHostname(newAddrs[i], host)
+	}
+	var sc balancer.SubConn
+	scw := &scWrapper{}
+	if len(addrs) > 0 {
+		scw.hostname = xdsresource.Hostname(addrs[0])
+		scw.localityID = xdsinternal.GetLocalityID(addrs[0])
+	}
+	oldListener := opts.StateListener
+	opts.StateListener = func(state balancer.SubConnState) {
+		b.updateSubConnState(sc, state, oldListener)
+	}
+	sc, err := b.ClientConn.NewSubConn(newAddrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	return scw, nil
+}
+
+func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// systemRootCertsProvider implements a certprovider.Provider that returns the
+// system default root certificates for validation.
+type systemRootCertsProvider struct{}
+
+func (systemRootCertsProvider) Close() {}
+
+func (systemRootCertsProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) {
+	rootCAs, err := internal.X509SystemCertPoolFunc()
+	if err != nil {
+		return nil, err
+	}
+	return &certprovider.KeyMaterial{Roots: rootCAs}, nil
+}

internal/xds/balancer/clusterimpl/config.go

@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// DropConfig contains the category, and drop ratio.
+type DropConfig struct {
+	Category           string
+	RequestsPerMillion uint32
+}
+
+// LBConfig is the balancer config for cluster_impl balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	Cluster     string                                `json:"cluster,omitempty"`
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}

internal/xds/balancer/clusterimpl/config_test.go

@@ -0,0 +1,132 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/roundrobin"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+const (
+	testJSONConfig = `{
+  "cluster": "test_cluster",
+  "edsServiceName": "test-eds",
+  "lrsLoadReportingServer": {
+    "server_uri": "trafficdirector.googleapis.com:443",
+    "channel_creds": [ { "type": "google_default" } ]
+  },
+  "maxConcurrentRequests": 123,
+  "dropCategories": [
+    {
+      "category": "drop-1",
+      "requestsPerMillion": 314
+    },
+    {
+      "category": "drop-2",
+      "requestsPerMillion": 159
+    }
+  ],
+  "childPolicy": [
+    {
+      "weighted_target_experimental": {
+        "targets": {
+          "wt-child-1": {
+            "weight": 75,
+            "childPolicy":[{"round_robin":{}}]
+          },
+          "wt-child-2": {
+            "weight": 25,
+            "childPolicy":[{"round_robin":{}}]
+          }
+        }
+      }
+    }
+  ]
+}`
+
+	wtName = "weighted_target_experimental"
+)
+
+var (
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON   = `{
+  "targets": {
+    "wt-child-1": {
+      "weight": 75,
+      "childPolicy":[{"round_robin":{}}]
+    },
+    "wt-child-2": {
+      "weight": 25,
+      "childPolicy":[{"round_robin":{}}]
+    }
+  }
+}`
+
+	wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON))
+)
+
+func (s) TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *LBConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "bad json",
+			js:      "{",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &LBConfig{
+				Cluster: "test_cluster",
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name:   wtName,
+					Config: wtConfig,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Errorf("parseConfig() got unexpected diff (-want, +got): %v", diff)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/internal/internal.go

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package internal contains code internal to the clusterimpl package.
+package internal
+
+import "crypto/x509"
+
+// X509SystemCertPoolFunc is used for overriding the system cert pool for
+// tests.
+var X509SystemCertPoolFunc = x509.SystemCertPool

internal/xds/balancer/clusterimpl/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-cluster-impl-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clusterimpl/picker.go

@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/wrr"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+// NewRandomWRR is used when calculating drops. It's exported so that tests can
+// override it.
+var NewRandomWRR = wrr.NewRandom
+
+const million = 1000000
+
+type dropper struct {
+	category string
+	w        wrr.WRR
+}
+
+// greatest common divisor (GCD) via Euclidean algorithm
+func gcd(a, b uint32) uint32 {
+	for b != 0 {
+		t := b
+		b = a % b
+		a = t
+	}
+	return a
+}
+
+func newDropper(c DropConfig) *dropper {
+	w := NewRandomWRR()
+	gcdv := gcd(c.RequestsPerMillion, million)
+	// Return true for RequestPerMillion, false for the rest.
+	w.Add(true, int64(c.RequestsPerMillion/gcdv))
+	w.Add(false, int64((million-c.RequestsPerMillion)/gcdv))
+
+	return &dropper{
+		category: c.Category,
+		w:        w,
+	}
+}
+
+func (d *dropper) drop() (ret bool) {
+	return d.w.Next().(bool)
+}
+
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallStarted(locality clients.Locality)
+	CallFinished(locality clients.Locality, err error)
+	CallServerLoad(locality clients.Locality, name string, val float64)
+	CallDropped(category string)
+}
+
+// Picker implements RPC drop, circuit breaking drop and load reporting.
+type picker struct {
+	drops           []*dropper
+	s               balancer.State
+	loadStore       loadReporter
+	counter         *xdsclient.ClusterRequestsCounter
+	countMax        uint32
+	telemetryLabels map[string]string
+	clusterName     string
+	metrics         *xdsresource.LRSReportEndpointMetricsConfig
+}
+
+func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// Unconditionally update labels if present, even dropped or queued RPC's can
+	// use these labels.
+	stats.UpdateLabels(info.Ctx, d.telemetryLabels)
+
+	// Don't drop unless the inner picker is READY. Similar to
+	// https://github.com/grpc/grpc-go/issues/2622.
+	if d.s.ConnectivityState == connectivity.Ready {
+		// Check if this RPC should be dropped by category.
+		for _, dp := range d.drops {
+			if dp.drop() {
+				if d.loadStore != nil {
+					d.loadStore.CallDropped(dp.category)
+				}
+				return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
+			}
+		}
+	}
+
+	// Check if this RPC should be dropped by circuit breaking.
+	if d.counter != nil {
+		if err := d.counter.StartRequest(d.countMax); err != nil {
+			// Drops by circuit breaking are reported with empty category. They
+			// will be reported only in total drops, but not in per category.
+			if d.loadStore != nil {
+				d.loadStore.CallDropped("")
+			}
+			return balancer.PickResult{}, status.Error(codes.Unavailable, err.Error())
+		}
+	}
+
+	var lID clients.Locality
+	pr, err := d.s.Picker.Pick(info)
+	if scw, ok := pr.SubConn.(*scWrapper); ok {
+		// This OK check also covers the case err!=nil, because SubConn will be
+		// nil.
+		pr.SubConn = scw.SubConn
+		// If locality ID isn't found in the wrapper, an empty locality ID will
+		// be used.
+		lID = scw.localityID
+
+		if scw.hostname != "" && autoHostRewriteEnabled(info.Ctx) {
+			if pr.Metadata == nil {
+				pr.Metadata = metadata.Pairs(":authority", scw.hostname)
+			} else {
+				pr.Metadata.Set(":authority", scw.hostname)
+			}
+		}
+	}
+
+	if err != nil {
+		if d.counter != nil {
+			// Release one request count if this pick fails.
+			d.counter.EndRequest()
+		}
+		return pr, err
+	}
+
+	stats.UpdateLabels(
+		info.Ctx,
+		map[string]string{
+			"grpc.lb.locality":        xdsinternal.LocalityString(lID),
+			"grpc.lb.backend_service": d.clusterName,
+		},
+	)
+
+	if d.loadStore != nil {
+		locality := clients.Locality{Region: lID.Region, Zone: lID.Zone, SubZone: lID.SubZone}
+		d.loadStore.CallStarted(locality)
+		oldDone := pr.Done
+		pr.Done = func(info balancer.DoneInfo) {
+			if oldDone != nil {
+				oldDone(info)
+			}
+			d.loadStore.CallFinished(locality, info.Err)
+
+			load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport)
+			if !ok || load == nil {
+				return
+			}
+
+			if envconfig.XDSORCAToLRSPropEnabled {
+				if d.metrics != nil {
+					if d.metrics.CPUUtilization {
+						d.loadStore.CallServerLoad(locality, "cpu_utilization", load.CpuUtilization)
+					}
+					if d.metrics.MemUtilization {
+						d.loadStore.CallServerLoad(locality, "mem_utilization", load.MemUtilization)
+					}
+					if d.metrics.ApplicationUtilization {
+						d.loadStore.CallServerLoad(locality, "application_utilization", load.ApplicationUtilization)
+					}
+					for n, c := range load.NamedMetrics {
+						_, ok := d.metrics.NamedMetrics[n]
+						if d.metrics.NamedMetricsAll || ok {
+							d.loadStore.CallServerLoad(locality, "named_metrics."+n, c)
+						}
+					}
+				}
+			} else {
+				for n, c := range load.NamedMetrics {
+					d.loadStore.CallServerLoad(locality, n, c)
+				}
+			}
+		}
+	}
+
+	if d.counter != nil {
+		// Update Done() so that when the RPC finishes, the request count will
+		// be released.
+		oldDone := pr.Done
+		pr.Done = func(doneInfo balancer.DoneInfo) {
+			d.counter.EndRequest()
+			if oldDone != nil {
+				oldDone(doneInfo)
+			}
+		}
+	}
+
+	return pr, err
+}
+
+// autoHostRewriteKey is the context key used to store the value of
+// route's autoHostRewrite in the RPC context.
+type autoHostRewriteKey struct{}
+
+// autoHostRewriteEnabled retrieves the autoHostRewrite value from the provided context.
+func autoHostRewriteEnabled(ctx context.Context) bool {
+	v, _ := ctx.Value(autoHostRewriteKey{}).(bool)
+	return v
+}
+
+// AutoHostRewriteEnabledForTesting returns the value of autoHostRewrite field;
+// to be used for testing only.
+func AutoHostRewriteEnabledForTesting(ctx context.Context) bool {
+	return autoHostRewriteEnabled(ctx)
+}
+
+// EnableAutoHostRewrite adds the autoHostRewrite value to the context for
+// the xds_cluster_impl LB policy to pick.
+func EnableAutoHostRewrite(ctx context.Context) context.Context {
+	return context.WithValue(ctx, autoHostRewriteKey{}, true)
+}

internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go

@@ -0,0 +1,783 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package clusterimpl_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	iclusterimpl "google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/testdata"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin.
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router"       // Register the router filter.
+	_ "google.golang.org/grpc/internal/xds/resolver"                // Register the xds resolver
+)
+
+// Common setup for security tests:
+//   - creates an xDS client with the specified bootstrap configuration
+//   - creates a gRPC channel that uses the passed in client creds
+//   - creates a test server that uses the passed in server creds
+//
+// Returns the following:
+// - a client channel to make RPCs
+// - address of the test backend server
+func setupForSecurityTests(t *testing.T, bootstrapContents []byte, clientCreds, serverCreds credentials.TransportCredentials) (*grpc.ClientConn, string) {
+	t.Helper()
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Create a ClientConn with the specified transport credentials.
+	cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	t.Cleanup(func() { cc.Close() })
+
+	// Start a test service backend with the specified transport credentials.
+	sOpts := []grpc.ServerOption{}
+	if serverCreds != nil {
+		sOpts = append(sOpts, grpc.Creds(serverCreds))
+	}
+	server := stubserver.StartTestService(t, nil, sOpts...)
+	t.Cleanup(server.Stop)
+
+	return cc, server.Address
+}
+
+// Creates transport credentials to be used on the client side that rely on xDS
+// to provide the security configuration. It falls back to insecure creds if no
+// security information is received from the management server.
+func xdsClientCredsWithInsecureFallback(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	xdsCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create xDS credentials: %v", err)
+	}
+	return xdsCreds
+}
+
+// Creates transport credentials to be used on the server side from certificate
+// files in testdata/x509.
+//
+// The certificate returned by this function has a CommonName of "test-server1".
+func tlsServerCreds(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("Failed to load server cert and key: %v", err)
+
+	}
+	pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("Failed to read client CA cert: %v", err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    roots,
+	}
+	return credentials.NewTLS(cfg)
+}
+
+// Checks the AuthInfo available in the peer if it matches the expected security
+// level of the connection.
+func verifySecurityInformationFromPeer(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel) {
+	// This is not a true helper in the Go sense, because it does not perform
+	// setup or cleanup tasks. Marking it a helper is to ensure that when the
+	// test fails, the line information of the caller is outputted instead of
+	// from here.
+	//
+	// And this function directly calls t.Fatalf() instead of returning an error
+	// and letting the caller decide what to do with it. This is also OK since
+	// all callers will simply end up calling t.Fatalf() with the returned
+	// error, and can't add any contextual information of value to the error
+	// message.
+	t.Helper()
+
+	switch wantSecLevel {
+	case e2e.SecurityLevelNone:
+		if pr.AuthInfo.AuthType() != "insecure" {
+			t.Fatalf("AuthType() is %s, want insecure", pr.AuthInfo.AuthType())
+		}
+	case e2e.SecurityLevelMTLS:
+		ai, ok := pr.AuthInfo.(credentials.TLSInfo)
+		if !ok {
+			t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{})
+		}
+		if len(ai.State.PeerCertificates) != 1 {
+			t.Fatalf("Number of peer certificates is %d, want 1", len(ai.State.PeerCertificates))
+		}
+		cert := ai.State.PeerCertificates[0]
+		const wantCommonName = "test-server1"
+		if cn := cert.Subject.CommonName; cn != wantCommonName {
+			t.Fatalf("Common name in peer certificate is %s, want %s", cn, wantCommonName)
+		}
+	}
+}
+
+// makeAggregateClusterResource returns an aggregate cluster resource with the
+// given name and list of child names.
+func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeAggregate,
+		ChildNames:  childNames,
+	})
+}
+
+// Tests the case where xDS credentials are not in use, but the cds LB policy
+// receives a Cluster update with security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with insecure creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, insecure.NewCredentials(), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where xDS credentials are in use, but the cds LB policy
+// receives a Cluster update without security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is not configured to return any security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the security config returned by the management server
+// cannot be resolved based on the contents of the bootstrap config. Verifies
+// that the cds LB policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server,
+	// and one that does not have certificate providers configuration.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       8080,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// A certificate provider builder that returns a nil Provider from the starter
+// func passed to certprovider.NewBuildableConfig().
+type errCertProviderBuilder struct{}
+
+const errCertProviderName = "err-cert-provider"
+
+func (e errCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) {
+	// Returning a nil Provider simulates the case where an error is encountered
+	// at the time of building the Provider.
+	bc := certprovider.NewBuildableConfig(errCertProviderName, nil, func(certprovider.BuildOptions) certprovider.Provider { return nil })
+	return bc, nil
+}
+
+func (e errCertProviderBuilder) Name() string {
+	return errCertProviderName
+}
+
+func init() {
+	certprovider.Register(errCertProviderBuilder{})
+}
+
+// Tests the case where the certprovider.Store returns an error when the cds LB
+// policy attempts to build a certificate provider. Verifies that the cds LB
+// policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestCertproviderStoreError(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration for
+	// errCertProviderBuilder.
+	nodeID := uuid.New().String()
+	providerCfg := json.RawMessage(fmt.Sprintf(`{
+		"plugin_name": "%s",
+		"config": {}
+	}`, errCertProviderName))
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+		CertificateProviders:               map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure.
+func (s) TestGoodSecurityConfig(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that contains a certificate provider instance
+// that is missing in the bootstrap file. Verifies that the channel moves to
+// TRANSIENT_FAILURE. Subsequently, the cds LB policy receives a cluster
+// resource that contains a certificate provider that is present in the
+// bootstrap file.  Verifies that the connection between the client and the
+// server is secure.
+func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
+	var (
+		target      = "test.service"
+		routeName   = "route1"
+		clusterName = "cluster1"
+		serviceName = "service1"
+	)
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)
+	cluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with initial resources: %v", err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Update the management server with a Cluster resource that contains a
+	// certificate provider instance that is present in the bootstrap
+	// configuration.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with valid resources: %v", err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource without security configuration. Verifies that this results in the
+// use of fallback credentials, which in this case is insecure creds.
+func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
+	target := "test.service"
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// Start a test service backend that does not expect a secure connection.
+	insecureServer := stubserver.StartTestService(t, nil)
+	t.Cleanup(insecureServer.Stop)
+
+	// Update the resources in the management server to contain no security
+	// configuration. This should result in the use of fallback credentials,
+	// which is insecure in our case.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, insecureServer.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to move to the new backend that expects
+	// connections without security.
+	for ctx.Err() == nil {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+			t.Logf("EmptyCall() failed: %v", err)
+		}
+		if peer.Addr.String() == insecureServer.Address {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timed out when waiting for connection to switch to second backend")
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource that is NACKed by the xDS client. Test verifies that the cds LB
+// policy continues to use the previous good configuration.
+func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	resources.Clusters[0] = e2e.DefaultCluster(resources.Clusters[0].Name, resources.Endpoints[0].ClusterName, e2e.SecurityLevelNone)
+	resources.Clusters[0].TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that specifies the use system root certs.
+// Verifies that the connection between the client and the server is secure.
+func (s) TestSystemRootCertsSecurityConfig(t *testing.T) {
+	origFlag := envconfig.XDSSystemRootCertsEnabled
+	origSRCF := iclusterimpl.X509SystemCertPoolFunc
+	defer func() {
+		envconfig.XDSSystemRootCertsEnabled = origFlag
+		iclusterimpl.X509SystemCertPoolFunc = origSRCF
+	}()
+	envconfig.XDSSystemRootCertsEnabled = true
+
+	systemRootCertsFuncCalled := false
+	iclusterimpl.X509SystemCertPoolFunc = func() (*x509.CertPool, error) {
+		certData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+		if err != nil {
+			return nil, fmt.Errorf("failed to read certificate file: %w", err)
+		}
+		certPool := x509.NewCertPool()
+
+		if ok := certPool.AppendCertsFromPEM(certData); !ok {
+			return nil, fmt.Errorf("failed to append certificate to cert pool")
+		}
+		systemRootCertsFuncCalled = true
+		return certPool, nil
+	}
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The cluster
+	// resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelTLSWithSystemRootCerts,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	if systemRootCertsFuncCalled != true {
+		t.Errorf("System root certs were not used during the test.")
+	}
+}
+
+// TestAggregateClusterSecurityConfig tests the case where the CDS LB policy
+// receives a top-level aggregate cluster pointing to a child EDS cluster. The
+// test verifies that the CDS LB policy correctly ignores the aggregate
+// cluster's security configuration and uses the child EDS cluster's security
+// configuration to establish the mTLS connection.
+func (s) TestAggregateClusterSecurityConfig(t *testing.T) {
+	var (
+		target         = "test.service"
+		routeName      = "route1"
+		clusterName    = "cluster1"
+		serviceName    = "service1"
+		edsClusterName = clusterName + "-eds"
+	)
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configures an aggregate cluster with a security config that is not NACKed
+	// by the xdsclient but would fail the handshake as it has an invalid SAN
+	// matcher.
+	aggCluster := makeAggregateClusterResource(clusterName, []string{edsClusterName})
+	aggCluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "bad-cn"}},
+								},
+							},
+							ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+						},
+					},
+				},
+			}),
+		},
+	}
+
+	// Create the child EDS cluster with a valid mTLS security configuration.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			aggCluster,
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelMTLS),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection since
+	// the EDS cluster has a valid mTLS configuration even though the top level
+	// cluster has an incorrect security configuration.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}

internal/xds/balancer/clustermanager/balancerstateaggregator.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+)
+
+type subBalancerState struct {
+	state balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
+}
+
+func (s *subBalancerState) String() string {
+	return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
+}
+
+type balancerStateAggregator struct {
+	cc     balancer.ClientConn
+	logger *grpclog.PrefixLogger
+	csEval *balancer.ConnectivityStateEvaluator
+
+	mu sync.Mutex
+	// This field is used to ensure that no updates are forwarded to the parent
+	// CC once the aggregator is closed. A closed sub-balancer could still send
+	// pickers to this aggregator.
+	closed bool
+	// Map from child policy name to last reported state.
+	idToPickerState map[string]*subBalancerState
+	// Set when UpdateState call propagation is paused.
+	pauseUpdateState bool
+	// Set when UpdateState call propagation is paused and an UpdateState call
+	// is suppressed.
+	needUpdateStateOnResume bool
+}
+
+func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator {
+	return &balancerStateAggregator{
+		cc:              cc,
+		logger:          logger,
+		csEval:          &balancer.ConnectivityStateEvaluator{},
+		idToPickerState: make(map[string]*subBalancerState),
+	}
+}
+
+func (bsa *balancerStateAggregator) close() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.closed = true
+}
+
+// add adds a sub-balancer in CONNECTING state.
+//
+// This is called when there's a new child.
+func (bsa *balancerStateAggregator) add(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+
+	bsa.idToPickerState[id] = &subBalancerState{
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
+	}
+	bsa.csEval.RecordTransition(connectivity.Shutdown, connectivity.Connecting)
+	bsa.buildAndUpdateLocked()
+}
+
+// remove removes the sub-balancer state. Future updates from this sub-balancer,
+// if any, will be ignored.
+//
+// This is called when a child is removed.
+func (bsa *balancerStateAggregator) remove(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	if _, ok := bsa.idToPickerState[id]; !ok {
+		return
+	}
+	// Setting the state of the deleted sub-balancer to Shutdown will get
+	// csEvltr to remove the previous state for any aggregated state
+	// evaluations. Transitions to and from connectivity.Shutdown are ignored
+	// by csEvltr.
+	bsa.csEval.RecordTransition(bsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown)
+	// Remove id and picker from picker map. This also results in future updates
+	// for this ID to be ignored.
+	delete(bsa.idToPickerState, id)
+	bsa.buildAndUpdateLocked()
+}
+
+// pauseStateUpdates causes UpdateState calls to not propagate to the parent
+// ClientConn.  The last state will be remembered and propagated when
+// ResumeStateUpdates is called.
+func (bsa *balancerStateAggregator) pauseStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = true
+	bsa.needUpdateStateOnResume = false
+}
+
+// resumeStateUpdates will resume propagating UpdateState calls to the parent,
+// and call UpdateState on the parent if any UpdateState call was suppressed.
+func (bsa *balancerStateAggregator) resumeStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = false
+	if bsa.needUpdateStateOnResume {
+		bsa.cc.UpdateState(bsa.buildLocked())
+	}
+}
+
+// UpdateState is called to report a balancer state change from sub-balancer.
+// It's usually called by the balancer group.
+//
+// It calls parent ClientConn's UpdateState with the new aggregated state.
+func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
+	bsa.logger.Infof("State update from sub-balancer %q: %+v", id, state)
+
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	pickerSt, ok := bsa.idToPickerState[id]
+	if !ok {
+		// All state starts with an entry in pickStateMap. If ID is not in map,
+		// it's either removed, or never existed.
+		return
+	}
+	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		bsa.csEval.RecordTransition(pickerSt.stateToAggregate, state.ConnectivityState)
+		pickerSt.stateToAggregate = state.ConnectivityState
+	}
+	pickerSt.state = state
+	bsa.buildAndUpdateLocked()
+}
+
+// buildAndUpdateLocked combines the sub-state from each sub-balancer into one
+// state, and sends a picker update to the parent ClientConn.
+func (bsa *balancerStateAggregator) buildAndUpdateLocked() {
+	if bsa.closed {
+		return
+	}
+	if bsa.pauseUpdateState {
+		// If updates are paused, do not call UpdateState, but remember that we
+		// need to call it when they are resumed.
+		bsa.needUpdateStateOnResume = true
+		return
+	}
+	bsa.cc.UpdateState(bsa.buildLocked())
+}
+
+// buildLocked combines sub-states into one.
+func (bsa *balancerStateAggregator) buildLocked() balancer.State {
+	// The picker's return error might not be consistent with the
+	// aggregatedState. Because for this LB policy, we want to always build
+	// picker with all sub-pickers (not only ready sub-pickers), so even if the
+	// overall state is Ready, pick for certain RPCs can behave like Connecting
+	// or TransientFailure.
+	bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState)
+	return balancer.State{
+		ConnectivityState: bsa.csEval.CurrentState(),
+		Picker:            newPickerGroup(bsa.idToPickerState),
+	}
+}

internal/xds/balancer/clustermanager/clustermanager.go

@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clustermanager implements the cluster manager LB policy for xds.
+package clustermanager
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/balancergroup"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const balancerName = "xds_cluster_manager_experimental"
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	b := &bal{}
+	b.logger = prefixLogger(b)
+	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
+	b.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               opts,
+		StateAggregator:         b.stateAggregator,
+		Logger:                  b.logger,
+		SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies
+	})
+	b.logger.Infof("Created")
+	return b
+}
+
+func (bb) Name() string {
+	return balancerName
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type bal struct {
+	logger          *internalgrpclog.PrefixLogger
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *balancerStateAggregator
+
+	children map[string]childConfig
+}
+
+func (b *bal) setErrorPickerForChild(childName string, err error) {
+	b.stateAggregator.UpdateState(childName, balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	})
+}
+
+func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) error {
+	endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints)
+
+	// Remove sub-balancers that are not in the new list from the aggregator and
+	// balancergroup.
+	for name := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			b.stateAggregator.remove(name)
+			b.bg.Remove(name)
+		}
+	}
+
+	var retErr error
+	for childName, childCfg := range newConfig.Children {
+		lbCfg := childCfg.ChildPolicy.Config
+		if _, ok := b.children[childName]; !ok {
+			// Add new sub-balancers to the aggregator and balancergroup.
+			b.stateAggregator.add(childName)
+			b.bg.Add(childName, balancer.Get(childCfg.ChildPolicy.Name))
+		} else {
+			// If the child policy type has changed for existing sub-balancers,
+			// parse the new config and send down the config update to the
+			// balancergroup, which will take care of gracefully switching the
+			// child over to the new policy.
+			//
+			// If we run into errors here, we need to ensure that RPCs to this
+			// child fail, while RPCs to other children with good configs
+			// continue to succeed.
+			newPolicyName, oldPolicyName := childCfg.ChildPolicy.Name, b.children[childName].ChildPolicy.Name
+			if newPolicyName != oldPolicyName {
+				var err error
+				var cfgJSON []byte
+				cfgJSON, err = childCfg.ChildPolicy.MarshalJSON()
+				if err != nil {
+					retErr = fmt.Errorf("failed to JSON marshal load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+				// This overwrites lbCfg to be in the format expected by the
+				// gracefulswitch balancer. So, when this config is pushed to
+				// the child (below), it will result in a graceful switch to the
+				// new child policy.
+				lbCfg, err = balancergroup.ParseConfig(cfgJSON)
+				if err != nil {
+					retErr = fmt.Errorf("failed to parse load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+			}
+		}
+
+		if err := b.bg.UpdateClientConnState(childName, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Endpoints:     endpointsSplit[childName],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: lbCfg,
+		}); err != nil {
+			retErr = fmt.Errorf("failed to push new configuration %v to child %q: %v", childCfg.ChildPolicy.Config, childName, err)
+			b.setErrorPickerForChild(childName, retErr)
+		}
+
+		// Picker update is sent to the parent ClientConn only after the
+		// new child policy returns a picker. So, there is no need to
+		// set needUpdateStateOnResume to true here.
+	}
+
+	b.children = newConfig.Children
+
+	// If multiple sub-balancers run into errors, we will return only the last
+	// one, which is still good enough, since the grpc channel will anyways
+	// return this error as balancer.ErrBadResolver to the name resolver,
+	// resulting in re-resolution attempts.
+	return retErr
+
+	// Adding or removing a sub-balancer will result in the
+	// needUpdateStateOnResume bit to true which results in a picker update once
+	// resumeStateUpdates() is called.
+}
+
+func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error {
+	if b.logger.V(2) {
+		b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
+	}
+
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	b.stateAggregator.pauseStateUpdates()
+	defer b.stateAggregator.resumeStateUpdates()
+	return b.updateChildren(s, newConfig)
+}
+
+func (b *bal) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *bal) Close() {
+	b.stateAggregator.close()
+	b.bg.Close()
+	b.logger.Infof("Shutdown")
+}
+
+func (b *bal) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+const prefix = "[xds-cluster-manager-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *bal) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clustermanager/clustermanager_test.go

@@ -0,0 +1,775 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+	testBackendAddrsCount   = 12
+)
+
+var testBackendAddrStrs []string
+
+func init() {
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+}
+
+func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) {
+	t.Helper()
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p.Pick(info)
+		if fmt.Sprint(err) != fmt.Sprint(wantErr) {
+			t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr)
+		}
+		if gotSCSt.SubConn != wantSC {
+			t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC)
+		}
+	}
+}
+
+func (s) TestClusterPicks(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestConfigUpdateAddCluster covers the cases the balancer receives config
+// update with extra clusters.
+func (s) TestConfigUpdateAddCluster(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with different routes, and different actions. Expect a
+	// new subconn and a picker update.
+	configJSON2 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_3":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], BalancerAttributes: nil})
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[2]}}, []string{"cds:cluster_3"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect exactly one new subconn.
+	addrs := <-cc.NewSubConnAddrsCh
+	sc := <-cc.NewSubConnCh
+	// Clear the attributes before adding to map.
+	addrs[0].BalancerAttributes = nil
+	m1[addrs[0]] = sc
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Should have no more newSubConn.
+	select {
+	case <-time.After(time.Millisecond * 500):
+	case <-cc.NewSubConnCh:
+		addrs := <-cc.NewSubConnAddrsCh
+		t.Fatalf("unexpected NewSubConn with address %v", addrs)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_3"),
+			},
+			wantSC: m1[wantAddrs[2]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives
+// config update with no clusters. Pick should fail with details in error.
+func (s) TestRoutingConfigUpdateDeleteAll(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with no clusters.
+	configJSON2 := `{}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect two removed subconns.
+	for range wantAddrs {
+		select {
+		case <-time.After(time.Millisecond * 500):
+			t.Fatalf("timeout waiting for remove subconn")
+		case <-cc.ShutdownSubConnCh:
+		}
+	}
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(ctx, "cds:notacluster")})
+		if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() {
+			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`)
+		}
+	}
+
+	// Resend the previous config with clusters
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m2 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m2[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p3 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m2[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m2[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+func (s) TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) {
+	const (
+		userAgent          = "ua"
+		defaultTestTimeout = 1 * time.Second
+	)
+
+	// Setup the stub balancer such that we can read the build options passed to
+	// it in the UpdateClientConnState method.
+	ccsCh := testutils.NewChannel()
+	bOpts := balancer.BuildOptions{
+		DialCreds:       insecure.NewCredentials(),
+		CustomUserAgent: userAgent,
+	}
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			if !cmp.Equal(bd.BuildOptions, bOpts) {
+				err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
+				ccsCh.Send(err)
+				return err
+			}
+			ccsCh.Send(nil)
+			return nil
+		},
+	})
+
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, bOpts)
+
+	configJSON1 := fmt.Sprintf(`{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"%s":""}] }
+}
+}`, t.Name())
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := ccsCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timed out waiting for UpdateClientConnState result: %v", err)
+	}
+	if v != nil {
+		t.Fatal(v)
+	}
+}
+
+const initIdleBalancerName = "test-init-idle-balancer"
+
+var errTestInitIdle = fmt.Errorf("init Idle balancer error 0")
+
+func init() {
+	stub.Register(initIdleBalancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
+			sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(state balancer.SubConnState) {
+					err := fmt.Errorf("wrong picker error")
+					if state.ConnectivityState == connectivity.Idle {
+						err = errTestInitIdle
+					}
+					bd.ClientConn.UpdateState(balancer.State{
+						ConnectivityState: state.ConnectivityState,
+						Picker:            &testutils.TestConstPicker{Err: err},
+					})
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+}
+
+// TestInitialIdle covers the case that if the child reports Idle, the overall
+// state will be Idle.
+func (s) TestInitialIdle(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"test-init-idle-balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		sc := <-cc.NewSubConnCh
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
+	}
+
+	if state1 := <-cc.NewStateCh; state1 != connectivity.Idle {
+		t.Fatalf("Received aggregated state: %v, want Idle", state1)
+	}
+}
+
+// TestClusterGracefulSwitch tests the graceful switch functionality for a child
+// of the cluster_manager. The test performs the following steps:
+//  1. Configures a single child for the cluster_manager with a round_robin LB
+//     policy. Ensures the child sends a picker update when the subchannel moves
+//     to READY.
+//  2. Updates the exiting child with a different LB policy (pick_first), and a
+//     different endpoint. This should trigger the graceful switch process, and
+//     cluster_manager should not send a picker update until the child has
+//     completed the graceful switch process.
+//  3. The child should complete the graceful switch process once it receives an
+//     update that moves its subchannel to READY. At this point, the cluster_manager
+//     should send a picker update reflecting the new pick_first balancer's picker.
+func (s) TestClusterGracefulSwitch(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	// Push round_robin config with one endpoint to the cluster_manager.
+	rrConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"round_robin": {}}] }
+		}
+	}`
+	rrConfig, err := parser.ParseConfig([]byte(rrConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse round_robin config: %v", err)
+	}
+	rrEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "rr-backend"}}}, []string{"csp:cluster"})
+	ccs := balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{rrEndpoint}},
+		BalancerConfig: rrConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send round_robin config update to cluster_manager: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Wait for a subchannel to be created.
+	var sc1 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the round_robin LB policy")
+	case sc1 = <-cc.NewSubConnCh:
+	}
+
+	// Update the subchannel's state to READY, and ensure we get a picker update.
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	var p1 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p1 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	pi := balancer.PickInfo{Ctx: SetPickedCluster(ctx, "csp:cluster")}
+	testPick(t, p1, pi, sc1, nil)
+
+	// Update the config to use pick_first, and a different endpoint.
+	pfConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"pick_first": {}}] }
+		}
+	}`
+	pfConfig, err := parser.ParseConfig([]byte(pfConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse pick_first config: %v", err)
+	}
+	pfEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "pf-backend"}}}, []string{"csp:cluster"})
+	ccs = balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{pfEndpoint}},
+		BalancerConfig: pfConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send pick_first config update to cluster_manager: %v", err)
+	}
+
+	// Wait for a subchannel to be created.
+	var sc2 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the pick_first LB policy")
+	case sc2 = <-cc.NewSubConnCh:
+	}
+
+	// Move the pick_first LB policy's subchannel to CONNECTING. This will cause
+	// the pick_first LB policy to send a connecting picker. But this should not
+	// result in the cluster_manager sending a picker update to the channel as
+	// the graceful switch process is not yet complete.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("Received picker from cluster_manager when none expected during graceful switch process")
+	case <-sCtx.Done():
+	}
+
+	// Move the pick_first LB policy's subchannel to READY. This will cause the
+	// pick_first LB policy to send a ready picker and complete the graceful
+	// switch process.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	var p2 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p2 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	testPick(t, p2, pi, sc2, nil)
+
+	// Completion of the graceful switch process should result in the old
+	// subchannel being removed.
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannels belonging to old child to be deleted")
+	case rsc := <-cc.ShutdownSubConnCh:
+		if rsc != sc1 {
+			t.Fatalf("Unexpected subchannel %v being shutdown, want %v", rsc, sc1)
+		}
+	}
+}
+
+// tcc wraps a testutils.TestClientConn but stores all state transitions in a
+// slice.
+type tcc struct {
+	*testutils.BalancerClientConn
+	states []balancer.State
+}
+
+func (t *tcc) UpdateState(bs balancer.State) {
+	t.states = append(t.states, bs)
+	t.BalancerClientConn.UpdateState(bs)
+}
+
+func (s) TestUpdateStatePauses(t *testing.T) {
+	cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)}
+
+	balFuncs := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil})
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil})
+			return nil
+		},
+	}
+	stub.Register("update_state_balancer", balFuncs)
+
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"update_state_balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that the only state update is the second one called by the child.
+	if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready {
+		t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states)
+	}
+}

internal/xds/balancer/clustermanager/config.go

@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type childConfig struct {
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+// lbConfig is the balancer config for xds routing policy.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Children map[string]childConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	cfg := &lbConfig{}
+	if err := json.Unmarshal(c, cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}

internal/xds/balancer/clustermanager/config_test.go

@@ -0,0 +1,144 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer"
+)
+
+const (
+	testJSONConfig = `{
+      "children":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "weighted:cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1" : {
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2" : {
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }
+            }
+          }]
+        },
+        "weighted:cluster_1_cluster_3_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1": {
+                  "weight":99,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_3": {
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+                }
+              }
+            }
+          }]
+        }
+      }
+}
+`
+
+	cdsName = "cds_experimental"
+	wtName  = "weighted_target_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON1  = `{
+	"targets": {
+	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
+	} }`
+	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
+	wtConfigJSON2 = `{
+    "targets": {
+      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
+    } }`
+	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
+)
+
+func (s) Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Children: map[string]childConfig{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig1},
+					},
+					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig2},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", d)
+			}
+		})
+	}
+}

internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go

@@ -0,0 +1,342 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
+	"github.com/google/uuid"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/xds" // Register the xDS name resolver and related LB policies.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+func makeEmptyCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("EmptyCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func makeUnaryCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("UnaryCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Configure client side xDS resources on the management server.
+	const (
+		serviceName     = "my-service-client-side-xds"
+		routeConfigName = "route-" + serviceName
+		clusterName1    = "cluster1-" + serviceName
+		clusterName2    = "cluster2-" + serviceName
+		endpointsName1  = "endpoints1-" + serviceName
+		endpointsName2  = "endpoints2-" + serviceName
+		endpointsName3  = "endpoints3-" + serviceName
+	)
+	// A single Listener resource pointing to the following Route
+	// configuration:
+	//   - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	//   - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+			},
+		}},
+	}}
+	// Two cluster resources corresponding to the ones mentioned in the above
+	// route configuration resource. These are configured with round_robin as
+	// their endpoint picking policy.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelNone),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+	}
+	// Spin up two test backends, one for each cluster below.
+	server1 := stubserver.StartTestService(t, nil)
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	// Two endpoints resources, each with one backend from above.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2.
+	if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a wrapped pickfirst LB policy. When the endpoint picking policy on
+	// the cluster resource is changed to pickfirst, this will allow us to
+	// verify that load balancing configuration is pushed to it.
+	pfBuilder := balancer.Get(pickfirst.Name)
+	internal.BalancerUnregister(pfBuilder.Name())
+
+	lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1)
+	stub.Register(pfBuilder.Name(), stub.BalancerFuncs{
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return pfBuilder.(balancer.ConfigParser).ParseConfig(lbCfg)
+		},
+		Init: func(bd *stub.BalancerData) {
+			bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			select {
+			case lbCfgCh <- ccs.BalancerConfig:
+			default:
+			}
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+	})
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to pickfirst. The endpoints resource is also changed here
+	// to ensure that we can verify that the new child policy
+	cluster2 := &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{
+							ShuffleAddressList: true,
+						}),
+					},
+				},
+			},
+		},
+	}
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+	endpoints3 := e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)})
+	resources.Clusters = append(resources.Clusters, cluster2)
+	resources.Endpoints = append(resources.Endpoints, endpoints3)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for configuration to be pushed to the new pickfirst child policy")
+	case <-lbCfgCh:
+	}
+
+	// Ensure RPCs are still succeeding.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2, and the
+	// new endpoints resource.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server3.Address); err == nil {
+			break
+		}
+		t.Log(err)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to be routed to the new endpoints resource")
+	}
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to an unsupported LB policy. This should result in
+	// failure of RPCs to that cluster.
+	cluster2 = &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						// The type not registered in gRPC Policy registry.
+						TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+							TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist",
+							Value:   &structpb.Struct{},
+						}),
+					},
+				},
+			},
+		},
+	}
+	resources.Clusters[1] = cluster2
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// At this point, RPCs to cluster1 should continue to succeed, while RPCs to
+	// cluster2 should start to fail.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it starts to fail.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		_, err := client.UnaryCall(ctx, &testpb.SimpleRequest{})
+		got := status.Code(err)
+		if got == codes.Unavailable {
+			break
+		}
+		t.Logf("UnaryCall() returned code: %v, want %v", got, codes.Unavailable)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to start failing")
+	}
+
+	// Channel should still be READY.
+	if got, want := cc.GetState(), connectivity.Ready; got != want {
+		t.Fatalf("grpc.ClientConn in state %v, want %v", got, want)
+	}
+}

internal/xds/balancer/clustermanager/picker.go

@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// pickerGroup contains a list of pickers. If the picker isn't ready, the pick
+// will be queued.
+type pickerGroup struct {
+	pickers map[string]balancer.Picker
+}
+
+func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup {
+	pickers := make(map[string]balancer.Picker)
+	for id, st := range idToPickerState {
+		pickers[id] = st.state.Picker
+	}
+	return &pickerGroup{
+		pickers: pickers,
+	}
+}
+
+func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	cluster := PickedCluster(info.Ctx)
+	if p := pg.pickers[cluster]; p != nil {
+		return p.Pick(info)
+	}
+	return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster)
+}
+
+type clusterKey struct{}
+
+// PickedCluster returns the cluster name stored in the context.
+func PickedCluster(ctx context.Context) string {
+	cluster, _ := ctx.Value(clusterKey{}).(string)
+	return cluster
+}
+
+// SetPickedCluster adds the selected cluster to the context for the
+// xds_cluster_manager LB policy to pick.
+func SetPickedCluster(ctx context.Context, cluster string) context.Context {
+	return context.WithValue(ctx, clusterKey{}, cluster)
+}

internal/xds/balancer/loadstore/load_store_wrapper.go

@@ -0,0 +1,128 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package loadstore contains the loadStoreWrapper shared by the balancers.
+package loadstore
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+)
+
+// NewWrapper creates a Wrapper.
+func NewWrapper() *Wrapper {
+	return &Wrapper{}
+}
+
+// Wrapper wraps a load store with cluster and edsService.
+//
+// It's store and cluster/edsService can be updated separately. And it will
+// update its internal perCluster store so that new stats will be added to the
+// correct perCluster.
+//
+// Note that this struct is a temporary workaround before we implement graceful
+// switch for EDS. Any update to the clusterName and serviceName is too early,
+// the perfect timing is when the picker is updated with the new connection.
+// This early update could cause picks for the old SubConn being reported to the
+// new services.
+//
+// When the graceful switch in EDS is done, there should be no need for this
+// struct. The policies that record/report load shouldn't need to handle update
+// of lrsServerName/cluster/edsService. Its parent should do a graceful switch
+// of the whole tree when one of that changes.
+type Wrapper struct {
+	mu         sync.RWMutex
+	cluster    string
+	edsService string
+	// store and perCluster are initialized as nil. They are only set by the
+	// balancer when LRS is enabled. Before that, all functions to record loads
+	// are no-op.
+	store      *lrsclient.LoadStore
+	perCluster *lrsclient.PerClusterReporter
+}
+
+// UpdateClusterAndService updates the cluster name and eds service for this
+// wrapper. If any one of them is changed from before, the perCluster store in
+// this wrapper will also be updated.
+func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if cluster == lsw.cluster && edsService == lsw.edsService {
+		return
+	}
+	lsw.cluster = cluster
+	lsw.edsService = edsService
+	if lsw.store == nil {
+		return
+	}
+	lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService)
+}
+
+// UpdateLoadStore updates the load store for this wrapper. If it is changed
+// from before, the perCluster store in this wrapper will also be updated.
+func (lsw *Wrapper) UpdateLoadStore(store *lrsclient.LoadStore) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	if lsw.store == nil {
+		lsw.perCluster = nil
+		return
+	}
+	lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService)
+}
+
+// CallStarted records a call started in the store.
+func (lsw *Wrapper) CallStarted(locality clients.Locality) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
+}
+
+// CallFinished records a call finished in the store.
+func (lsw *Wrapper) CallFinished(locality clients.Locality, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
+}
+
+// CallServerLoad records the server load in the store.
+func (lsw *Wrapper) CallServerLoad(locality clients.Locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
+}
+
+// CallDropped records a call dropped in the store.
+func (lsw *Wrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
+}

internal/xds/balancer/outlierdetection/balancer.go

@@ -0,0 +1,927 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package outlierdetection provides an implementation of the outlier detection
+// LB policy, as defined in
+// https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md.
+package outlierdetection
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	rand "math/rand/v2"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Globals to stub out in tests.
+var (
+	afterFunc = time.AfterFunc
+	now       = time.Now
+)
+
+// Name is the name of the outlier detection balancer.
+const Name = "outlier_detection_experimental"
+
+var (
+	ejectionsEnforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_enforced",
+		Description: "EXPERIMENTAL. Number of outlier ejections enforced by detection method",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method"},
+		Default:     false,
+	})
+
+	ejectionsUnenforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_unenforced",
+		Description: "EXPERIMENTAL. Number of unenforced outlier ejections due to either `max_ejection_percentage` or `enforcement_percentage`",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method", "grpc.lb.outlier_detection.unenforced_reason"},
+		Default:     false,
+	})
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &outlierDetectionBalancer{
+		ClientConn:      cc,
+		closed:          grpcsync.NewEvent(),
+		done:            grpcsync.NewEvent(),
+		addrs:           make(map[string]*endpointInfo),
+		scUpdateCh:      buffer.NewUnbounded[any](),
+		pickerUpdateCh:  buffer.NewUnbounded[any](),
+		channelzParent:  bOpts.ChannelzParent,
+		endpoints:       resolver.NewEndpointMap[*endpointInfo](),
+		metricsRecorder: cc.MetricsRecorder(), // we use an explicit field instead of using cc.MetricsRecorder() so we can override the metric recorder in tests.
+		target:          bOpts.Target.String(),
+	}
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	b.child = synchronizingBalancerWrapper{lb: gracefulswitch.NewBalancer(b, bOpts)}
+	go b.run()
+	return b
+}
+
+func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	lbCfg := &LBConfig{
+		// Default top layer values as documented in A50.
+		Interval:           iserviceconfig.Duration(10 * time.Second),
+		BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+		MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+		MaxEjectionPercent: 10,
+	}
+
+	// This unmarshalling handles underlying layers sre and fpe which have their
+	// own defaults for their fields if either sre or fpe are present.
+	if err := json.Unmarshal(s, lbCfg); err != nil { // Validates child config if present as well.
+		return nil, fmt.Errorf("xds: unable to unmarshal LBconfig: %s, error: %v", string(s), err)
+	}
+
+	// Note: in the xds flow, these validations will never fail. The xdsclient
+	// performs the same validations as here on the xds Outlier Detection
+	// resource before parsing resource into JSON which this function gets
+	// called with. A50 defines two separate places for these validations to
+	// take place, the xdsclient and this ParseConfig method. "When parsing a
+	// config from JSON, if any of these requirements is violated, that should
+	// be treated as a parsing error." - A50
+	switch {
+	// "The google.protobuf.Duration fields interval, base_ejection_time, and
+	// max_ejection_time must obey the restrictions in the
+	// google.protobuf.Duration documentation and they must have non-negative
+	// values." - A50
+	// Approximately 290 years is the maximum time that time.Duration (int64)
+	// can represent. The restrictions on the protobuf.Duration field are to be
+	// within +-10000 years. Thus, just check for negative values.
+	case lbCfg.Interval < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.interval = %s; must be >= 0", lbCfg.Interval)
+	case lbCfg.BaseEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.base_ejection_time = %s; must be >= 0", lbCfg.BaseEjectionTime)
+	case lbCfg.MaxEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_time = %s; must be >= 0", lbCfg.MaxEjectionTime)
+
+	// "The fields max_ejection_percent,
+	// success_rate_ejection.enforcement_percentage,
+	// failure_percentage_ejection.threshold, and
+	// failure_percentage.enforcement_percentage must have values less than or
+	// equal to 100." - A50
+	case lbCfg.MaxEjectionPercent > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_percent = %v; must be <= 100", lbCfg.MaxEjectionPercent)
+	case lbCfg.SuccessRateEjection != nil && lbCfg.SuccessRateEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = %v; must be <= 100", lbCfg.SuccessRateEjection.EnforcementPercentage)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.Threshold > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = %v; must be <= 100", lbCfg.FailurePercentageEjection.Threshold)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = %v; must be <= 100", lbCfg.FailurePercentageEjection.EnforcementPercentage)
+	}
+	return lbCfg, nil
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+// scUpdate wraps a subConn update to be sent to the child balancer.
+type scUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type ejectionUpdate struct {
+	scw       *subConnWrapper
+	isEjected bool // true for ejected, false for unejected
+}
+
+type lbCfgUpdate struct {
+	lbCfg *LBConfig
+	// to make sure picker is updated synchronously.
+	done chan struct{}
+}
+
+type scHealthUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type outlierDetectionBalancer struct {
+	balancer.ClientConn
+	// These fields are safe to be accessed without holding any mutex because
+	// they are synchronized in run(), which makes these field accesses happen
+	// serially.
+	//
+	// childState is the latest balancer state received from the child.
+	childState balancer.State
+	// recentPickerNoop represents whether the most recent picker sent upward to
+	// the balancer.ClientConn is a noop picker, which doesn't count RPC's. Used
+	// to suppress redundant picker updates.
+	recentPickerNoop bool
+
+	closed          *grpcsync.Event
+	done            *grpcsync.Event
+	logger          *grpclog.PrefixLogger
+	channelzParent  channelz.Identifier
+	metricsRecorder estats.MetricsRecorder
+	target          string
+
+	child synchronizingBalancerWrapper
+
+	// mu guards access to the following fields. It also helps to synchronize
+	// behaviors of the following events: config updates, firing of the interval
+	// timer, SubConn State updates, SubConn address updates, and child state
+	// updates.
+	//
+	// For example, when we receive a config update in the middle of the
+	// interval timer algorithm, which uses knobs present in the config, the
+	// balancer will wait for the interval timer algorithm to finish before
+	// persisting the new configuration.
+	//
+	// Another example would be the updating of the endpoints or addrs map, such
+	// as from a SubConn address update in the middle of the interval timer
+	// algorithm which uses endpoints. This balancer waits for the interval
+	// timer algorithm to finish before making the update to the endpoints map.
+	//
+	// This mutex is never held when calling methods on the child policy
+	// (within the context of a single goroutine).
+	mu sync.Mutex
+	// endpoints stores pointers to endpointInfo objects for each endpoint.
+	endpoints *resolver.EndpointMap[*endpointInfo]
+	// addrs stores pointers to endpointInfo objects for each address. Addresses
+	// belonging to the same endpoint point to the same object.
+	addrs                 map[string]*endpointInfo
+	cfg                   *LBConfig
+	timerStartTime        time.Time
+	intervalTimer         *time.Timer
+	inhibitPickerUpdates  bool
+	updateUnconditionally bool
+	numEndpointsEjected   int // For fast calculations of percentage of endpoints ejected
+
+	scUpdateCh     *buffer.Unbounded[any]
+	pickerUpdateCh *buffer.Unbounded[any]
+}
+
+// noopConfig returns whether this balancer is configured with a logical no-op
+// configuration or not.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) noopConfig() bool {
+	return b.cfg.SuccessRateEjection == nil && b.cfg.FailurePercentageEjection == nil
+}
+
+// onIntervalConfig handles logic required specifically on the receipt of a
+// configuration which specifies to count RPC's and periodically perform passive
+// health checking based on heuristics defined in configuration every configured
+// interval.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onIntervalConfig() {
+	var interval time.Duration
+	if b.timerStartTime.IsZero() {
+		b.timerStartTime = time.Now()
+		for _, epInfo := range b.endpoints.All() {
+			epInfo.callCounter.clear()
+		}
+		interval = time.Duration(b.cfg.Interval)
+	} else {
+		interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime)
+		if interval < 0 {
+			interval = 0
+		}
+	}
+	b.intervalTimer = afterFunc(interval, b.intervalTimerAlgorithm)
+}
+
+// onNoopConfig handles logic required specifically on the receipt of a
+// configuration which specifies the balancer to be a noop.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onNoopConfig() {
+	// "If a config is provided with both the `success_rate_ejection` and
+	// `failure_percentage_ejection` fields unset, skip starting the timer and
+	// do the following:"
+	// "Unset the timer start timestamp."
+	b.timerStartTime = time.Time{}
+	for _, epInfo := range b.endpoints.All() {
+		// "Uneject all currently ejected endpoints."
+		if !epInfo.latestEjectionTimestamp.IsZero() {
+			b.unejectEndpoint(epInfo)
+		}
+		// "Reset each endpoint's ejection time multiplier to 0."
+		epInfo.ejectionTimeMultiplier = 0
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	lbCfg, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		b.logger.Errorf("received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+
+	// Reject whole config if child policy doesn't exist, don't persist it for
+	// later.
+	bb := balancer.Get(lbCfg.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("outlier detection: child balancer %q not registered", lbCfg.ChildPolicy.Name)
+	}
+
+	// It is safe to read b.cfg here without holding the mutex, as the only
+	// write to b.cfg happens later in this function. This function is part of
+	// the balancer.Balancer API, so it is guaranteed to be called in a
+	// synchronous manner, so it cannot race with this read.
+	if b.cfg == nil || b.cfg.ChildPolicy.Name != lbCfg.ChildPolicy.Name {
+		if err := b.child.switchTo(bb); err != nil {
+			return fmt.Errorf("outlier detection: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err)
+		}
+	}
+
+	b.mu.Lock()
+	// Inhibit child picker updates until this UpdateClientConnState() call
+	// completes. If needed, a picker update containing the no-op config bit
+	// determined from this config and most recent state from the child will be
+	// sent synchronously upward at the end of this UpdateClientConnState()
+	// call.
+	b.inhibitPickerUpdates = true
+	b.updateUnconditionally = false
+	b.cfg = lbCfg
+
+	newEndpoints := resolver.NewEndpointMap[bool]()
+	for _, ep := range s.ResolverState.Endpoints {
+		newEndpoints.Set(ep, true)
+		if _, ok := b.endpoints.Get(ep); !ok {
+			b.endpoints.Set(ep, newEndpointInfo())
+		}
+	}
+
+	for ep := range b.endpoints.All() {
+		if _, ok := newEndpoints.Get(ep); !ok {
+			b.endpoints.Delete(ep)
+		}
+	}
+
+	// populate the addrs map.
+	b.addrs = map[string]*endpointInfo{}
+	for _, ep := range s.ResolverState.Endpoints {
+		epInfo, _ := b.endpoints.Get(ep)
+		for _, addr := range ep.Addresses {
+			if _, ok := b.addrs[addr.Addr]; ok {
+				b.logger.Errorf("Endpoints contain duplicate address %q", addr.Addr)
+				continue
+			}
+			b.addrs[addr.Addr] = epInfo
+		}
+	}
+
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+
+	if b.noopConfig() {
+		b.onNoopConfig()
+	} else {
+		b.onIntervalConfig()
+	}
+	b.mu.Unlock()
+
+	err := b.child.updateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: b.cfg.ChildPolicy.Config,
+	})
+
+	done := make(chan struct{})
+	b.pickerUpdateCh.Put(lbCfgUpdate{
+		lbCfg: lbCfg,
+		done:  done,
+	})
+	<-done
+
+	return err
+}
+
+func (b *outlierDetectionBalancer) ResolverError(err error) {
+	b.child.resolverError(err)
+}
+
+func (b *outlierDetectionBalancer) updateSubConnState(scw *subConnWrapper, state balancer.SubConnState) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	scw.setLatestConnectivityState(state.ConnectivityState)
+	b.scUpdateCh.Put(&scUpdate{
+		scw:   scw,
+		state: state,
+	})
+}
+
+func (b *outlierDetectionBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *outlierDetectionBalancer) Close() {
+	b.closed.Fire()
+	<-b.done.Done()
+	b.child.closeLB()
+
+	b.scUpdateCh.Close()
+	b.pickerUpdateCh.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+}
+
+func (b *outlierDetectionBalancer) ExitIdle() {
+	b.child.exitIdle()
+}
+
+// wrappedPicker delegates to the child policy's picker, and when the request
+// finishes, it increments the corresponding counter in the map entry referenced
+// by the subConnWrapper that was picked. If both the `success_rate_ejection`
+// and `failure_percentage_ejection` fields are unset in the configuration, this
+// picker will not count.
+type wrappedPicker struct {
+	childPicker balancer.Picker
+	noopPicker  bool
+}
+
+func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	pr, err := wp.childPicker.Pick(info)
+	if err != nil {
+		return balancer.PickResult{}, err
+	}
+
+	done := func(di balancer.DoneInfo) {
+		if !wp.noopPicker {
+			incrementCounter(pr.SubConn, di)
+		}
+		if pr.Done != nil {
+			pr.Done(di)
+		}
+	}
+	scw, ok := pr.SubConn.(*subConnWrapper)
+	if !ok {
+		// This can never happen, but check is present for defensive
+		// programming.
+		logger.Errorf("Picked SubConn from child picker is not a SubConnWrapper")
+		return balancer.PickResult{
+			SubConn:  pr.SubConn,
+			Done:     done,
+			Metadata: pr.Metadata,
+		}, nil
+	}
+	return balancer.PickResult{
+		SubConn:  scw.SubConn,
+		Done:     done,
+		Metadata: pr.Metadata,
+	}, nil
+}
+
+func incrementCounter(sc balancer.SubConn, info balancer.DoneInfo) {
+	scw, ok := sc.(*subConnWrapper)
+	if !ok {
+		// Shouldn't happen, as comes from child
+		return
+	}
+
+	// After reading callCounter.activeBucket in this picker a swap call can
+	// concurrently change what activeBucket points to. A50 says to swap the
+	// pointer, which will cause this race to write to deprecated memory the
+	// interval timer algorithm will never read, which makes this race alright.
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	ab := epInfo.callCounter.activeBucket.Load()
+
+	if info.Err == nil {
+		atomic.AddUint32(&ab.numSuccesses, 1)
+	} else {
+		atomic.AddUint32(&ab.numFailures, 1)
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateState(s balancer.State) {
+	b.pickerUpdateCh.Put(s)
+}
+
+func (b *outlierDetectionBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	oldListener := opts.StateListener
+	scw := &subConnWrapper{
+		addresses:         addrs,
+		scUpdateCh:        b.scUpdateCh,
+		listener:          oldListener,
+		latestHealthState: balancer.SubConnState{ConnectivityState: connectivity.Connecting},
+	}
+	opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(scw, state) }
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	sc, err := b.ClientConn.NewSubConn(addrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	if len(addrs) != 1 {
+		return scw, nil
+	}
+	epInfo, ok := b.addrs[addrs[0].Addr]
+	if !ok {
+		return scw, nil
+	}
+	epInfo.sws = append(epInfo.sws, scw)
+	scw.endpointInfo = epInfo
+	if !epInfo.latestEjectionTimestamp.IsZero() {
+		scw.eject()
+	}
+	return scw, nil
+}
+
+func (b *outlierDetectionBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+// removeSubConnFromEndpointMapEntry removes the scw from its map entry if
+// present.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) removeSubConnFromEndpointMapEntry(scw *subConnWrapper) {
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	for i, sw := range epInfo.sws {
+		if scw == sw {
+			epInfo.sws = append(epInfo.sws[:i], epInfo.sws[i+1:]...)
+			return
+		}
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// handleSubConnUpdate stores the recent state and forward the update.
+func (b *outlierDetectionBalancer) handleSubConnUpdate(u *scUpdate) {
+	scw := u.scw
+	scw.clearHealthListener()
+	b.child.updateSubConnState(scw, u.state)
+	if u.state.ConnectivityState == connectivity.Shutdown {
+		b.mu.Lock()
+		b.removeSubConnFromEndpointMapEntry(scw)
+		b.mu.Unlock()
+	}
+}
+
+func (b *outlierDetectionBalancer) handleSubConnHealthUpdate(u *scHealthUpdate) {
+	b.child.updateSubConnHealthState(u.scw, u.state)
+}
+
+// handleEjectedUpdate handles any SubConns that get ejected/unejected, and
+// forwards the appropriate corresponding subConnState to the child policy.
+func (b *outlierDetectionBalancer) handleEjectedUpdate(u *ejectionUpdate) {
+	b.child.handleEjectionUpdate(u)
+}
+
+// handleChildStateUpdate forwards the picker update wrapped in a wrapped picker
+// with the noop picker bit present.
+func (b *outlierDetectionBalancer) handleChildStateUpdate(u balancer.State) {
+	b.childState = u
+	b.mu.Lock()
+	if b.inhibitPickerUpdates {
+		// If a child's state is updated during the suppression of child
+		// updates, the synchronous handleLBConfigUpdate function with respect
+		// to UpdateClientConnState should return a picker unconditionally.
+		b.updateUnconditionally = true
+		b.mu.Unlock()
+		return
+	}
+	noopCfg := b.noopConfig()
+	b.mu.Unlock()
+	b.recentPickerNoop = noopCfg
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: b.childState.ConnectivityState,
+		Picker: &wrappedPicker{
+			childPicker: b.childState.Picker,
+			noopPicker:  noopCfg,
+		},
+	})
+}
+
+// handleLBConfigUpdate compares whether the new config is a noop config or not,
+// to the noop bit in the picker if present. It updates the picker if this bit
+// changed compared to the picker currently in use.
+func (b *outlierDetectionBalancer) handleLBConfigUpdate(u lbCfgUpdate) {
+	lbCfg := u.lbCfg
+	noopCfg := lbCfg.SuccessRateEjection == nil && lbCfg.FailurePercentageEjection == nil
+	// If the child has sent its first update and this config flips the noop
+	// bit compared to the most recent picker update sent upward, then a new
+	// picker with this updated bit needs to be forwarded upward. If a child
+	// update was received during the suppression of child updates within
+	// UpdateClientConnState(), then a new picker needs to be forwarded with
+	// this updated state, irregardless of whether this new configuration flips
+	// the bit.
+	if b.childState.Picker != nil && noopCfg != b.recentPickerNoop || b.updateUnconditionally {
+		b.recentPickerNoop = noopCfg
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker: &wrappedPicker{
+				childPicker: b.childState.Picker,
+				noopPicker:  noopCfg,
+			},
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.updateUnconditionally = false
+	close(u.done)
+}
+
+func (b *outlierDetectionBalancer) run() {
+	defer b.done.Fire()
+	for {
+		select {
+		case update, ok := <-b.scUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.scUpdateCh.Load()
+			if b.closed.HasFired() { // don't send SubConn updates to child after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case *scUpdate:
+				b.handleSubConnUpdate(u)
+			case *ejectionUpdate:
+				b.handleEjectedUpdate(u)
+			case *scHealthUpdate:
+				b.handleSubConnHealthUpdate(u)
+			}
+		case update, ok := <-b.pickerUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.pickerUpdateCh.Load()
+			if b.closed.HasFired() { // don't send picker updates to grpc after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case balancer.State:
+				b.handleChildStateUpdate(u)
+			case lbCfgUpdate:
+				b.handleLBConfigUpdate(u)
+			}
+		case <-b.closed.Done():
+			return
+		}
+	}
+}
+
+// intervalTimerAlgorithm ejects and unejects endpoints based on the Outlier
+// Detection configuration and data about each endpoint from the previous
+// interval.
+func (b *outlierDetectionBalancer) intervalTimerAlgorithm() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.timerStartTime = time.Now()
+
+	for _, epInfo := range b.endpoints.All() {
+		epInfo.callCounter.swap()
+	}
+
+	if b.cfg.SuccessRateEjection != nil {
+		b.successRateAlgorithm()
+	}
+
+	if b.cfg.FailurePercentageEjection != nil {
+		b.failurePercentageAlgorithm()
+	}
+
+	for _, epInfo := range b.endpoints.All() {
+		if epInfo.latestEjectionTimestamp.IsZero() && epInfo.ejectionTimeMultiplier > 0 {
+			epInfo.ejectionTimeMultiplier--
+			continue
+		}
+		if epInfo.latestEjectionTimestamp.IsZero() {
+			// Endpoint is already not ejected, so no need to check for whether
+			// to uneject the endpoint below.
+			continue
+		}
+		et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(epInfo.ejectionTimeMultiplier)
+		met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime))
+		uet := epInfo.latestEjectionTimestamp.Add(min(et, met))
+		if now().After(uet) {
+			b.unejectEndpoint(epInfo)
+		}
+	}
+
+	// This conditional only for testing (since the interval timer algorithm is
+	// called manually), will never hit in production.
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+	b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm)
+}
+
+// endpointsWithAtLeastRequestVolume returns a slice of endpoint information of
+// all endpoints with at least request volume passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) endpointsWithAtLeastRequestVolume(requestVolume uint32) []*endpointInfo {
+	var endpoints []*endpointInfo
+	for _, epInfo := range b.endpoints.All() {
+		bucket1 := epInfo.callCounter.inactiveBucket
+		rv := bucket1.numSuccesses + bucket1.numFailures
+		if rv >= requestVolume {
+			endpoints = append(endpoints, epInfo)
+		}
+	}
+	return endpoints
+}
+
+// meanAndStdDev returns the mean and std dev of the fractions of successful
+// requests of the endpoints passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) meanAndStdDev(endpoints []*endpointInfo) (float64, float64) {
+	var totalFractionOfSuccessfulRequests float64
+	var mean float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		totalFractionOfSuccessfulRequests += float64(bucket.numSuccesses) / float64(rv)
+	}
+	mean = totalFractionOfSuccessfulRequests / float64(len(endpoints))
+	var sumOfSquares float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		devFromMean := (float64(bucket.numSuccesses) / float64(rv)) - mean
+		sumOfSquares += devFromMean * devFromMean
+	}
+	variance := sumOfSquares / float64(len(endpoints))
+	return mean, math.Sqrt(variance)
+}
+
+// successRateAlgorithm ejects any endpoints where the success rate falls below
+// the other endpoints according to mean and standard deviation, and if overall
+// applicable from other set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) successRateAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.SuccessRateEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.SuccessRateEjection.MinimumHosts) {
+		return
+	}
+	mean, stddev := b.meanAndStdDev(endpointsToConsider)
+	ejectionCfg := b.cfg.SuccessRateEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures)
+		requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)
+		if successRate < requiredSuccessRate {
+			channelz.Infof(logger, b.channelzParent, "SuccessRate algorithm detected outlier: %s. Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", epInfo, successRate, mean, stddev, requiredSuccessRate)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "success_rate")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// failurePercentageAlgorithm ejects any endpoints where the failure percentage
+// rate exceeds a set enforcement percentage, if overall applicable from other
+// set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) failurePercentageAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.FailurePercentageEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.FailurePercentageEjection.MinimumHosts) {
+		return
+	}
+
+	ejectionCfg := b.cfg.FailurePercentageEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100
+		if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) {
+			channelz.Infof(logger, b.channelzParent, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", epInfo, failurePercentage)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "failure_percentage")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) ejectEndpoint(epInfo *endpointInfo, detectionMethod string) {
+	b.numEndpointsEjected++
+	epInfo.latestEjectionTimestamp = b.timerStartTime
+	epInfo.ejectionTimeMultiplier++
+	for _, sbw := range epInfo.sws {
+		sbw.eject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel ejected: %s", sbw)
+	}
+
+	// Record the enforced ejection metric.
+	ejectionsEnforcedMetric.Record(b.metricsRecorder, 1, b.target, detectionMethod)
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) unejectEndpoint(epInfo *endpointInfo) {
+	b.numEndpointsEjected--
+	epInfo.latestEjectionTimestamp = time.Time{}
+	for _, sbw := range epInfo.sws {
+		sbw.uneject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel unejected: %s", sbw)
+	}
+}
+
+// synchronizingBalancerWrapper serializes calls into balancer (to uphold the
+// balancer.Balancer API guarantee of synchronous calls). It also ensures a
+// consistent order of locking mutexes when using SubConn listeners to avoid
+// deadlocks.
+type synchronizingBalancerWrapper struct {
+	// mu should not be used directly from outside this struct, instead use
+	// methods defined on the struct.
+	mu sync.Mutex
+	lb *gracefulswitch.Balancer
+}
+
+func (sbw *synchronizingBalancerWrapper) switchTo(builder balancer.Builder) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.SwitchTo(builder)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateClientConnState(state balancer.ClientConnState) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.UpdateClientConnState(state)
+}
+
+func (sbw *synchronizingBalancerWrapper) resolverError(err error) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ResolverError(err)
+}
+
+func (sbw *synchronizingBalancerWrapper) closeLB() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.Close()
+}
+
+func (sbw *synchronizingBalancerWrapper) exitIdle() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ExitIdle()
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnHealthState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnHealthState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnConnectivityState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) handleEjectionUpdate(u *ejectionUpdate) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	if u.isEjected {
+		u.scw.handleEjection()
+	} else {
+		u.scw.handleUnejection()
+	}
+}
+
+// endpointInfo contains the runtime information about an endpoint that pertains
+// to Outlier Detection. This struct and all of its fields is protected by
+// outlierDetectionBalancer.mu in the case where it is accessed through the
+// address or endpoint map. In the case of Picker callbacks, the writes to the
+// activeBucket of callCounter are protected by atomically loading and storing
+// unsafe.Pointers (see further explanation in incrementCounter()).
+type endpointInfo struct {
+	// The call result counter object.
+	callCounter *callCounter
+
+	// The latest ejection timestamp, or zero if the endpoint is currently not
+	// ejected.
+	latestEjectionTimestamp time.Time
+
+	// The current ejection time multiplier, starting at 0.
+	ejectionTimeMultiplier int64
+
+	// A list of subchannel wrapper objects that correspond to this endpoint.
+	sws []*subConnWrapper
+}
+
+func (a *endpointInfo) String() string {
+	var res strings.Builder
+	res.WriteString("[")
+	for _, sw := range a.sws {
+		res.WriteString(sw.String())
+	}
+	res.WriteString("]")
+	return res.String()
+}
+
+func newEndpointInfo() *endpointInfo {
+	return &endpointInfo{
+		callCounter: newCallCounter(),
+	}
+}

internal/xds/balancer/outlierdetection/balancer_ext_test.go

@@ -0,0 +1,518 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection_test
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestParseConfig verifies the ParseConfig() method in the Outlier Detection
+// Balancer.
+func (s) TestParseConfig(t *testing.T) {
+	const errParseConfigName = "errParseConfigBalancer"
+	stub.Register(errParseConfigName, stub.BalancerFuncs{
+		ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return nil, errors.New("some error")
+		},
+	})
+
+	parser := balancer.Get(outlierdetection.Name).(balancer.ConfigParser)
+	const (
+		defaultInterval                       = iserviceconfig.Duration(10 * time.Second)
+		defaultBaseEjectionTime               = iserviceconfig.Duration(30 * time.Second)
+		defaultMaxEjectionTime                = iserviceconfig.Duration(300 * time.Second)
+		defaultMaxEjectionPercent             = 10
+		defaultSuccessRateStdevFactor         = 1900
+		defaultEnforcingSuccessRate           = 100
+		defaultSuccessRateMinimumHosts        = 5
+		defaultSuccessRateRequestVolume       = 100
+		defaultFailurePercentageThreshold     = 85
+		defaultEnforcingFailurePercentage     = 0
+		defaultFailurePercentageMinimumHosts  = 5
+		defaultFailurePercentageRequestVolume = 50
+	)
+	tests := []struct {
+		name    string
+		input   string
+		wantCfg serviceconfig.LoadBalancingConfig
+		wantErr string
+	}{
+		{
+			name: "Default",
+			input: `{
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+
+		{
+			name: "TopLevelSet",
+			input: `{
+				"interval": "15s",
+				"maxEjectionTime": "350s",
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for unset fields.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(15 * time.Second),
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    iserviceconfig.Duration(350 * time.Second),
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Defaults",
+			input: `{
+				"successRateEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of success-rate-ejection struct.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           defaultSuccessRateStdevFactor,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          defaultSuccessRateMinimumHosts,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Partial",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"minimumHosts": 5
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          5,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Full",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"enforcementPercentage": 50,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: 50,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Defaults",
+			input: `{
+				"failurePercentageEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of failure percentage ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             defaultFailurePercentageThreshold,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          defaultFailurePercentageMinimumHosts,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Partial",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"minimumHosts": 10
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          10,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Full",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"enforcementPercentage": 100,
+					"minimumHosts": 10,
+					"requestVolume": 40
+                },
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: 100,
+					MinimumHosts:          10,
+					RequestVolume:         40,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{ // to make sure zero values aren't overwritten by defaults
+			name: "AllFields_Zero",
+			input: `{
+				"interval": "0s",
+				"baseEjectionTime": "0s",
+				"maxEjectionTime": "0s",
+				"maxEjectionPercent": 0,
+				"successRateEjection": {
+					"stdevFactor": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+				"failurePercentageEjection": {
+					"threshold": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				SuccessRateEjection:       &outlierdetection.SuccessRateEjection{},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "AllFields_Set",
+			input: `{
+				"interval": "10s",
+				"baseEjectionTime": "30s",
+				"maxEjectionTime": "300s",
+				"maxEjectionPercent": 10,
+				"successRateEjection": {
+					"stdevFactor": 1900,
+					"enforcementPercentage": 100,
+					"minimumHosts": 5,
+					"requestVolume": 100
+				},
+				"failurePercentageEjection": {
+					"threshold": 85,
+					"enforcementPercentage": 5,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(10 * time.Second),
+				BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+				MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+				MaxEjectionPercent: 10,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1900,
+					EnforcementPercentage: 100,
+					MinimumHosts:          5,
+					RequestVolume:         100,
+				},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             85,
+					EnforcementPercentage: 5,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name:    "Interval_Negative",
+			input:   `{"interval": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0",
+		},
+		{
+			name:    "BaseEjectionTime_Negative",
+			input:   `{"baseEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionTime_Negative",
+			input:   `{"maxEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionPercent_TooHigh",
+			input:   `{"maxEjectionPercent": 150}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_percent = 150; must be <= 100",
+		},
+		{
+			name: "SuccessRate_Enforcement_TooHigh",
+			input: `{
+				"successRateEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Threshold_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Enforcement_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "ChildPolicy_ParseError",
+			input: `{
+				"childPolicy": [
+				{
+					"errParseConfigBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"",
+		},
+		{
+			name: "ChildPolicy_NotSupported",
+			input: `{
+				"childPolicy": [
+				{
+					"doesNotExistBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input))
+			if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if (gotErr != nil) != (test.wantErr != "") {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" {
+				t.Fatalf("parseConfig(%v) got unexpected output, diff (-got +want): %v", string(test.input), diff)
+			}
+		})
+	}
+}

internal/xds/balancer/outlierdetection/callcounter.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"sync/atomic"
+)
+
+type bucket struct {
+	numSuccesses uint32
+	numFailures  uint32
+}
+
+func newCallCounter() *callCounter {
+	cc := &callCounter{
+		inactiveBucket: &bucket{},
+	}
+	cc.activeBucket.Store(&bucket{})
+	return cc
+}
+
+// callCounter has two buckets, which each count successful and failing RPC's.
+// The activeBucket is used to actively count any finished RPC's, and the
+// inactiveBucket is populated with this activeBucket's data every interval for
+// use by the Outlier Detection algorithm.
+type callCounter struct {
+	// activeBucket updates every time a call finishes (from picker passed to
+	// Client Conn), so protect pointer read with atomic load of the pointer
+	// so picker does not have to grab a mutex per RPC, the critical path.
+	activeBucket   atomic.Pointer[bucket]
+	inactiveBucket *bucket
+}
+
+func (cc *callCounter) clear() {
+	cc.activeBucket.Store(&bucket{})
+	cc.inactiveBucket = &bucket{}
+}
+
+// "When the timer triggers, the inactive bucket is zeroed and swapped with the
+// active bucket. Then the inactive bucket contains the number of successes and
+// failures since the last time the timer triggered. Those numbers are used to
+// evaluate the ejection criteria." - A50.
+func (cc *callCounter) swap() {
+	ib := cc.inactiveBucket
+	*ib = bucket{}
+	ab := cc.activeBucket.Swap(ib)
+	cc.inactiveBucket = &bucket{
+		numSuccesses: atomic.LoadUint32(&ab.numSuccesses),
+		numFailures:  atomic.LoadUint32(&ab.numFailures),
+	}
+}

internal/xds/balancer/outlierdetection/callcounter_test.go

@@ -0,0 +1,92 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func (b1 *bucket) Equal(b2 *bucket) bool {
+	if b1 == nil && b2 == nil {
+		return true
+	}
+	if (b1 != nil) != (b2 != nil) {
+		return false
+	}
+	if b1.numSuccesses != b2.numSuccesses {
+		return false
+	}
+	return b1.numFailures == b2.numFailures
+}
+
+func (cc *callCounter) Equal(cc2 *callCounter) bool {
+	if cc == nil && cc2 == nil {
+		return true
+	}
+	if (cc != nil) != (cc2 != nil) {
+		return false
+	}
+	ab1 := cc.activeBucket.Load()
+	ab2 := cc2.activeBucket.Load()
+	if !ab1.Equal(ab2) {
+		return false
+	}
+	return cc.inactiveBucket.Equal(cc2.inactiveBucket)
+}
+
+// TestClear tests that clear on the call counter clears (everything set to 0)
+// the active and inactive buckets.
+func (s) TestClear(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	cc.clear()
+	// Both the active and inactive buckets should be cleared.
+	ccWant := newCallCounter()
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}
+
+// TestSwap tests that swap() on the callCounter successfully has the desired
+// end result of inactive bucket containing the previous active buckets data,
+// and the active bucket being cleared.
+func (s) TestSwap(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	ib := cc.inactiveBucket
+	cc.swap()
+	// Inactive should pick up active's data, active should be swapped to zeroed
+	// inactive.
+	ccWant := newCallCounter()
+	ccWant.inactiveBucket.numSuccesses = 1
+	ccWant.inactiveBucket.numFailures = 2
+	ccWant.activeBucket.Store(ib)
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}

internal/xds/balancer/outlierdetection/config.go

@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"encoding/json"
+	"time"
+
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// SuccessRateEjection is parameters for the success rate ejection algorithm.
+// This algorithm monitors the request success rate for all endpoints and ejects
+// individual endpoints whose success rates are statistical outliers.
+type SuccessRateEjection struct {
+	// StddevFactor is used to determine the ejection threshold for
+	// success rate outlier ejection. The ejection threshold is the difference
+	// between the mean success rate, and the product of this factor and the
+	// standard deviation of the mean success rate: mean - (stdev *
+	// success_rate_stdev_factor). This factor is divided by a thousand to get a
+	// double. That is, if the desired factor is 1.9, the runtime value should
+	// be 1900. Defaults to 1900.
+	StdevFactor uint32 `json:"stdevFactor,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually ejected
+	// when an outlier status is detected through success rate statistics. This
+	// setting can be used to disable ejection or to ramp it up slowly. Defaults
+	// to 100.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the number of hosts in a cluster that must have enough
+	// request volume to detect success rate outliers. If the number of hosts is
+	// less than this setting, outlier detection via success rate statistics is
+	// not performed for any host in the cluster. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// include this host in success rate based outlier detection. If the volume
+	// is lower than this setting, outlier detection via success rate statistics
+	// is not performed for that host. Defaults to 100.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type successRateEjection SuccessRateEjection
+
+// UnmarshalJSON unmarshals JSON into SuccessRateEjection. If a
+// SuccessRateEjection field is not set, that field will get its default value.
+func (sre *SuccessRateEjection) UnmarshalJSON(j []byte) error {
+	sre.StdevFactor = 1900
+	sre.EnforcementPercentage = 100
+	sre.MinimumHosts = 5
+	sre.RequestVolume = 100
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*successRateEjection)(sre))
+}
+
+// Equal returns whether the SuccessRateEjection is the same with the parameter.
+func (sre *SuccessRateEjection) Equal(sre2 *SuccessRateEjection) bool {
+	if sre == nil && sre2 == nil {
+		return true
+	}
+	if (sre != nil) != (sre2 != nil) {
+		return false
+	}
+	if sre.StdevFactor != sre2.StdevFactor {
+		return false
+	}
+	if sre.EnforcementPercentage != sre2.EnforcementPercentage {
+		return false
+	}
+	if sre.MinimumHosts != sre2.MinimumHosts {
+		return false
+	}
+	return sre.RequestVolume == sre2.RequestVolume
+}
+
+// FailurePercentageEjection is parameters for the failure percentage algorithm.
+// This algorithm ejects individual endpoints whose failure rate is greater than
+// some threshold, independently of any other endpoint.
+type FailurePercentageEjection struct {
+	// Threshold is the failure percentage to use when determining failure
+	// percentage-based outlier detection. If the failure percentage of a given
+	// host is greater than or equal to this value, it will be ejected. Defaults
+	// to 85.
+	Threshold uint32 `json:"threshold,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually
+	// ejected when an outlier status is detected through failure percentage
+	// statistics. This setting can be used to disable ejection or to ramp it up
+	// slowly. Defaults to 0.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the minimum number of hosts in a cluster in order to
+	// perform failure percentage-based ejection. If the total number of hosts
+	// in the cluster is less than this value, failure percentage-based ejection
+	// will not be performed. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// perform failure percentage-based ejection for this host. If the volume is
+	// lower than this setting, failure percentage-based ejection will not be
+	// performed for this host. Defaults to 50.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type failurePercentageEjection FailurePercentageEjection
+
+// UnmarshalJSON unmarshals JSON into FailurePercentageEjection. If a
+// FailurePercentageEjection field is not set, that field will get its default
+// value.
+func (fpe *FailurePercentageEjection) UnmarshalJSON(j []byte) error {
+	fpe.Threshold = 85
+	fpe.EnforcementPercentage = 0
+	fpe.MinimumHosts = 5
+	fpe.RequestVolume = 50
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*failurePercentageEjection)(fpe))
+}
+
+// Equal returns whether the FailurePercentageEjection is the same with the
+// parameter.
+func (fpe *FailurePercentageEjection) Equal(fpe2 *FailurePercentageEjection) bool {
+	if fpe == nil && fpe2 == nil {
+		return true
+	}
+	if (fpe != nil) != (fpe2 != nil) {
+		return false
+	}
+	if fpe.Threshold != fpe2.Threshold {
+		return false
+	}
+	if fpe.EnforcementPercentage != fpe2.EnforcementPercentage {
+		return false
+	}
+	if fpe.MinimumHosts != fpe2.MinimumHosts {
+		return false
+	}
+	return fpe.RequestVolume == fpe2.RequestVolume
+}
+
+// LBConfig is the config for the outlier detection balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+	// Interval is the time interval between ejection analysis sweeps. This can
+	// result in both new ejections as well as addresses being returned to
+	// service. Defaults to 10s.
+	Interval iserviceconfig.Duration `json:"interval,omitempty"`
+	// BaseEjectionTime is the base time that a host is ejected for. The real
+	// time is equal to the base time multiplied by the number of times the host
+	// has been ejected and is capped by MaxEjectionTime. Defaults to 30s.
+	BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"`
+	// MaxEjectionTime is the maximum time that an endpoint is ejected for. If
+	// not specified, the default value (300s) or the BaseEjectionTime value is
+	// applied, whichever is larger.
+	MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"`
+	// MaxEjectionPercent is the maximum % of an upstream cluster that can be
+	// ejected due to outlier detection. Defaults to 10% but will eject at least
+	// one host regardless of the value.
+	MaxEjectionPercent uint32 `json:"maxEjectionPercent,omitempty"`
+	// SuccessRateEjection is the parameters for the success rate ejection
+	// algorithm. If set, success rate ejections will be performed.
+	SuccessRateEjection *SuccessRateEjection `json:"successRateEjection,omitempty"`
+	// FailurePercentageEjection is the parameters for the failure percentage
+	// algorithm. If set, failure rate ejections will be performed.
+	FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"`
+	// ChildPolicy is the config for the child policy.
+	ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type lbConfig LBConfig
+
+// UnmarshalJSON unmarshals JSON into LBConfig. If a top level LBConfig field
+// (i.e. not next layer sre or fpe) is not set, that field will get its default
+// value. If sre or fpe is not set, it will stay unset, otherwise it will
+// unmarshal on those types populating with default values for their fields if
+// needed.
+func (lbc *LBConfig) UnmarshalJSON(j []byte) error {
+	// Default top layer values as documented in A50.
+	lbc.Interval = iserviceconfig.Duration(10 * time.Second)
+	lbc.BaseEjectionTime = iserviceconfig.Duration(30 * time.Second)
+	lbc.MaxEjectionTime = iserviceconfig.Duration(300 * time.Second)
+	lbc.MaxEjectionPercent = 10
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*lbConfig)(lbc))
+}
+
+// EqualIgnoringChildPolicy returns whether the LBConfig is same with the
+// parameter outside of the child policy, only comparing the Outlier Detection
+// specific configuration.
+func (lbc *LBConfig) EqualIgnoringChildPolicy(lbc2 *LBConfig) bool {
+	if lbc == nil && lbc2 == nil {
+		return true
+	}
+	if (lbc != nil) != (lbc2 != nil) {
+		return false
+	}
+	if lbc.Interval != lbc2.Interval {
+		return false
+	}
+	if lbc.BaseEjectionTime != lbc2.BaseEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionTime != lbc2.MaxEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionPercent != lbc2.MaxEjectionPercent {
+		return false
+	}
+	if !lbc.SuccessRateEjection.Equal(lbc2.SuccessRateEjection) {
+		return false
+	}
+	return lbc.FailurePercentageEjection.Equal(lbc2.FailurePercentageEjection)
+}

internal/xds/balancer/outlierdetection/config_test.go

@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSuccessRateEjection(t *testing.T) {
+	fields := map[string]bool{
+		"StdevFactor":           true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(SuccessRateEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in SuccessRateEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsFailurePercentageEjection(t *testing.T) {
+	fields := map[string]bool{
+		"Threshold":             true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(FailurePercentageEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in FailurePercentageEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsLBConfig(t *testing.T) {
+	fields := map[string]bool{
+		"LoadBalancingConfig":       true,
+		"Interval":                  true,
+		"BaseEjectionTime":          true,
+		"MaxEjectionTime":           true,
+		"MaxEjectionPercent":        true,
+		"SuccessRateEjection":       true,
+		"FailurePercentageEjection": true,
+		"ChildPolicy":               true,
+	}
+	typ := reflect.TypeOf(LBConfig{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in LBConfig %q, update this test and EqualIgnoringChildPolicy", n)
+		}
+	}
+}

internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go

@@ -0,0 +1,425 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e_test contains e2e test cases for the Outlier Detection LB Policy.
+package e2e_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // To register helper functions which register/unregister Outlier Detection LB Policy.
+)
+
+var (
+	defaultTestTimeout = 5 * time.Second
+	leafPolicyName     = "round_robin"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// Setup spins up three test backends, each listening on a port on localhost.
+// Two of the backends are configured to always reply with an empty response and
+// no error and one is configured to always return an error.
+func setupBackends(t *testing.T) ([]string, func()) {
+	t.Helper()
+
+	backends := make([]*stubserver.StubServer, 3)
+	addresses := make([]string, 3)
+	// Construct and start 2 working backends.
+	for i := 0; i < 2; i++ {
+		backend := &stubserver.StubServer{
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+				return &testpb.Empty{}, nil
+			},
+		}
+		if err := backend.StartServer(); err != nil {
+			t.Fatalf("Failed to start backend: %v", err)
+		}
+		t.Logf("Started good TestService backend at: %q", backend.Address)
+		backends[i] = backend
+		addresses[i] = backend.Address
+	}
+
+	// Construct and start a failing backend.
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started bad TestService backend at: %q", backend.Address)
+	backends[2] = backend
+	addresses[2] = backend.Address
+	cancel := func() {
+		for _, backend := range backends {
+			backend.Stop()
+		}
+	}
+	return addresses, cancel
+}
+
+// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn,
+// connected to a server exposing the test.grpc_testing.TestService, are
+// roundrobined across the given backend addresses.
+//
+// Returns a non-nil error if context deadline expires before RPCs start to get
+// roundrobined across the given backends.
+func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error {
+	wantAddrCount := make(map[string]int)
+	for _, addr := range addrs {
+		wantAddrCount[addr.Addr]++
+	}
+	gotAddrCount := make(map[string]int)
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		gotAddrCount = make(map[string]int)
+		// Perform 3 iterations.
+		var iterations [][]string
+		for i := 0; i < 3; i++ {
+			iteration := make([]string, len(addrs))
+			for c := 0; c < len(addrs); c++ {
+				var peer peer.Peer
+				client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer))
+				if peer.Addr != nil {
+					iteration[c] = peer.Addr.String()
+				}
+			}
+			iterations = append(iterations, iteration)
+		}
+		// Ensure the first iteration contains all addresses in addrs.
+		for _, addr := range iterations[0] {
+			gotAddrCount[addr]++
+		}
+		if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" {
+			continue
+		}
+		// Ensure all three iterations contain the same addresses.
+		if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) {
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount)
+}
+
+// TestPickFirstHealthListenerDisabled verifies that outlier detection doesn't
+// eject subchannels created by pick_first when it is not under a petiole
+// policy. In that configuration, pick_first does not register a health
+// listener and ejection updates must not affect connectivity.
+func (s) TestPickFirstHealthListenerDisabled(t *testing.T) {
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	defer backend.Stop()
+
+	const serviceConfig = `{
+	  "loadBalancingConfig": [{
+		"outlier_detection_experimental": {
+		  "interval": "0.050s",
+		  "failurePercentageEjection": {
+			"threshold": 50,
+			"enforcementPercentage": 100,
+			"minimumHosts": 0,
+			"requestVolume": 2
+		  },
+		  "childPolicy": [{"pick_first": {}}]
+		}
+	  }]
+	}`
+	cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(serviceConfig))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	client.EmptyCall(ctx, &testpb.Empty{})
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	for range 10 {
+		client.EmptyCall(ctx, &testpb.Empty{})
+	}
+
+	// Wait for three outlier detection intervals to verify that the subchannel is not ejected.
+	sCtx, sCancel := context.WithTimeout(ctx, 150*time.Millisecond)
+	defer sCancel()
+	testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready)
+}
+
+// TestOutlierDetectionAlgorithmsE2E tests the Outlier Detection Success Rate
+// and Failure Percentage algorithms in an e2e fashion. The Outlier Detection
+// Balancer is configured as the top level LB Policy of the channel with a Round
+// Robin child, and connects to three upstreams. Two of the upstreams are healthy and
+// one is unhealthy. The two algorithms should at some point eject the failing
+// upstream, causing RPC's to not be routed to that upstream, and only be
+// Round Robined across the two healthy upstreams. Other than the intervals the
+// unhealthy upstream is ejected, RPC's should regularly round robin
+// across all three upstreams.
+func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) {
+	tests := []struct {
+		name     string
+		odscJSON string
+	}{
+		{
+			name: "Success Rate Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"successRateEjection": {
+						"stdevFactor": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+		{
+			name: "Failure Percentage Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"failurePercentageEjection": {
+						"threshold": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}
+					]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			addresses, cancel := setupBackends(t)
+			defer cancel()
+
+			mr := manual.NewBuilderWithScheme("od-e2e")
+			defer mr.Close()
+
+			sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(test.odscJSON)
+			// The full list of addresses.
+			fullAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+				{Addr: addresses[2]},
+			}
+			mr.InitialState(resolver.State{
+				Addresses:     fullAddresses,
+				ServiceConfig: sc,
+			})
+
+			cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+			// At first, due to no statistics on each of the backends, the 3
+			// upstreams should all be round robined across.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The addresses which don't return errors.
+			okAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+			}
+			// After calling the three upstreams, one of them constantly error
+			// and should eventually be ejected for a period of time. This
+			// period of time should cause the RPC's to be round robined only
+			// across the two that are healthy.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The failing upstream isn't ejected indefinitely, and eventually
+			// should be unejected in subsequent iterations of the interval
+			// algorithm as per the spec for the two specific algorithms.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+		})
+	}
+}
+
+// TestNoopConfiguration tests the Outlier Detection Balancer configured with a
+// noop configuration. The noop configuration should cause the Outlier Detection
+// Balancer to not count RPC's, and thus never eject any upstreams and continue
+// to route to every upstream connected to, even if they continuously error.
+// Once the Outlier Detection Balancer gets reconfigured with configuration
+// requiring counting RPC's, the Outlier Detection Balancer should start
+// ejecting any upstreams as specified in the configuration.
+func (s) TestNoopConfiguration(t *testing.T) {
+	addresses, cancel := setupBackends(t)
+	defer cancel()
+
+	mr := manual.NewBuilderWithScheme("od-e2e")
+	defer mr.Close()
+
+	noopODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(noopODServiceConfigJSON)
+	// The full list of addresses.
+	fullAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+		{Addr: addresses[2]},
+	}
+	mr.InitialState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+	for i := 0; i < 2; i++ {
+		// Since the Outlier Detection Balancer starts with a noop
+		// configuration, it shouldn't count RPCs or eject any upstreams. Thus,
+		// even though an upstream it connects to constantly errors, it should
+		// continue to Round Robin across every upstream.
+		if err := checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+			t.Fatalf("error in expected round robin: %v", err)
+		}
+	}
+
+	// Reconfigure the Outlier Detection Balancer with a configuration that
+	// specifies to count RPC's and eject upstreams. Due to the balancer no
+	// longer being a noop, it should eject any unhealthy addresses as specified
+	// by the failure percentage portion of the configuration.
+	countingODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"failurePercentageEjection": {
+				"threshold": 50,
+				"enforcementPercentage": 100,
+				"minimumHosts": 3,
+				"requestVolume": 5
+			},
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(countingODServiceConfigJSON)
+
+	mr.UpdateState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+
+	// At first on the reconfigured balancer, the balancer has no stats
+	// collected about upstreams. Thus, it should at first route across the full
+	// upstream list.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+	}
+	// Now that the reconfigured balancer has data about the failing upstream,
+	// it should eject the upstream and only route across the two healthy
+	// upstreams.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}

internal/xds/balancer/outlierdetection/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[outlier-detection-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *outlierDetectionBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/outlierdetection/subconn_wrapper.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/resolver"
+)
+
+// subConnWrapper wraps every created SubConn in the Outlier Detection Balancer,
+// to help track the latest state update from the underlying SubConn, and also
+// whether or not this SubConn is ejected.
+type subConnWrapper struct {
+	balancer.SubConn
+	// endpointInfo is a pointer to the subConnWrapper's corresponding endpoint
+	// map entry, if the map entry exists.
+	endpointInfo *endpointInfo
+	// The following fields are set during object creation and read-only after
+	// that.
+
+	listener func(balancer.SubConnState)
+
+	scUpdateCh *buffer.Unbounded[any]
+
+	// The following fields are only referenced in the context of a work
+	// serializing buffer and don't need to be protected by a mutex.
+
+	// These two pieces of state will reach eventual consistency due to sync in
+	// run(), and child will always have the correctly updated SubConnState.
+
+	ejected bool
+
+	// addresses is the list of address(es) this SubConn was created with to
+	// help support any change in address(es)
+	addresses []resolver.Address
+	// latestHealthState is tracked to update the child policy during
+	// unejection.
+	latestHealthState balancer.SubConnState
+
+	// Access to the following fields are protected by a mutex. These fields
+	// should not be accessed from outside this file, instead use methods
+	// defined on the struct.
+	mu             sync.Mutex
+	healthListener func(balancer.SubConnState)
+	// latestReceivedConnectivityState is the SubConn's most recent connectivity
+	// state received. It may not be delivered to the child balancer yet. It is
+	// used to ensure a health listener is registered with the SubConn only when
+	// the SubConn is READY.
+	latestReceivedConnectivityState connectivity.State
+}
+
+// eject causes the wrapper to report a state update with the TRANSIENT_FAILURE
+// state, and to stop passing along updates from the underlying subchannel.
+func (scw *subConnWrapper) eject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: true,
+	})
+}
+
+// uneject causes the wrapper to report a state update with the latest update
+// from the underlying subchannel, and resume passing along updates from the
+// underlying subchannel.
+func (scw *subConnWrapper) uneject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: false,
+	})
+}
+
+func (scw *subConnWrapper) String() string {
+	return fmt.Sprintf("%+v", scw.addresses)
+}
+
+func (scw *subConnWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) {
+	// gRPC currently supports two mechanisms that provide a health signal for
+	// a connection: client-side health checking and outlier detection. Earlier
+	// both these mechanisms signaled unhealthiness by setting the subchannel
+	// state to TRANSIENT_FAILURE. As part of the dualstack changes to make
+	// pick_first the universal leaf policy (see A61), both these mechanisms
+	// started using the new health listener to make health signal visible to
+	// the petiole policies without affecting the underlying connectivity
+	// management of the pick_first policy.
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+
+	if scw.latestReceivedConnectivityState != connectivity.Ready {
+		return
+	}
+	scw.healthListener = listener
+	if listener == nil {
+		scw.SubConn.RegisterHealthListener(nil)
+		return
+	}
+
+	scw.SubConn.RegisterHealthListener(func(scs balancer.SubConnState) {
+		scw.scUpdateCh.Put(&scHealthUpdate{
+			scw:   scw,
+			state: scs,
+		})
+	})
+}
+
+// updateSubConnHealthState stores the latest health state for unejection and
+// sends updates the health listener.
+func (scw *subConnWrapper) updateSubConnHealthState(scs balancer.SubConnState) {
+	scw.latestHealthState = scs
+	if scw.ejected {
+		return
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(scs)
+	}
+}
+
+// updateSubConnConnectivityState stores the latest connectivity state for
+// unejection and updates the raw connectivity listener.
+func (scw *subConnWrapper) updateSubConnConnectivityState(scs balancer.SubConnState) {
+	if scw.listener != nil {
+		scw.listener(scs)
+	}
+}
+
+func (scw *subConnWrapper) clearHealthListener() {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.healthListener = nil
+}
+
+func (scw *subConnWrapper) handleUnejection() {
+	scw.ejected = false
+	// If scw.latestHealthState has never been written to will use the health
+	// state CONNECTING set during object creation.
+	scw.updateSubConnHealthState(scw.latestHealthState)
+}
+
+func (scw *subConnWrapper) handleEjection() {
+	scw.ejected = true
+	stateToUpdate := balancer.SubConnState{
+		ConnectivityState: connectivity.TransientFailure,
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(stateToUpdate)
+	}
+}
+
+func (scw *subConnWrapper) setLatestConnectivityState(state connectivity.State) {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.latestReceivedConnectivityState = state
+}

internal/xds/balancer/priority/balancer.go

@@ -0,0 +1,292 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package priority implements the priority balancer.
+//
+// This balancer will be kept in internal until we use it in the xds balancers,
+// and are confident its functionalities are stable. It will then be exported
+// for more users.
+package priority
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancergroup"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Name is the name of the priority balancer.
+const Name = "priority_experimental"
+
+// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for
+// testing.
+var DefaultSubBalancerCloseTimeout = 15 * time.Minute
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &priorityBalancer{
+		cc:                       cc,
+		done:                     grpcsync.NewEvent(),
+		children:                 make(map[string]*childBalancer),
+		childBalancerStateUpdate: buffer.NewUnbounded[any](),
+	}
+
+	b.logger = prefixLogger(b)
+	b.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               bOpts,
+		StateAggregator:         b,
+		Logger:                  b.logger,
+		SubBalancerCloseTimeout: DefaultSubBalancerCloseTimeout,
+	})
+	go b.run()
+	b.logger.Infof("Created")
+	return b
+}
+
+func (b bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(s)
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+// timerWrapper wraps a timer with a boolean. So that when a race happens
+// between AfterFunc and Stop, the func is guaranteed to not execute.
+type timerWrapper struct {
+	stopped bool
+	timer   *time.Timer
+}
+
+type priorityBalancer struct {
+	logger                   *grpclog.PrefixLogger
+	cc                       balancer.ClientConn
+	bg                       *balancergroup.BalancerGroup
+	done                     *grpcsync.Event
+	childBalancerStateUpdate *buffer.Unbounded[any]
+
+	mu         sync.Mutex
+	childInUse string
+	// priorities is a list of child names from higher to lower priority.
+	priorities []string
+	// children is a map from child name to sub-balancers.
+	children map[string]*childBalancer
+
+	// Set during UpdateClientConnState when calling into sub-balancers.
+	// Prevents child updates from recomputing the active priority or sending
+	// an update of the aggregated picker to the parent.  Cleared after all
+	// sub-balancers have finished UpdateClientConnState, after which
+	// syncPriority is called manually.
+	inhibitPickerUpdates bool
+}
+
+func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	if b.logger.V(2) {
+		b.logger.Infof("Received an update with balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
+	}
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints)
+
+	b.mu.Lock()
+	// Create and remove children, since we know all children from the config
+	// are used by some priority.
+	for name, newSubConfig := range newConfig.Children {
+		bb := balancer.Get(newSubConfig.Config.Name)
+		if bb == nil {
+			b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name)
+			continue
+		}
+
+		currentChild, ok := b.children[name]
+		if !ok {
+			// This is a new child, add it to the children list. But note that
+			// the balancer isn't built, because this child can be a low
+			// priority. If necessary, it will be built when syncing priorities.
+			cb := newChildBalancer(name, b, bb.Name(), b.cc)
+			cb.updateConfig(newSubConfig, resolver.State{
+				Endpoints:     endpointsSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			})
+			b.children[name] = cb
+			continue
+		}
+
+		// This is not a new child. But the config/addresses could change.
+
+		// The balancing policy name is changed, close the old child. But don't
+		// rebuild, rebuild will happen when syncing priorities.
+		if currentChild.balancerName != bb.Name() {
+			currentChild.stop(true)
+			currentChild.updateBalancerName(bb.Name())
+		}
+
+		// Update config and address, but note that this doesn't send the
+		// updates to non-started child balancers (the child balancer might not
+		// be built, if it's a low priority).
+		currentChild.updateConfig(newSubConfig, resolver.State{
+			Endpoints:     endpointsSplit[name],
+			ServiceConfig: s.ResolverState.ServiceConfig,
+			Attributes:    s.ResolverState.Attributes,
+		})
+	}
+	// Cleanup resources used by children removed from the config.
+	for name, oldChild := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			oldChild.stop(!envconfig.EnablePriorityLBChildPolicyCache)
+			delete(b.children, name)
+		}
+	}
+
+	// Update priorities and handle priority changes.
+	b.priorities = newConfig.Priorities
+
+	// Everything was removed by the update.
+	if len(b.priorities) == 0 {
+		b.childInUse = ""
+		b.cc.UpdateState(balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(ErrAllPrioritiesRemoved),
+		})
+		b.mu.Unlock()
+		return nil
+	}
+
+	// This will sync the states of all children to the new updated
+	// priorities. Includes starting/stopping child balancers when necessary.
+	// Block picker updates until all children have had a chance to call
+	// UpdateState to prevent races where, e.g., the active priority reports
+	// transient failure but a higher priority may have reported something that
+	// made it active, and if the transient failure update is handled first,
+	// RPCs could fail.
+	b.inhibitPickerUpdates = true
+	// Add an item to queue to notify us when the current items in the queue
+	// are done and syncPriority has been called.
+	done := make(chan struct{})
+	b.childBalancerStateUpdate.Put(resumePickerUpdates{done: done})
+	b.mu.Unlock()
+	<-done
+
+	return nil
+}
+
+func (b *priorityBalancer) ResolverError(err error) {
+	if b.logger.V(2) {
+		b.logger.Infof("Received error from the resolver: %v", err)
+	}
+	b.bg.ResolverError(err)
+}
+
+func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *priorityBalancer) Close() {
+	b.bg.Close()
+	b.childBalancerStateUpdate.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.done.Fire()
+	// Clear states of the current child in use, so if there's a race in picker
+	// update, it will be dropped.
+	b.childInUse = ""
+	// Stop the child policies, this is necessary to stop the init timers in the
+	// children.
+	for _, child := range b.children {
+		child.stop(true)
+	}
+}
+
+func (b *priorityBalancer) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+// UpdateState implements balancergroup.BalancerStateAggregator interface. The
+// balancer group sends new connectivity state and picker here.
+func (b *priorityBalancer) UpdateState(childName string, state balancer.State) {
+	b.childBalancerStateUpdate.Put(childBalancerState{
+		name: childName,
+		s:    state,
+	})
+}
+
+type childBalancerState struct {
+	name string
+	s    balancer.State
+}
+
+type resumePickerUpdates struct {
+	done chan struct{}
+}
+
+// run handles child update in a separate goroutine, so if the child sends
+// updates inline (when called by parent), it won't cause deadlocks (by trying
+// to hold the same mutex).
+func (b *priorityBalancer) run() {
+	for {
+		select {
+		case u, ok := <-b.childBalancerStateUpdate.Get():
+			if !ok {
+				return
+			}
+			b.childBalancerStateUpdate.Load()
+			// Needs to handle state update in a goroutine, because each state
+			// update needs to start/close child policy, could result in
+			// deadlock.
+			b.mu.Lock()
+			if b.done.HasFired() {
+				b.mu.Unlock()
+				return
+			}
+			switch s := u.(type) {
+			case childBalancerState:
+				b.handleChildStateUpdate(s.name, s.s)
+			case resumePickerUpdates:
+				b.inhibitPickerUpdates = false
+				b.syncPriority(b.childInUse)
+				close(s.done)
+			}
+			b.mu.Unlock()
+		case <-b.done.Done():
+			return
+		}
+	}
+}

internal/xds/balancer/priority/balancer_child.go

@@ -0,0 +1,186 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+var timeAfterFunc = time.AfterFunc
+
+type childBalancer struct {
+	name         string
+	parent       *priorityBalancer
+	parentCC     balancer.ClientConn
+	balancerName string
+	cc           *ignoreResolveNowClientConn
+
+	ignoreReresolutionRequests bool
+	config                     serviceconfig.LoadBalancingConfig
+	rState                     resolver.State
+
+	started bool
+	// This is set when the child reports TransientFailure, and unset when it
+	// reports Ready or Idle. It is used to decide whether the failover timer
+	// should start when the child is transitioning into Connecting. The timer
+	// will be restarted if the child has not reported TF more recently than it
+	// reported Ready or Idle.
+	reportedTF bool
+	// The latest state the child balancer provided.
+	state balancer.State
+	// The timer to give a priority some time to connect. And if the priority
+	// doesn't go into Ready/Failure, the next priority will be started.
+	initTimer *timerWrapper
+}
+
+// newChildBalancer creates a child balancer place holder, but doesn't
+// build/start the child balancer.
+func newChildBalancer(name string, parent *priorityBalancer, balancerName string, cc balancer.ClientConn) *childBalancer {
+	return &childBalancer{
+		name:         name,
+		parent:       parent,
+		parentCC:     cc,
+		balancerName: balancerName,
+		cc:           newIgnoreResolveNowClientConn(cc, false),
+		started:      false,
+		// Start with the connecting state and picker with re-pick error, so
+		// that when a priority switch causes this child picked before it's
+		// balancing policy is created, a re-pick will happen.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+	}
+}
+
+// updateBalancerName updates balancer name for the child, but doesn't build a
+// new one. The parent priority LB always closes the child policy before
+// updating the balancer name, and the new balancer is built when it gets added
+// to the balancergroup as part of start().
+func (cb *childBalancer) updateBalancerName(balancerName string) {
+	cb.balancerName = balancerName
+	cb.cc = newIgnoreResolveNowClientConn(cb.parentCC, cb.ignoreReresolutionRequests)
+}
+
+// updateConfig sets childBalancer's config and state, but doesn't send update to
+// the child balancer unless it is started.
+func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) {
+	cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests
+	cb.config = child.Config.Config
+	cb.rState = rState
+	if cb.started {
+		cb.sendUpdate()
+	}
+}
+
+// start builds the child balancer if it's not already started.
+//
+// It doesn't do it directly. It asks the balancer group to build it.
+func (cb *childBalancer) start() {
+	if cb.started {
+		return
+	}
+	cb.started = true
+	cb.parent.bg.AddWithClientConn(cb.name, cb.balancerName, cb.cc)
+	cb.startInitTimer()
+	cb.sendUpdate()
+}
+
+// sendUpdate sends the addresses and config to the child balancer.
+func (cb *childBalancer) sendUpdate() {
+	cb.cc.updateIgnoreResolveNow(cb.ignoreReresolutionRequests)
+	// TODO: return and aggregate the returned error in the parent.
+	err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{
+		ResolverState:  cb.rState,
+		BalancerConfig: cb.config,
+	})
+	// Report TF if update to the child fails.
+	if err != nil {
+		cb.parent.logger.Warningf("Failed to update state for child policy %q: %v", cb.name, err)
+		cb.reportedTF = true
+		cb.state = balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(err),
+		}
+		cb.parent.handleChildStateUpdate(cb.name, cb.state)
+	}
+}
+
+// stop stops the child balancer and resets the state.
+//
+// It doesn't do it directly. It asks the balancer group to remove it.
+//
+// Note that the underlying balancer group could keep the child in a cache.
+func (cb *childBalancer) stop(immediate bool) {
+	if !cb.started {
+		return
+	}
+	cb.stopInitTimer()
+	if immediate {
+		cb.parent.bg.RemoveImmediately(cb.name)
+	} else {
+		cb.parent.bg.Remove(cb.name)
+	}
+	cb.started = false
+	cb.state = balancer.State{
+		ConnectivityState: connectivity.Connecting,
+		Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+	}
+	// Clear child.reportedTF, so that if this child is started later, it will
+	// be given time to connect.
+	cb.reportedTF = false
+}
+
+func (cb *childBalancer) startInitTimer() {
+	if cb.initTimer != nil {
+		return
+	}
+	// Need this local variable to capture timerW in the AfterFunc closure
+	// to check the stopped boolean.
+	timerW := &timerWrapper{}
+	cb.initTimer = timerW
+	timerW.timer = timeAfterFunc(DefaultPriorityInitTimeout, func() {
+		cb.parent.mu.Lock()
+		defer cb.parent.mu.Unlock()
+		if timerW.stopped {
+			return
+		}
+		cb.initTimer = nil
+		// Re-sync the priority. This will switch to the next priority if
+		// there's any. Note that it's important sync() is called after setting
+		// initTimer to nil.
+		cb.parent.syncPriority("")
+	})
+}
+
+func (cb *childBalancer) stopInitTimer() {
+	timerW := cb.initTimer
+	if timerW == nil {
+		return
+	}
+	cb.initTimer = nil
+	timerW.stopped = true
+	timerW.timer.Stop()
+}

internal/xds/balancer/priority/balancer_priority.go

@@ -0,0 +1,223 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"errors"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+)
+
+var (
+	// ErrAllPrioritiesRemoved is returned by the picker when there's no priority available.
+	ErrAllPrioritiesRemoved = errors.New("no priority is provided, all priorities are removed")
+	// DefaultPriorityInitTimeout is the timeout after which if a priority is
+	// not READY, the next will be started. It's exported to be overridden by
+	// tests.
+	DefaultPriorityInitTimeout = 10 * time.Second
+)
+
+// syncPriority handles priority after a config update or a child balancer
+// connectivity state update. It makes sure the balancer state (started or not)
+// is in sync with the priorities (even in tricky cases where a child is moved
+// from a priority to another).
+//
+// It's guaranteed that after this function returns:
+//
+//	If some child is READY, it is childInUse, and all lower priorities are
+//	closed.
+//
+//	If some child is newly started(in Connecting for the first time), it is
+//	childInUse, and all lower priorities are closed.
+//
+//	Otherwise, the lowest priority is childInUse (none of the children is
+//	ready, and the overall state is not ready).
+//
+// Steps:
+//
+//	If all priorities were deleted, unset childInUse (to an empty string), and
+//	set parent ClientConn to TransientFailure
+//
+//	Otherwise, Scan all children from p0, and check balancer stats:
+//
+//	  For any of the following cases:
+//
+//	    If balancer is not started (not built), this is either a new child with
+//	    high priority, or a new builder for an existing child.
+//
+//	    If balancer is Connecting and has non-nil initTimer (meaning it
+//	    transitioned from Ready or Idle to connecting, not from TF, so we
+//	    should give it init-time to connect).
+//
+//	    If balancer is READY or IDLE
+//
+//	    If this is the lowest priority
+//
+//	 do the following:
+//
+//	    if this is not the old childInUse, override picker so old picker is no
+//	    longer used.
+//
+//	    switch to it (because all higher priorities are neither new or Ready)
+//
+//	    forward the new addresses and config
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) syncPriority(childUpdating string) {
+	if b.inhibitPickerUpdates {
+		if b.logger.V(2) {
+			b.logger.Infof("Skipping update from child policy %q", childUpdating)
+		}
+		return
+	}
+	for p, name := range b.priorities {
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Warningf("Priority name %q is not found in list of child policies", name)
+			continue
+		}
+
+		if !child.started ||
+			child.state.ConnectivityState == connectivity.Ready ||
+			child.state.ConnectivityState == connectivity.Idle ||
+			(child.state.ConnectivityState == connectivity.Connecting && child.initTimer != nil) ||
+			p == len(b.priorities)-1 {
+			if b.childInUse != child.name || child.name == childUpdating {
+				if b.logger.V(2) {
+					b.logger.Infof("childInUse, childUpdating: %q, %q", b.childInUse, child.name)
+				}
+				// If we switch children or the child in use just updated its
+				// picker, push the child's picker to the parent.
+				b.cc.UpdateState(child.state)
+			}
+			if b.logger.V(2) {
+				b.logger.Infof("Switching to (%q, %v) in syncPriority", child.name, p)
+			}
+			b.switchToChild(child, p)
+			break
+		}
+	}
+}
+
+// Stop priorities [p+1, lowest].
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) {
+	for i := p + 1; i < len(b.priorities); i++ {
+		name := b.priorities[i]
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Warningf("Priority name %q is not found in list of child policies", name)
+			continue
+		}
+		child.stop(false)
+	}
+}
+
+// switchToChild does the following:
+// - stop all child with lower priorities
+// - if childInUse is not this child
+//   - set childInUse to this child
+//   - if this child is not started, start it
+//
+// Note that it does NOT send the current child state (picker) to the parent
+// ClientConn. The caller needs to send it if necessary.
+//
+// this can be called when
+// 1. first update, start p0
+// 2. an update moves a READY child from a lower priority to higher
+// 2. a different builder is updated for this child
+// 3. a high priority goes Failure, start next
+// 4. a high priority init timeout, start next
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) {
+	// Stop lower priorities even if childInUse is same as this child. It's
+	// possible this child was moved from a priority to another.
+	b.stopSubBalancersLowerThanPriority(priority)
+
+	// If this child is already in use, do nothing.
+	//
+	// This can happen:
+	// - all priorities are not READY, an config update always triggers switch
+	// to the lowest. In this case, the lowest child could still be connecting,
+	// so we don't stop the init timer.
+	// - a high priority is READY, an config update always triggers switch to
+	// it.
+	if b.childInUse == child.name && child.started {
+		return
+	}
+	b.childInUse = child.name
+
+	if !child.started {
+		child.start()
+	}
+}
+
+// handleChildStateUpdate start/close priorities based on the connectivity
+// state.
+func (b *priorityBalancer) handleChildStateUpdate(childName string, newState balancer.State) {
+	// Update state in child. The updated picker will be sent to parent later if
+	// necessary.
+	child, ok := b.children[childName]
+	if !ok {
+		b.logger.Warningf("Child policy not found for %q", childName)
+		return
+	}
+	if !child.started {
+		b.logger.Warningf("Ignoring update from child policy %q which is not in started state: %+v", childName, newState)
+		return
+	}
+	oldState := child.state
+	child.state = newState
+
+	// We start/stop the init timer of this child based on the new connectivity
+	// state. syncPriority() later will need the init timer (to check if it's
+	// nil or not) to decide which child to switch to.
+	switch newState.ConnectivityState {
+	case connectivity.Ready, connectivity.Idle:
+		child.reportedTF = false
+		child.stopInitTimer()
+	case connectivity.TransientFailure:
+		child.reportedTF = true
+		child.stopInitTimer()
+	case connectivity.Connecting:
+		// The init timer is created when the child is created and is reset when
+		// it reports Ready or Idle. Most child policies start off in
+		// Connecting, but ring_hash starts off in Idle and moves to Connecting
+		// when a request comes in. To support such cases, we restart the init
+		// timer when we see Connecting, but only if the child has not reported
+		// TransientFailure more recently than it reported Ready or Idle. See
+		// gRFC A42 for details on why ring_hash is special and what provisions
+		// are required to make it work as a child of the priority LB policy.
+		//
+		// We don't want to restart the timer if the child was already in
+		// Connecting, because we want failover to happen once the timer elapses
+		// even when the child is still in Connecting.
+		if !child.reportedTF && oldState.ConnectivityState != connectivity.Connecting {
+			child.startInitTimer()
+		}
+	default:
+		// New state is Shutdown, should never happen. Don't forward.
+	}
+
+	child.parent.syncPriority(childName)
+}

internal/xds/balancer/priority/config.go

@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"encoding/json"
+	"fmt"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Child is a child of priority balancer.
+type Child struct {
+	Config                     *internalserviceconfig.BalancerConfig `json:"config,omitempty"`
+	IgnoreReresolutionRequests bool                                  `json:"ignoreReresolutionRequests,omitempty"`
+}
+
+// LBConfig represents priority balancer's config.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	// Children is a map from the child balancer names to their configs. Child
+	// names can be found in field Priorities.
+	Children map[string]*Child `json:"children,omitempty"`
+	// Priorities is a list of child balancer names. They are sorted from
+	// highest priority to low. The type/config for each child can be found in
+	// field Children, with the balancer name as the key.
+	Priorities []string `json:"priorities,omitempty"`
+}
+
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+
+	prioritiesSet := make(map[string]bool)
+	for _, name := range cfg.Priorities {
+		if _, ok := cfg.Children[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children)
+		}
+		prioritiesSet[name] = true
+	}
+	for name := range cfg.Children {
+		if _, ok := prioritiesSet[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities)
+		}
+	}
+	return &cfg, nil
+}

internal/xds/balancer/priority/config_test.go

@@ -0,0 +1,108 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer/roundrobin"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *LBConfig
+		wantErr bool
+	}{
+		{
+			name: "child not found",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "child not used",
+			js: `{
+  "priorities": ["child-1", "child-2"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "good",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			want: &LBConfig{
+				Children: map[string]*Child{
+					"child-1": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+						IgnoreReresolutionRequests: true,
+					},
+					"child-2": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+					"child-3": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+				},
+				Priorities: []string{"child-1", "child-2", "child-3"},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff)
+			}
+		})
+	}
+}

internal/xds/balancer/priority/ignore_resolve_now.go

@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"sync/atomic"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/resolver"
+)
+
+// ignoreResolveNowClientConn wraps a balancer.ClientConn and overrides the
+// ResolveNow() method to ignore those calls if the ignoreResolveNow bit is set.
+type ignoreResolveNowClientConn struct {
+	balancer.ClientConn
+	ignoreResolveNow atomic.Bool
+}
+
+func newIgnoreResolveNowClientConn(cc balancer.ClientConn, ignore bool) *ignoreResolveNowClientConn {
+	ret := &ignoreResolveNowClientConn{ClientConn: cc}
+	ret.updateIgnoreResolveNow(ignore)
+	return ret
+}
+
+func (i *ignoreResolveNowClientConn) updateIgnoreResolveNow(b bool) {
+	i.ignoreResolveNow.Store(b)
+}
+
+func (i *ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) {
+	if i.ignoreResolveNow.Load() {
+		return
+	}
+	i.ClientConn.ResolveNow(o)
+}

internal/xds/balancer/priority/ignore_resolve_now_test.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+)
+
+func (s) TestIgnoreResolveNowClientConn(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	ignoreCC := newIgnoreResolveNowClientConn(cc, false)
+
+	// Call ResolveNow() on the CC, it should be forwarded.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for ResolveNow()")
+	}
+
+	// Update ignoreResolveNow to true, call ResolveNow() on the CC, they should
+	// all be ignored.
+	ignoreCC.updateIgnoreResolveNow(true)
+	for i := 0; i < 5; i++ {
+		ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	}
+	select {
+	case <-cc.ResolveNowCh:
+		t.Fatalf("got unexpected ResolveNow() call")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Update ignoreResolveNow to false, new ResolveNow() calls should be
+	// forwarded.
+	ignoreCC.updateIgnoreResolveNow(false)
+	ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+}

internal/xds/balancer/priority/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[priority-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/wrrlocality/balancer.go

@@ -0,0 +1,201 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package wrrlocality provides an implementation of the wrr locality LB policy,
+// as defined in [A52 - xDS Custom LB Policies].
+//
+// [A52 - xDS Custom LB Policies]: https://github.com/grpc/proposal/blob/master/A52-xds-custom-lb-policies.md
+package wrrlocality
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/weightedtarget"
+	"google.golang.org/grpc/internal/grpclog"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Name is the name of wrr_locality balancer.
+const Name = "xds_wrr_locality_experimental"
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Name() string {
+	return Name
+}
+
+// LBConfig is the config for the wrr locality balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+	// ChildPolicy is the config for the child policy.
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+// To plumb in a different child in tests.
+var weightedTargetName = weightedtarget.Name
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	builder := balancer.Get(weightedTargetName)
+	if builder == nil {
+		// Shouldn't happen, registered through imported weighted target,
+		// defensive programming.
+		return nil
+	}
+
+	// Doesn't need to intercept any balancer.ClientConn operations; pass
+	// through by just giving cc to child balancer.
+	wtb := builder.Build(cc, bOpts)
+	if wtb == nil {
+		// shouldn't happen, defensive programming.
+		return nil
+	}
+	wtbCfgParser, ok := builder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported weighted target builder has this method.
+		return nil
+	}
+	wrrL := &wrrLocalityBalancer{
+		child:       wtb,
+		childParser: wtbCfgParser,
+	}
+
+	wrrL.logger = prefixLogger(wrrL)
+	wrrL.logger.Infof("Created")
+	return wrrL
+}
+
+func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	var lbCfg *LBConfig
+	if err := json.Unmarshal(s, &lbCfg); err != nil {
+		return nil, fmt.Errorf("xds_wrr_locality: invalid LBConfig: %s, error: %v", string(s), err)
+	}
+	if lbCfg == nil || lbCfg.ChildPolicy == nil {
+		return nil, errors.New("xds_wrr_locality: invalid LBConfig: child policy field must be set")
+	}
+	return lbCfg, nil
+}
+
+type attributeKey struct{}
+
+// Equal allows the values to be compared by Attributes.Equal.
+func (a AddrInfo) Equal(o any) bool {
+	oa, ok := o.(AddrInfo)
+	return ok && oa.LocalityWeight == a.LocalityWeight
+}
+
+// AddrInfo is the locality weight of the locality an address is a part of.
+type AddrInfo struct {
+	LocalityWeight uint32
+}
+
+// SetAddrInfo returns a copy of endpoint in which the Attributes field is
+// updated with AddrInfo.
+func SetAddrInfo(endpoint resolver.Endpoint, addrInfo AddrInfo) resolver.Endpoint {
+	endpoint.Attributes = endpoint.Attributes.WithValue(attributeKey{}, addrInfo)
+	return endpoint
+}
+
+func (a AddrInfo) String() string {
+	return fmt.Sprintf("Locality Weight: %d", a.LocalityWeight)
+}
+
+// getAddrInfo returns the AddrInfo stored in the Attributes field of
+// ep. Returns false if no AddrInfo found.
+func getAddrInfo(ep resolver.Endpoint) (AddrInfo, bool) {
+	v := ep.Attributes.Value(attributeKey{})
+	ai, ok := v.(AddrInfo)
+	return ai, ok
+}
+
+// wrrLocalityBalancer wraps a weighted target balancer, and builds
+// configuration for the weighted target once it receives configuration
+// specifying the weighted target child balancer and locality weight
+// information.
+type wrrLocalityBalancer struct {
+	// child will be a weighted target balancer, and will be built it at
+	// wrrLocalityBalancer build time. Other than preparing configuration, other
+	// balancer operations are simply pass through.
+	child balancer.Balancer
+
+	childParser balancer.ConfigParser
+
+	logger *grpclog.PrefixLogger
+}
+
+func (b *wrrLocalityBalancer) ExitIdle() {
+	b.child.ExitIdle()
+}
+
+func (b *wrrLocalityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	lbCfg, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		b.logger.Errorf("Received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+
+	weightedTargets := make(map[string]weightedtarget.Target)
+	for _, ep := range s.ResolverState.Endpoints {
+		// This get of LocalityID could potentially return a zero value. This
+		// shouldn't happen though (this attribute that is set actually gets
+		// used to build localities in the first place), and thus don't error
+		// out, and just build a weighted target with undefined behavior.
+		locality := xdsinternal.LocalityString(xdsinternal.LocalityIDFromEndpoint(ep))
+		ai, ok := getAddrInfo(ep)
+		if !ok {
+			return fmt.Errorf("xds_wrr_locality: missing locality weight information in endpoint %q", ep)
+		}
+		weightedTargets[locality] = weightedtarget.Target{Weight: ai.LocalityWeight, ChildPolicy: lbCfg.ChildPolicy}
+	}
+	wtCfg := &weightedtarget.LBConfig{Targets: weightedTargets}
+	wtCfgJSON, err := json.Marshal(wtCfg)
+	if err != nil {
+		// Shouldn't happen.
+		return fmt.Errorf("xds_wrr_locality: error marshalling prepared config: %v", wtCfg)
+	}
+	var sc serviceconfig.LoadBalancingConfig
+	if sc, err = b.childParser.ParseConfig(wtCfgJSON); err != nil {
+		return fmt.Errorf("xds_wrr_locality: config generated %v is invalid: %v", wtCfgJSON, err)
+	}
+
+	return b.child.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: sc,
+	})
+}
+
+func (b *wrrLocalityBalancer) ResolverError(err error) {
+	b.child.ResolverError(err)
+}
+
+func (b *wrrLocalityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *wrrLocalityBalancer) Close() {
+	b.child.Close()
+}

internal/xds/balancer/wrrlocality/balancer_test.go

@@ -0,0 +1,250 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package wrrlocality
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/balancer/weightedtarget"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	defaultTestTimeout = 5 * time.Second
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestParseConfig(t *testing.T) {
+	const errParseConfigName = "errParseConfigBalancer"
+	stub.Register(errParseConfigName, stub.BalancerFuncs{
+		ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return nil, errors.New("some error")
+		},
+	})
+
+	parser := bb{}
+	tests := []struct {
+		name    string
+		input   string
+		wantCfg serviceconfig.LoadBalancingConfig
+		wantErr string
+	}{
+		{
+			name:  "happy-case-round robin-child",
+			input: `{"childPolicy": [{"round_robin": {}}]}`,
+			wantCfg: &LBConfig{
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: roundrobin.Name,
+				},
+			},
+		},
+		{
+			name:    "invalid-json",
+			input:   "{{invalidjson{{",
+			wantErr: "invalid character",
+		},
+
+		{
+			name:    "child-policy-field-isn't-set",
+			input:   `{}`,
+			wantErr: "child policy field must be set",
+		},
+		{
+			name:    "child-policy-type-is-empty",
+			input:   `{"childPolicy": []}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in []",
+		},
+		{
+			name:    "child-policy-empty-config",
+			input:   `{"childPolicy": [{"": {}}]}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in []",
+		},
+		{
+			name:    "child-policy-type-isn't-registered",
+			input:   `{"childPolicy": [{"doesNotExistBalancer": {"cluster": "test_cluster"}}]}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in [doesNotExistBalancer]",
+		},
+		{
+			name:    "child-policy-config-is-invalid",
+			input:   `{"childPolicy": [{"errParseConfigBalancer": {"cluster": "test_cluster"}}]}`,
+			wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input))
+			// Substring match makes this very tightly coupled to the
+			// internalserviceconfig.BalancerConfig error strings. However, it
+			// is important to distinguish the different types of error messages
+			// possible as the parser has a few defined buckets of ways it can
+			// error out.
+			if (gotErr != nil) != (test.wantErr != "") {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" {
+				t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff)
+			}
+		})
+	}
+}
+
+// TestUpdateClientConnState tests the UpdateClientConnState method of the
+// wrr_locality_experimental balancer. This UpdateClientConn operation should
+// take the localities and their weights in the addresses passed in, alongside
+// the endpoint picking policy defined in the Balancer Config and construct a
+// weighted target configuration corresponding to these inputs.
+func (s) TestUpdateClientConnState(t *testing.T) {
+	// Configure the stub balancer defined below as the child policy of
+	// wrrLocalityBalancer.
+	cfgCh := testutils.NewChannel()
+	oldWeightedTargetName := weightedTargetName
+	defer func() {
+		weightedTargetName = oldWeightedTargetName
+	}()
+	weightedTargetName = "fake_weighted_target"
+	stub.Register("fake_weighted_target", stub.BalancerFuncs{
+		ParseConfig: func(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			var cfg weightedtarget.LBConfig
+			if err := json.Unmarshal(c, &cfg); err != nil {
+				return nil, err
+			}
+			return &cfg, nil
+		},
+		UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error {
+			wtCfg, ok := ccs.BalancerConfig.(*weightedtarget.LBConfig)
+			if !ok {
+				return errors.New("child received config that was not a weighted target config")
+			}
+			defer cfgCh.Send(wtCfg)
+			return nil
+		},
+	})
+
+	builder := balancer.Get(Name)
+	if builder == nil {
+		t.Fatalf("balancer.Get(%q) returned nil", Name)
+	}
+	tcc := testutils.NewBalancerClientConn(t)
+	bal := builder.Build(tcc, balancer.BuildOptions{})
+	defer bal.Close()
+	wrrL := bal.(*wrrLocalityBalancer)
+
+	// Create the addresses with two localities with certain locality weights.
+	// This represents what addresses the wrr_locality balancer will receive in
+	// UpdateClientConnState.
+	ep1 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-1"}}}
+	ep1 = xdsinternal.SetLocalityIDInEndpoint(ep1, clients.Locality{
+		Region:  "region-1",
+		Zone:    "zone-1",
+		SubZone: "subzone-1",
+	})
+	ep1 = SetAddrInfo(ep1, AddrInfo{LocalityWeight: 2})
+
+	ep2 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-2"}}}
+	ep2 = xdsinternal.SetLocalityIDInEndpoint(ep2, clients.Locality{
+		Region:  "region-2",
+		Zone:    "zone-2",
+		SubZone: "subzone-2",
+	})
+	ep2 = SetAddrInfo(ep2, AddrInfo{LocalityWeight: 1})
+
+	eps := []resolver.Endpoint{ep1, ep2}
+
+	err := wrrL.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &LBConfig{
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: "round_robin",
+			},
+		},
+		ResolverState: resolver.State{
+			Endpoints: eps,
+		},
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Note that these inline strings declared as the key in Targets built from
+	// Locality ID are not exactly what is shown in the example in the gRFC.
+	// However, this is an implementation detail that does not affect
+	// correctness (confirmed with Java team). The important thing is to get
+	// those three pieces of information region, zone, and subzone down to the
+	// child layer.
+	wantWtCfg := &weightedtarget.LBConfig{
+		Targets: map[string]weightedtarget.Target{
+			"{region=\"region-1\", zone=\"zone-1\", sub_zone=\"subzone-1\"}": {
+				Weight: 2,
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: "round_robin",
+				},
+			},
+			"{region=\"region-2\", zone=\"zone-2\", sub_zone=\"subzone-2\"}": {
+				Weight: 1,
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: "round_robin",
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cfg, err := cfgCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("No signal received from UpdateClientConnState() on the child: %v", err)
+	}
+
+	gotWtCfg, ok := cfg.(*weightedtarget.LBConfig)
+	if !ok {
+		// Shouldn't happen - only sends a config on this channel.
+		t.Fatalf("Unexpected config type: %T", gotWtCfg)
+	}
+
+	if diff := cmp.Diff(gotWtCfg, wantWtCfg); diff != "" {
+		t.Fatalf("Child received unexpected config, diff (-got, +want): %v", diff)
+	}
+}

internal/xds/balancer/wrrlocality/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package wrrlocality
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[wrrlocality-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *wrrLocalityBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}