snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/balancer/balancer.go

@@ -0,0 +1,30 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package balancer installs all the xds balancers.
+package balancer
+
+import (
+	_ "google.golang.org/grpc/balancer/leastrequest"                  // Register the least_request_experimental balancer
+	_ "google.golang.org/grpc/balancer/weightedtarget"                // Register the weighted_target balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer"      // Register the CDS balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/clusterimpl"      // Register the xds_cluster_impl balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/clustermanager"   // Register the xds_cluster_manager balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // Register the outlier_detection balancer
+	_ "google.golang.org/grpc/internal/xds/balancer/priority"         // Register the priority balancer
+)

internal/xds/balancer/cdsbalancer/aggregate_cluster_test.go

@@ -0,0 +1,932 @@
+/*
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/balancer/pickfirst"
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter.
+	_ "google.golang.org/grpc/internal/xds/resolver"          // Register the xds resolver
+)
+
+// makeAggregateClusterResource returns an aggregate cluster resource with the
+// given name and list of child names.
+func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeAggregate,
+		ChildNames:  childNames,
+	})
+}
+
+// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the
+// given name and given DNS host and port.
+func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeLogicalDNS,
+		DNSHostName: dnsHost,
+		DNSPort:     dnsPort,
+	})
+}
+
+// verifyDNSResolution verifies that the DNS resolver is started for the expected target.
+func verifyDNSResolution(ctx context.Context, t *testing.T, dnsTargetCh chan resolver.Target, dnsR *manual.Resolver, host string, port uint32) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for DNS watch")
+	case target := <-dnsTargetCh:
+		addr := fmt.Sprintf("%s:%d", host, port)
+		if target.Endpoint() != addr {
+			t.Fatalf("DNS resolution started for target %q, want %q", target.Endpoint(), addr)
+		}
+		dnsR.UpdateState(resolver.State{
+			Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: addr}}}},
+		})
+	}
+}
+
+// Tests the case where the cluster resource requested is a leaf cluster. The
+// management server sends two updates for the same leaf cluster resource. The
+// test verifies that the load balancing configuration pushed to the priority LB
+// policy contains the expected discovery mechanism corresponding to the leaf
+// cluster, on both occasions.
+func (s) TestAggregateClusterSuccess_LeafNode(t *testing.T) {
+	tests := []struct {
+		name                  string
+		firstClusterResource  *v3clusterpb.Cluster
+		secondClusterResource *v3clusterpb.Cluster
+		wantFirstChildCfg     serviceconfig.LoadBalancingConfig
+		wantSecondChildCfg    serviceconfig.LoadBalancingConfig
+	}{
+		{
+			name:                  "eds",
+			firstClusterResource:  e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone),
+			secondClusterResource: e2e.DefaultCluster(clusterName, serviceName+"-new", e2e.SecurityLevelNone),
+			wantFirstChildCfg: &priority.LBConfig{
+				Children: map[string]*priority.Child{
+					"priority-0-0": {
+						Config:                     createPriorityConfig(clusterName),
+						IgnoreReresolutionRequests: true,
+					},
+				},
+				Priorities: []string{"priority-0-0"},
+			},
+			wantSecondChildCfg: &priority.LBConfig{
+				Children: map[string]*priority.Child{
+					"priority-1-0": {
+						Config:                     createPriorityConfig(clusterName),
+						IgnoreReresolutionRequests: true,
+					},
+				},
+				Priorities: []string{"priority-1-0"},
+			},
+		},
+		{
+			name:                  "dns",
+			firstClusterResource:  makeLogicalDNSClusterResource(clusterName, "dns_host", uint32(port)),
+			secondClusterResource: makeLogicalDNSClusterResource(clusterName, "dns_host_new", uint32(port)),
+			wantFirstChildCfg: &priority.LBConfig{
+				Children:   map[string]*priority.Child{"priority-0": {Config: createPriorityConfig(clusterName)}},
+				Priorities: []string{"priority-0"},
+			},
+			wantSecondChildCfg: &priority.LBConfig{
+				Children:   map[string]*priority.Child{"priority-1": {Config: createPriorityConfig(clusterName)}},
+				Priorities: []string{"priority-1"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+			mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+			// Push the first cluster resource through the management server and
+			// verify the configuration pushed to the child policy.
+			resources := e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+				Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+				Clusters:       []*v3clusterpb.Cluster{test.firstClusterResource},
+				Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+				SkipValidation: true,
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+			if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantFirstChildCfg); err != nil {
+				t.Fatal(err)
+			}
+
+			// Push the second cluster resource through the management server and
+			// verify the configuration pushed to the child policy.
+			resources.Clusters[0] = test.secondClusterResource
+			resources.Endpoints[0] = e2e.DefaultEndpoint(serviceName+"-new", host, []uint32{port})
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+			if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantSecondChildCfg); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// Tests the case where the cluster resource requested is an aggregate cluster
+// root pointing to two child clusters, one of type EDS and the other of type
+// LogicalDNS. The test verifies that load balancing configuration is pushed to
+// the priority LB policy only when all child clusters are resolved and it also
+// verifies that the pushed configuration contains the expected priority config.
+// The test then updates the aggregate cluster to point to two child clusters,
+// the same leaf cluster of type EDS and a different leaf cluster of type
+// LogicalDNS and verifies that the load balancing configuration pushed to the
+// priority LB policy contains the expected config.
+func (s) TestAggregateClusterSuccess_ThenUpdateChildClusters(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with the aggregate cluster resource
+	// pointing to two child clusters, one EDS and one LogicalDNS. Include the
+	// resource corresponding to the EDS cluster here, but don't include
+	// resource corresponding to the LogicalDNS cluster yet.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the LogicalDNS cluster in the management server. This
+	// should result in configuration being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	const dnsClusterNameNew = dnsClusterName + "-new"
+	const dnsHostNameNew = dnsHostName + "-new"
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterNameNew}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+			makeLogicalDNSClusterResource(dnsClusterNameNew, dnsHostNameNew, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostNameNew, dnsPort)
+
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-2": {Config: createPriorityConfig(dnsClusterNameNew)},
+		},
+		Priorities: []string{"priority-0-0", "priority-2"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where the cluster resource requested is an aggregate cluster
+// root pointing to two child clusters, one of type EDS and the other of type
+// LogicalDNS. The test verifies that the load balancing configuration pushed to
+// the priority LB policy contains the discovery mechanisms for both child
+// clusters. The test then updates the root cluster resource requested by the
+// cds LB policy to a leaf cluster of type EDS and verifies the load balancing
+// configuration pushed to the priority LB policy contains a single discovery
+// mechanism.
+func (s) TestAggregateClusterSuccess_ThenChangeRootToEDS(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with the aggregate cluster resource
+	// pointing to two child clusters.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	// Since the service name of the EDS cluster remains same, same priority name
+	// is used.
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where a requested cluster resource switches between being a
+// leaf and an aggregate cluster pointing to an EDS and LogicalDNS child
+// cluster. In each of these cases, the test verifies that the load balancing
+// configuration pushed to the priority LB policy contains the expected config.
+func (s) TestAggregatedClusterSuccess_SwitchBetweenLeafAndAggregate(t *testing.T) {
+	dnsTargetCh, dnsR := setupDNS(t)
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Start off with the requested cluster being a leaf EDS cluster.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch the requested cluster to be an aggregate cluster pointing to two
+	// child clusters.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}),
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelNone),
+			makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	verifyDNSResolution(ctx, t, dnsTargetCh, dnsR, dnsHostName, dnsPort)
+
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(edsClusterName),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1": {Config: createPriorityConfig(dnsClusterName)},
+		},
+		Priorities: []string{"priority-0-0", "priority-1"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch the cluster back to a leaf EDS cluster.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	wantChildCfg = &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterName),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where an aggregate cluster exceeds the maximum depth,
+// which is 16. Verifies that the channel moves to TRANSIENT_FAILURE, and the
+// error is propagated to RPC callers. The test then modifies the graph to no
+// longer exceed maximum depth, but be at the maximum allowed depth, and
+// verifies that an RPC can be made successfully.
+func (s) TestAggregatedClusterFailure_ExceedsMaxStackDepth(t *testing.T) {
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}),
+			makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}),
+			makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}),
+			makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}),
+			makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}),
+			makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}),
+			makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}),
+			makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}),
+			makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}),
+			makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}),
+			makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}),
+			makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}),
+			makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}),
+			makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}),
+			makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}),
+			makeAggregateClusterResource(clusterName+"-15", []string{clusterName + "-16"}),
+			e2e.DefaultCluster(clusterName+"-16", serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	const wantErr = "aggregate cluster graph exceeds max depth"
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if code := status.Code(err); code != codes.Unavailable {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", code, codes.Unavailable)
+	}
+	if err != nil && !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want err containing: %v", err, wantErr)
+	}
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	// Update the aggregate cluster resource to no longer exceed max depth, and
+	// be at the maximum depth allowed.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterName, []string{clusterName + "-1"}),
+			makeAggregateClusterResource(clusterName+"-1", []string{clusterName + "-2"}),
+			makeAggregateClusterResource(clusterName+"-2", []string{clusterName + "-3"}),
+			makeAggregateClusterResource(clusterName+"-3", []string{clusterName + "-4"}),
+			makeAggregateClusterResource(clusterName+"-4", []string{clusterName + "-5"}),
+			makeAggregateClusterResource(clusterName+"-5", []string{clusterName + "-6"}),
+			makeAggregateClusterResource(clusterName+"-6", []string{clusterName + "-7"}),
+			makeAggregateClusterResource(clusterName+"-7", []string{clusterName + "-8"}),
+			makeAggregateClusterResource(clusterName+"-8", []string{clusterName + "-9"}),
+			makeAggregateClusterResource(clusterName+"-9", []string{clusterName + "-10"}),
+			makeAggregateClusterResource(clusterName+"-10", []string{clusterName + "-11"}),
+			makeAggregateClusterResource(clusterName+"-11", []string{clusterName + "-12"}),
+			makeAggregateClusterResource(clusterName+"-12", []string{clusterName + "-13"}),
+			makeAggregateClusterResource(clusterName+"-13", []string{clusterName + "-14"}),
+			makeAggregateClusterResource(clusterName+"-14", []string{clusterName + "-15"}),
+			e2e.DefaultCluster(clusterName+"-15", serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests a diamond shaped aggregate cluster (A->[B,C]; B->D; C->D). Verifies
+// that the load balancing configuration pushed to the priority LB
+// policy specifies cluster D only once. Also verifies that configuration is
+// pushed only after all child clusters are resolved.
+func (s) TestAggregatedClusterSuccess_DiamondDependency(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with an aggregate cluster resource having
+	// a diamond dependency pattern, (A->[B,C]; B->D; C->D). Includes resources
+	// for cluster A, B and D, but don't include the resource for cluster C yet.
+	// This will help us verify that no configuration is pushed to the child
+	// policy until the whole cluster graph is resolved.
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+		clusterNameD = clusterName + "-D"
+	)
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameD}),
+			e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the resource for cluster C in the management server,
+	// thereby completing the cluster graph. This should result in configuration
+	// being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, makeAggregateClusterResource(clusterNameC, []string{clusterNameD}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameD),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the case where the aggregate cluster graph contains duplicates (A->[B,
+// C]; B->[C, D]). Verifies that the load balancing configuration pushed to the
+// priority LB policy does not contain duplicates, and that the priority
+// corresponding to cluster C is higher than that for cluster D. Also verifies
+// that the configuration is pushed only after all child clusters are resolved.
+func (s) TestAggregatedClusterSuccess_IgnoreDups(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, nil)
+
+	// Configure the management server with an aggregate cluster resource that
+	// has duplicates in the graph, (A->[B, C]; B->[C, D]). Include resources
+	// for clusters A, B and D, but don't configure the resource for cluster C
+	// yet. This will help us verify that no configuration is pushed to the
+	// child policy until the whole cluster graph is resolved.
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+		clusterNameD = clusterName + "-D"
+	)
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB, clusterNameC}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameC, clusterNameD}),
+			e2e.DefaultCluster(clusterNameD, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{port})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that no configuration is pushed to the child policy yet, because
+	// not all clusters making up the aggregate cluster have been resolved yet.
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Now configure the resource for cluster C in the management server,
+	// thereby completing the cluster graph. This should result in configuration
+	// being pushed down to the child policy.
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterNameC, edsClusterName, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{1234}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameC),
+				IgnoreReresolutionRequests: true,
+			},
+			"priority-1-0": {
+				Config:                     createPriorityConfig(clusterNameD),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0", "priority-1-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph has a node that has
+// child node of itself. The case for this is A -> A, and since there is no base
+// cluster (EDS or Logical DNS), no configuration should be pushed to the child
+// policy.  The channel is expected to move to TRANSIENT_FAILURE and RPCs are
+// expected to fail with code UNAVAILABLE and an error message specifying that
+// the aggregate cluster graph has no leaf clusters.  Then the test updates A -> B,
+// where B is a leaf EDS cluster. Verifies that configuration is pushed to the
+// child policy and that an RPC can be successfully made.
+func (s) TestAggregatedCluster_NodeChildOfItself(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+	)
+	// Configure the management server with an aggregate cluster resource whose
+	// child is itself.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{makeAggregateClusterResource(clusterNameA, []string{clusterNameA})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Verify that the RPC fails with expected code.
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode)
+	}
+	const wantErr = "aggregate cluster graph has no leaf clusters"
+	if !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want error containing %s", err, wantErr)
+	}
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	// Update the aggregate cluster to point to a leaf EDS cluster.
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			e2e.DefaultCluster(clusterNameB, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the configuration pushed to the child policy.
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameB),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph contains a cycle and
+// contains no leaf clusters. The case used here is [A -> B, B -> A]. As there
+// are no leaf clusters in this graph, no configuration should be pushed to the
+// child policy. The channel is expected to move to TRANSIENT_FAILURE and RPCs
+// are expected to fail with code UNAVAILABLE and an error message specifying
+// that the aggregate cluster graph has no leaf clusters.
+func (s) TestAggregatedCluster_CycleWithNoLeafNode(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+	)
+	// Configure the management server with an aggregate cluster resource graph
+	// that contains a cycle and no leaf clusters.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameA}),
+		},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case cfg := <-lbCfgCh:
+		t.Fatalf("Child policy received configuration when not expected to: %s", pretty.ToJSON(cfg))
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Verify that the RPC fails with expected code.
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err := client.EmptyCall(ctx, &testpb.Empty{})
+	if gotCode, wantCode := status.Code(err), codes.Unavailable; gotCode != wantCode {
+		t.Fatalf("EmptyCall() failed with code: %v, want %v", gotCode, wantCode)
+	}
+	const wantErr = "aggregate cluster graph has no leaf clusters"
+	if !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed with err: %v, want %s", err, wantErr)
+	}
+}
+
+// Tests the scenario where the aggregate cluster graph contains a cycle and
+// also contains a leaf cluster. The case used here is [A -> B, B -> A, C]. As
+// there is a leaf cluster in this graph , configuration should be pushed to the
+// child policy and RPCs should get routed to that leaf cluster.
+func (s) TestAggregatedCluster_CycleWithLeafNode(t *testing.T) {
+	lbCfgCh, _, _, _ := registerWrappedPriorityPolicy(t)
+	mgmtServer, nodeID, cc := setupWithManagementServer(t, nil, nil)
+
+	// Start a test service backend.
+	server := stubserver.StartTestService(t, nil)
+	t.Cleanup(server.Stop)
+
+	const (
+		clusterNameA = clusterName // cluster name in cds LB policy config
+		clusterNameB = clusterName + "-B"
+		clusterNameC = clusterName + "-C"
+	)
+	// Configure the management server with an aggregate cluster resource graph
+	// that contains a cycle, but also contains a leaf cluster.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterNameA, []string{clusterNameB}),
+			makeAggregateClusterResource(clusterNameB, []string{clusterNameA, clusterNameC}),
+			e2e.DefaultCluster(clusterNameC, serviceName, e2e.SecurityLevelNone),
+		},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, host, []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the configuration pushed to the child policy.
+	wantChildCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config:                     createPriorityConfig(clusterNameC),
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+	if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the scenario where the cluster tree changes, and verifies that the
+// watchers for the cds balancer are updated accordingly. That is the cluster
+// removed from the tree no longer has a watcher and the new cluster added has a
+// new watcher.
+func (s) TestWatchers(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cdsResourceRequestedCh := make(chan []string, 1)
+	onStreamReq := func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+		if req.GetTypeUrl() == version.V3ClusterURL {
+			if len(req.GetResourceNames()) > 0 {
+				select {
+				case cdsResourceRequestedCh <- req.GetResourceNames():
+				case <-ctx.Done():
+				}
+			}
+		}
+		return nil
+	}
+	mgmtServer, nodeID, _ := setupWithManagementServer(t, nil, onStreamReq)
+
+	const (
+		clusterA = clusterName
+		clusterB = clusterName + "-B"
+		clusterC = clusterName + "-C"
+		clusterD = clusterName + "-D"
+	)
+
+	// Initial CDS resources: A -> B, C
+	initialResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterA)},
+		Clusters: []*v3clusterpb.Cluster{
+			makeAggregateClusterResource(clusterA, []string{clusterB, clusterC}),
+		},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, initialResources); err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+	wantNames := []string{clusterA, clusterB, clusterC}
+	if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the CDS resources to remove cluster C and add cluster D.
+	initialResources.Clusters = []*v3clusterpb.Cluster{
+		makeAggregateClusterResource(clusterA, []string{clusterB, clusterD}),
+	}
+	if err := mgmtServer.Update(ctx, initialResources); err != nil {
+		t.Fatalf("Update failed: %v", err)
+	}
+	wantNames = []string{clusterA, clusterB, clusterD}
+	if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/balancer/cdsbalancer/cdsbalancer.go

@@ -0,0 +1,472 @@
+/*
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package cdsbalancer implements a balancer to handle CDS responses.
+package cdsbalancer
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/attributes"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/nop"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsdepmgr"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const cdsName = "cds_experimental"
+
+var (
+	// newChildBalancer is a helper function to build a new priority balancer
+	// and will be overridden in unittests.
+	newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) {
+		builder := balancer.Get(priority.Name)
+		if builder == nil {
+			return nil, fmt.Errorf("xds: no balancer builder with name %v", priority.Name)
+		}
+		// We directly pass the parent clientConn to the underlying priority
+		// balancer because the cdsBalancer does not deal with subConns.
+		return builder.Build(cc, opts), nil
+	}
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+// bb implements the balancer.Builder interface to help build a cdsBalancer.
+// It also implements the balancer.ConfigParser interface to help parse the
+// JSON service config, to be passed to the cdsBalancer.
+type bb struct{}
+
+// Build creates a new CDS balancer with the ClientConn.
+func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	builder := balancer.Get(priority.Name)
+	if builder == nil {
+		// Shouldn't happen, registered through imported Priority builder. Still,
+		// defensive programming.
+		logger.Errorf("%q LB policy is needed but not registered", priority.Name)
+		return nop.NewBalancer(cc, fmt.Errorf("%q LB policy is needed but not registered", priority.Name))
+	}
+	parser, ok := builder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported Priority builder has this method.
+		logger.Errorf("%q LB policy does not implement a config parser", priority.Name)
+		return nop.NewBalancer(cc, fmt.Errorf("%q LB policy does not implement a config parser", priority.Name))
+	}
+
+	b := &cdsBalancer{
+		bOpts:             opts,
+		childConfigParser: parser,
+		clusterConfigs:    make(map[string]*xdsresource.ClusterResult),
+		priorityConfigs:   make(map[string]*priorityConfig),
+		cc:                cc,
+	}
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	return b
+}
+
+// Name returns the name of balancers built by this builder.
+func (bb) Name() string {
+	return cdsName
+}
+
+// lbConfig represents the loadBalancingConfig section of the service config
+// for the cdsBalancer.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	ClusterName string `json:"cluster"`
+	IsDynamic   bool   `json:"isDynamic"`
+}
+
+// ParseConfig parses the JSON load balancer config provided into an
+// internal form or returns an error if the config is invalid.
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	var cfg lbConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, fmt.Errorf("xds: unable to unmarshal lbconfig: %s, error: %v", string(c), err)
+	}
+	return &cfg, nil
+}
+
+// cdsBalancer implements a CDS based LB policy. It instantiates a
+// cluster_resolver balancer to further resolve the serviceName received from
+// CDS, into localities and endpoints. Implements the balancer.Balancer
+// interface which is exposed to gRPC and implements the balancer.ClientConn
+// interface which is exposed to the cluster_resolver balancer.
+type cdsBalancer struct {
+	// The following fields are initialized at build time and are either
+	// read-only after that or provide their own synchronization, and therefore
+	// do not need to be guarded by a mutex.
+	cc                balancer.ClientConn   // ClientConn interface passed to child LB.
+	bOpts             balancer.BuildOptions // BuildOptions passed to child LB.
+	childConfigParser balancer.ConfigParser // Config parser for cluster_resolver LB policy.
+	logger            *grpclog.PrefixLogger // Prefix logger for all logging.
+
+	// All fields below are accessed only from methods implementing the
+	// balancer.Balancer interface. Since gRPC guarantees that these methods are
+	// never invoked concurrently, no additional synchronization is required to
+	// protect access to these fields.
+	xdsClient         xdsclient.XDSClient
+	childLB           balancer.Balancer                     // Child policy, built upon resolution of the cluster graph.
+	clusterConfigs    map[string]*xdsresource.ClusterResult // Cluster name to the last received result for that cluster.
+	priorityConfigs   map[string]*priorityConfig            // Hostname to priority config for that leaf cluster.
+	lbCfg             *lbConfig                             // Current load balancing configuration.
+	priorities        []*priorityConfig                     // List of priorities in the order.
+	unsubscribe       func()                                // For dynamic cluster unsubscription.
+	isSubscribed      bool                                  // True if a dynamic cluster has been subscribed to.
+	clusterSubscriber xdsdepmgr.ClusterSubscriber           // To subscribe to dynamic cluster resource.
+	xdsLBPolicy       internalserviceconfig.BalancerConfig  // Stores the locality and endpoint picking policy.
+	attributes        *attributes.Attributes                // Attributes from resolver state.
+	serviceConfig     *serviceconfig.ParseResult
+	// Each new leaf cluster needs a child name generator to reuse child policy
+	// names. But to make sure the names across leaf clusters doesn't conflict,
+	// we need a seq ID. This ID is incremented for each new cluster.
+	childNameGeneratorSeqID uint64
+}
+
+// UpdateClientConnState receives the serviceConfig, xdsConfig,
+// ClusterSubscriber and the xdsClient object from the xdsResolver. If an error
+// is encountered, the parent (clustermanager) sets the corresponding cluster’s
+// picker to transient_failure. Otherwise, the received configuration is
+// processed and forwarded to the appropriate child policy.
+func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error {
+	if b.xdsClient == nil {
+		c := xdsclient.FromResolverState(state.ResolverState)
+		if c == nil {
+			b.logger.Warningf("Received balancer config with no xDS client")
+			return balancer.ErrBadResolverState
+		}
+		b.xdsClient = c
+	}
+	b.logger.Infof("Received balancer config update: %s", pretty.ToJSON(state.BalancerConfig))
+
+	xdsConfig := xdsresource.XDSConfigFromResolverState(state.ResolverState)
+	if xdsConfig == nil {
+		b.logger.Warningf("Received balancer config with no xDS config")
+		return balancer.ErrBadResolverState
+	}
+	b.clusterConfigs = xdsConfig.Clusters
+	b.clusterSubscriber = xdsdepmgr.XDSClusterSubscriberFromResolverState(state.ResolverState)
+	if b.clusterSubscriber == nil {
+		b.logger.Warningf("Received balancer config with no cluster subscriber")
+		return balancer.ErrBadResolverState
+	}
+	// The errors checked here should ideally never happen because the
+	// ServiceConfig in this case is prepared by the xdsResolver and is not
+	// something that is received on the wire.
+	lbCfg, ok := state.BalancerConfig.(*lbConfig)
+	if !ok {
+		b.logger.Warningf("Received unexpected balancer config type: %T", state.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+	if lbCfg.ClusterName == "" {
+		b.logger.Warningf("Received balancer config with no cluster name")
+		return balancer.ErrBadResolverState
+	}
+
+	b.lbCfg = lbCfg
+	b.serviceConfig = state.ResolverState.ServiceConfig
+	b.attributes = state.ResolverState.Attributes
+	return b.handleXDSConfigUpdate()
+}
+
+// handleXDSConfigUpdate processes the XDSConfig update from the xDS resolver.
+func (b *cdsBalancer) handleXDSConfigUpdate() error {
+	clusterName := b.lbCfg.ClusterName
+
+	// If the cluster is dynamic and we dont have a subscription yet, create
+	// one.
+	if b.lbCfg.IsDynamic && !b.isSubscribed {
+		b.unsubscribe = b.clusterSubscriber.SubscribeToCluster(clusterName)
+		b.isSubscribed = true
+		return nil
+	}
+
+	clusterUpdate, ok := b.clusterConfigs[clusterName]
+	if !ok {
+		// If the cluster is missing from the config, check if it is dynamic.
+		// For dynamic clusters, the xDS config may be updated before the
+		// corresponding cluster resource is received. This should never occur
+		// for static clusters.
+		if b.lbCfg.IsDynamic {
+			return nil
+		}
+		return b.annotateErrorWithNodeID(fmt.Errorf("did not find the cluster %q in XDSConfig", clusterName))
+	}
+	// If the cluster resource has an error, return the error.
+	if clusterUpdate.Err != nil {
+		return clusterUpdate.Err
+	}
+	return b.handleClusterUpdate()
+}
+
+// handleClusterUpdate handles a good XDSConfig update from the xDS resolver.
+// Builds the child policy config and pushes it down.
+func (b *cdsBalancer) handleClusterUpdate() error {
+	clusterName := b.lbCfg.ClusterName
+	clusterConfig := b.clusterConfigs[clusterName].Config
+
+	var newPriorities []*priorityConfig
+	switch clusterConfig.Cluster.ClusterType {
+	case xdsresource.ClusterTypeEDS, xdsresource.ClusterTypeLogicalDNS:
+		p := b.updatePriorityConfig(clusterName, &clusterConfig)
+		newPriorities = append(newPriorities, p)
+	case xdsresource.ClusterTypeAggregate:
+		for _, leaf := range clusterConfig.AggregateConfig.LeafClusters {
+			leafCluster := b.clusterConfigs[leaf]
+			// Update priority config for leaf clusters.
+			p := b.updatePriorityConfig(leaf, &leafCluster.Config)
+			newPriorities = append(newPriorities, p)
+		}
+	}
+	b.priorities = newPriorities
+
+	if err := b.updateOutlierDetection(); err != nil {
+		return b.annotateErrorWithNodeID(fmt.Errorf("failed to correctly update Outlier Detection config %v", err))
+	}
+
+	// The LB policy is configured by the root cluster.
+	if err := json.Unmarshal(clusterConfig.Cluster.LBPolicy, &b.xdsLBPolicy); err != nil {
+		return b.annotateErrorWithNodeID(fmt.Errorf("error unmarshalling xDS LB Policy: %v", err))
+	}
+	if err := b.updateChildConfig(); err != nil {
+		return b.annotateErrorWithNodeID(err)
+	}
+	return nil
+}
+
+// updateChildConfig builds child policy configuration using endpoint addresses
+// returned from the XDSConfig and child policy configuration.
+//
+// A child policy is created if one doesn't already exist. The newly built
+// configuration is then pushed to the child policy.
+func (b *cdsBalancer) updateChildConfig() error {
+	if b.childLB == nil {
+		childLB, err := newChildBalancer(b.cc, b.bOpts)
+		if err != nil {
+			return fmt.Errorf("failed to create child policy of type %s: %v", priority.Name, err)
+		}
+		b.childLB = childLB
+	}
+
+	childCfgBytes, endpoints, err := buildPriorityConfigJSON(b.priorities, &b.xdsLBPolicy)
+	if err != nil {
+		return fmt.Errorf("failed to build child policy config: %v", err)
+	}
+	childCfg, err := b.childConfigParser.ParseConfig(childCfgBytes)
+	if err != nil {
+		return fmt.Errorf("failed to parse child policy config. This should never happen because the config was generated: %v", err)
+	}
+	if b.logger.V(2) {
+		b.logger.Infof("Built child policy config: %s", pretty.ToJSON(childCfg))
+	}
+
+	for i := range endpoints {
+		for j := range endpoints[i].Addresses {
+			addr := endpoints[i].Addresses[j]
+			addr.BalancerAttributes = endpoints[i].Attributes
+			// BalancerAttributes are used for the following:
+			// * Authority Override.
+			// * grpc.lb.backend_service metric label propagation.
+			// See https://github.com/grpc/grpc-go/issues/6472
+			endpoints[i].Addresses[j] = addr
+		}
+	}
+	if err := b.childLB.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Endpoints:     endpoints,
+			ServiceConfig: b.serviceConfig,
+			Attributes:    b.attributes,
+		},
+		BalancerConfig: childCfg,
+	}); err != nil {
+		return fmt.Errorf("failed to push config to child policy: %v", err)
+	}
+	return nil
+}
+
+// updatePriorityConfig updates the priority configuration for the specified EDS
+// or DNS cluster, creating it if it does not already exist.
+func (b *cdsBalancer) updatePriorityConfig(clusterName string, clusterConfig *xdsresource.ClusterConfig) *priorityConfig {
+	name := hostName(clusterName, *clusterConfig.Cluster)
+	pc, ok := b.priorityConfigs[name]
+	if !ok {
+		pc = &priorityConfig{
+			childNameGen: newNameGenerator(b.childNameGeneratorSeqID),
+		}
+		b.priorityConfigs[name] = pc
+		// Increment the seq ID for the next new cluster. This is done to make
+		// sure that the child policy names generated for different clusters
+		// don't conflict with each other.
+		b.childNameGeneratorSeqID++
+	}
+	pc.clusterConfig = clusterConfig
+	return pc
+}
+
+// updateOutlierDetection updates Outlier Detection config for all priorities.
+func (b *cdsBalancer) updateOutlierDetection() error {
+	odBuilder := balancer.Get(outlierdetection.Name)
+	if odBuilder == nil {
+		// Shouldn't happen, registered through imported Outlier Detection,
+		// defensive programming.
+		return fmt.Errorf("%q LB policy is needed but not registered", outlierdetection.Name)
+	}
+
+	odParser, ok := odBuilder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported Outlier Detection builder has this method.
+		return fmt.Errorf("%q LB policy does not implement a config parser", outlierdetection.Name)
+	}
+
+	for _, p := range b.priorities {
+		// Update Outlier Detection Config.
+		odJSON := p.clusterConfig.Cluster.OutlierDetection
+		if odJSON == nil {
+			odJSON = json.RawMessage(`{}`)
+		}
+
+		lbCfg, err := odParser.ParseConfig(odJSON)
+		if err != nil {
+			return fmt.Errorf("error parsing Outlier Detection config %v: %v", odJSON, err)
+		}
+
+		odCfg, ok := lbCfg.(*outlierdetection.LBConfig)
+		if !ok {
+			// Shouldn't happen, Parser built at build time with Outlier
+			// Detection builder pulled from gRPC LB Registry.
+			return fmt.Errorf("config parser for Outlier Detection returned config with unexpected type %T: %v", lbCfg, lbCfg)
+		}
+		p.outlierDetection = *odCfg
+	}
+	return nil
+}
+
+// ResolverError handles errors reported by the xdsResolver.
+func (b *cdsBalancer) ResolverError(err error) {
+	// Missing Listener or RouteConfiguration on the management server
+	// results in a 'resource not found' error from the xDS resolver. In
+	// these cases, we should report transient failure.
+	if xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound {
+		b.closeChildPolicyAndReportTF(err)
+		return
+	}
+	var root string
+	if b.lbCfg != nil {
+		root = b.lbCfg.ClusterName
+	}
+	b.onClusterError(root, err)
+}
+
+// UpdateSubConnState handles subConn updates from gRPC.
+func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+// closeChildPolicyAndReportTF closes the child policy, if it exists, and
+// updates the connectivity state of the channel to TransientFailure with an
+// error picker.
+func (b *cdsBalancer) closeChildPolicyAndReportTF(err error) {
+	if b.childLB != nil {
+		b.childLB.Close()
+		b.childLB = nil
+	}
+	b.cc.UpdateState(balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	})
+}
+
+// Close closes the child policy, unsubscribes to the dynamic cluster, and
+// closes the cdsBalancer.
+func (b *cdsBalancer) Close() {
+	if b.childLB != nil {
+		b.childLB.Close()
+		b.childLB = nil
+	}
+	if b.unsubscribe != nil {
+		b.unsubscribe()
+	}
+	b.logger.Infof("Shutdown")
+}
+
+func (b *cdsBalancer) ExitIdle() {
+	if b.childLB == nil {
+		b.logger.Warningf("Received ExitIdle with no child policy")
+		return
+	}
+	// This implementation assumes the child balancer supports
+	// ExitIdle (but still checks for the interface's existence to
+	// avoid a panic if not). If the child does not, no subconns
+	// will be connected.
+	b.childLB.ExitIdle()
+}
+
+// Node ID needs to be manually added to errors generated in the following
+// scenarios:
+//   - resource-does-not-exist: since the xDS watch API uses a separate callback
+//     instead of returning an error value. TODO(gRFC A88): Once A88 is
+//     implemented, the xDS client will be able to add the node ID to
+//     resource-does-not-exist errors as well, and we can get rid of this
+//     special handling.
+//   - received a good update from the xDS client, but the update either contains
+//     an invalid security configuration or contains invalid aggragate cluster
+//     config.
+func (b *cdsBalancer) annotateErrorWithNodeID(err error) error {
+	nodeID := b.xdsClient.BootstrapConfig().Node().GetId()
+	return fmt.Errorf("[xDS node id: %v]: %w", nodeID, err)
+}
+
+// onClusterAmbientError handles an ambient error, if a childLB already has a
+// good update, it should continue using that.
+func (b *cdsBalancer) onClusterAmbientError(name string, err error) {
+	b.logger.Warningf("Cluster resource %q received ambient error update: %v", name, err)
+
+	if xdsresource.ErrType(err) != xdsresource.ErrorTypeConnection && b.childLB != nil {
+		// Connection errors will be sent to the child balancers directly.
+		// There's no need to forward them.
+		b.childLB.ResolverError(err)
+	}
+}
+
+// onClusterResourceError handles errors to stop using the previously seen
+// resource. Propagates the error down to the child policy if one exists, and
+// puts the channel in TRANSIENT_FAILURE.
+func (b *cdsBalancer) onClusterResourceError(name string, err error) {
+	b.logger.Warningf("CDS watch for resource %q reported resource error", name)
+	b.closeChildPolicyAndReportTF(err)
+}
+
+func (b *cdsBalancer) onClusterError(name string, err error) {
+	if b.childLB != nil {
+		b.onClusterAmbientError(name, err)
+	} else {
+		b.onClusterResourceError(name, err)
+	}
+}

internal/xds/balancer/cdsbalancer/configbuilder.go

@@ -0,0 +1,375 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cdsbalancer
+
+import (
+	"encoding/json"
+	"fmt"
+	"maps"
+	"slices"
+
+	"google.golang.org/grpc/experimental/balancer/hostname"
+	"google.golang.org/grpc/experimental/balancer/weight"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/proxyattributes"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+)
+
+const million = 1000000
+
+// priorityConfig is config for one priority. For example, if there's an EDS and
+// a DNS, the priority list will be [priorityConfig{EDS}, priorityConfig{DNS}].
+//
+// Each priorityConfig corresponds to one leaf cluster retrieved from XDSConfig
+// for the top-level cluster.
+type priorityConfig struct {
+	// clusterConfig has the cluster update as well as EDS or DNS endpoints
+	// depending on the leaf cluster type.
+	clusterConfig *xdsresource.ClusterConfig
+	// outlierDetection is the Outlier Detection LB configuration for this
+	// priority.
+	outlierDetection outlierdetection.LBConfig
+	// Each leaf cluster has a name generator so that the child policies can
+	// reuse names between updates (EDS updates for example).
+	childNameGen *nameGenerator
+}
+
+// hostName returns the name of the host for the given cluster.
+//
+// For EDS, it's the EDSServiceName (or ClusterName if empty).
+// For DNS, it's the DNSHostName.
+func hostName(clusterName string, update xdsresource.ClusterUpdate) string {
+	switch update.ClusterType {
+	case xdsresource.ClusterTypeEDS:
+		if update.EDSServiceName != "" {
+			return update.EDSServiceName
+		}
+		return clusterName
+	case xdsresource.ClusterTypeLogicalDNS:
+		return update.DNSHostName
+	default:
+		return ""
+	}
+}
+
+// buildPriorityConfigJSON builds balancer config for the passed in
+// priorities.
+//
+// The built tree of balancers (see test for the output struct).
+//
+//	          ┌────────┐
+//	          │priority│
+//	          └┬──────┬┘
+//	           │      │
+//	┌──────────▼─┐  ┌─▼──────────┐
+//	│cluster_impl│  │cluster_impl│
+//	└──────┬─────┘  └─────┬──────┘
+//	       │              │
+//	┌──────▼─────┐  ┌─────▼──────┐
+//	│xDSLBPolicy │  │xDSLBPolicy │ (Locality and Endpoint picking layer)
+//	└────────────┘  └────────────┘
+func buildPriorityConfigJSON(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]byte, []resolver.Endpoint, error) {
+	pc, endpoints, err := buildPriorityConfig(priorities, xdsLBPolicy)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to build priority config: %v", err)
+	}
+	ret, err := json.Marshal(pc)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to marshal built priority config struct into json: %v", err)
+	}
+	return ret, endpoints, nil
+}
+
+func buildPriorityConfig(priorities []*priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*priority.LBConfig, []resolver.Endpoint, error) {
+	var (
+		retConfig    = &priority.LBConfig{Children: make(map[string]*priority.Child)}
+		retEndpoints []resolver.Endpoint
+	)
+	for _, p := range priorities {
+		clusterUpdate := p.clusterConfig.Cluster
+		switch clusterUpdate.ClusterType {
+		case xdsresource.ClusterTypeEDS:
+			names, configs, endpoints, err := buildClusterImplConfigForEDS(p.childNameGen, p.clusterConfig, xdsLBPolicy)
+			if err != nil {
+				return nil, nil, err
+			}
+			retConfig.Priorities = append(retConfig.Priorities, names...)
+			retEndpoints = append(retEndpoints, endpoints...)
+			odCfgs := convertClusterImplMapToOutlierDetection(configs, p.outlierDetection)
+			for n, c := range odCfgs {
+				retConfig.Children[n] = &priority.Child{
+					Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: c},
+					// Ignore all re-resolution from EDS children.
+					IgnoreReresolutionRequests: true,
+				}
+			}
+			continue
+		case xdsresource.ClusterTypeLogicalDNS:
+			name, config, endpoints := buildClusterImplConfigForDNS(p.childNameGen, p.clusterConfig, xdsLBPolicy)
+			retConfig.Priorities = append(retConfig.Priorities, name)
+			retEndpoints = append(retEndpoints, endpoints...)
+			odCfg := makeClusterImplOutlierDetectionChild(config, p.outlierDetection)
+			retConfig.Children[name] = &priority.Child{
+				Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: odCfg},
+				// Not ignore re-resolution from DNS children, they will trigger
+				// DNS to re-resolve.
+				IgnoreReresolutionRequests: false,
+			}
+			continue
+		}
+	}
+	return retConfig, retEndpoints, nil
+}
+
+func convertClusterImplMapToOutlierDetection(ciCfgs map[string]*clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) map[string]*outlierdetection.LBConfig {
+	odCfgs := make(map[string]*outlierdetection.LBConfig, len(ciCfgs))
+	for n, c := range ciCfgs {
+		odCfgs[n] = makeClusterImplOutlierDetectionChild(c, odCfg)
+	}
+	return odCfgs
+}
+
+func makeClusterImplOutlierDetectionChild(ciCfg *clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) *outlierdetection.LBConfig {
+	odCfgRet := odCfg
+	odCfgRet.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: ciCfg}
+	return &odCfgRet
+}
+
+func buildClusterImplConfigForDNS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (string, *clusterimpl.LBConfig, []resolver.Endpoint) {
+	pName := fmt.Sprintf("priority-%v", g.prefix)
+	clusterUpdate := config.Cluster
+	lbconfig := &clusterimpl.LBConfig{
+		Cluster:     clusterUpdate.ClusterName,
+		ChildPolicy: xdsLBPolicy,
+	}
+	endpoints := config.EndpointConfig.DNSEndpoints.Endpoints
+	if len(endpoints) == 0 {
+		return pName, lbconfig, nil
+	}
+	var retEndpoint resolver.Endpoint
+	for _, e := range endpoints {
+		// LOGICAL_DNS requires all resolved addresses to be grouped into a
+		// single logical endpoint. We iterate over the input endpoints and
+		// aggregate their addresses into a new endpoint variable.
+		retEndpoint.Addresses = append(retEndpoint.Addresses, e.Addresses...)
+	}
+	// Even though localities are not a thing for the LOGICAL_DNS cluster and
+	// its endpoint(s), we add an empty locality attribute here to ensure that
+	// LB policies that rely on locality information (like weighted_target)
+	// continue to work.
+	localityStr := xdsinternal.LocalityString(clients.Locality{})
+	retEndpoint = hostname.Set(hierarchy.SetInEndpoint(retEndpoint, []string{pName, localityStr}), clusterUpdate.DNSHostName)
+	// Set the locality weight to 1. This is required because the child policy
+	// like weighted_target which relies on locality weights to distribute
+	// traffic. These policies may drop traffic if the weight is 0.
+	retEndpoint = wrrlocality.SetAddrInfo(retEndpoint, wrrlocality.AddrInfo{LocalityWeight: 1})
+	return pName, lbconfig, []resolver.Endpoint{retEndpoint}
+}
+
+// buildClusterImplConfigForEDS returns a list of cluster_impl configs, one for
+// each priority, sorted by priority, and the addresses for each priority (with
+// hierarchy attributes set).
+//
+// For example, if there are two priorities, the returned values will be
+// - ["p0", "p1"]
+// - map{"p0":p0_config, "p1":p1_config}
+// - [p0_address_0, p0_address_1, p1_address_0, p1_address_1]
+//   - p0 addresses' hierarchy attributes are set to p0
+func buildClusterImplConfigForEDS(g *nameGenerator, config *xdsresource.ClusterConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]string, map[string]*clusterimpl.LBConfig, []resolver.Endpoint, error) {
+	edsUpdate := config.EndpointConfig.EDSUpdate
+
+	// Localities of length 0 is triggered by an NACK or resource-not-found
+	// error before update, or an empty localities list in an update. In either
+	// case want to create a priority, and send down empty address list, causing
+	// TF for that priority. "If any discovery mechanism instance experiences an
+	// error retrieving data, and it has not previously reported any results, it
+	// should report a result that is a single priority with no endpoints." -
+	// A37
+	priorities := [][]xdsresource.Locality{{}}
+	if len(edsUpdate.Localities) != 0 {
+		priorities = groupLocalitiesByPriority(edsUpdate.Localities)
+	}
+	retNames := g.generate(priorities)
+	retConfigs := make(map[string]*clusterimpl.LBConfig, len(retNames))
+	var retEndpoints []resolver.Endpoint
+	for i, pName := range retNames {
+		priorityLocalities := priorities[i]
+		cfg, endpoints, err := priorityLocalitiesToClusterImpl(priorityLocalities, pName, *config.Cluster, xdsLBPolicy)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		retConfigs[pName] = cfg
+		retEndpoints = append(retEndpoints, endpoints...)
+	}
+	return retNames, retConfigs, retEndpoints, nil
+}
+
+// groupLocalitiesByPriority returns the localities grouped by priority.
+//
+// The returned list is sorted from higher priority to lower. Each item in the
+// list is a group of localities.
+//
+// For example, for L0-p0, L1-p0, L2-p1, results will be
+// - [[L0, L1], [L2]]
+func groupLocalitiesByPriority(localities []xdsresource.Locality) [][]xdsresource.Locality {
+	priorities := make(map[int][]xdsresource.Locality)
+	for _, locality := range localities {
+		priority := int(locality.Priority)
+		priorities[priority] = append(priorities[priority], locality)
+	}
+	// Sort the priorities based on the int value, deduplicate, and then turn
+	// the sorted list into a string list. This will be child names, in priority
+	// order.
+	priorityIntSlice := slices.Sorted(maps.Keys(priorities))
+	ret := make([][]xdsresource.Locality, 0, len(priorityIntSlice))
+	for _, p := range priorityIntSlice {
+		ret = append(ret, priorities[p])
+	}
+	return ret
+}
+
+// priorityLocalitiesToClusterImpl takes a list of localities (with the same
+// priority), and generates a cluster impl policy config, and a list of
+// addresses with their path hierarchy set to [priority-name, locality-name], so
+// priority and the xDS LB Policy know which child policy each address is for.
+func priorityLocalitiesToClusterImpl(localities []xdsresource.Locality, priorityName string, clusterUpdate xdsresource.ClusterUpdate, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*clusterimpl.LBConfig, []resolver.Endpoint, error) {
+	var retEndpoints []resolver.Endpoint
+
+	// Compute the sum of locality weights to normalize locality weights. The
+	// xDS client guarantees that the sum of locality weights (within a
+	// priority) will not exceed MaxUint32.
+	var localityWeightSum uint32
+	for _, locality := range localities {
+		localityWeightSum += locality.Weight
+	}
+
+	for _, locality := range localities {
+		// Compute the sum of endpoint weights to normalize endpoint weights.
+		// The xDS client does not currently guarantee that the sum of endpoint
+		// weights (within a locality) will not exceed MaxUint32. TODO(i/8862):
+		// Once the xDS client guarantees that the sum of endpoint weights does
+		// not exceed MaxUint32, we can change the type of this variable from
+		// uint64 to uint32.
+		var endpointWeightSum uint64
+		for _, endpoint := range locality.Endpoints {
+			endpointWeightSum += uint64(endpoint.Weight)
+		}
+
+		localityStr := xdsinternal.LocalityString(locality.ID)
+		for _, endpoint := range locality.Endpoints {
+			// Filter out all "unhealthy" endpoints (unknown and healthy are
+			// both considered to be healthy:
+			// https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus).
+			if endpoint.HealthStatus != xdsresource.EndpointHealthStatusHealthy && endpoint.HealthStatus != xdsresource.EndpointHealthStatusUnknown {
+				continue
+			}
+
+			// Create a copy of endpoint.ResolverEndpoint to avoid race between
+			// the xDS Client (which owns this shared object in its cache) and
+			// the Cluster Resolver (which is trying to modify attributes).
+			resolverEndpoint := endpoint.ResolverEndpoint
+			resolverEndpoint.Addresses = slices.Clone(endpoint.ResolverEndpoint.Addresses)
+
+			if clusterUpdate.IsHTTP11ProxyEnabled {
+				var proxyAddrStr string
+				if val, ok := endpoint.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok {
+					proxyAddrStr = val.Address
+				} else if val, ok := locality.Metadata["envoy.http11_proxy_transport_socket.proxy_address"].(xdsresource.ProxyAddressMetadataValue); ok {
+					proxyAddrStr = val.Address
+				}
+				if proxyAddrStr != "" {
+					for idx, addr := range resolverEndpoint.Addresses {
+						connectAddr := addr.Addr
+						addr.Addr = proxyAddrStr
+						resolverEndpoint.Addresses[idx] = proxyattributes.Set(addr, proxyattributes.Options{
+							ConnectAddr: connectAddr,
+						})
+					}
+				}
+			}
+
+			resolverEndpoint = hierarchy.SetInEndpoint(resolverEndpoint, []string{priorityName, localityStr})
+			resolverEndpoint = xdsinternal.SetLocalityIDInEndpoint(resolverEndpoint, locality.ID)
+			// "To provide the xds_wrr_locality load balancer information about
+			// locality weights received from EDS, the cluster resolver will
+			// populate a new locality weight attribute for each address The
+			// attribute will have the weight (as an integer) of the locality
+			// the address is part of." - A52
+			resolverEndpoint = wrrlocality.SetAddrInfo(resolverEndpoint, wrrlocality.AddrInfo{LocalityWeight: locality.Weight})
+
+			if envconfig.PickFirstWeightedShuffling {
+				normalizedLocalityWeight := fractionToFixedPoint(uint64(locality.Weight), uint64(localityWeightSum))
+				normalizedEndpointWeight := fractionToFixedPoint(uint64(endpoint.Weight), endpointWeightSum)
+				endpointWeight := fixedPointMultiply(normalizedEndpointWeight, normalizedLocalityWeight)
+				if endpointWeight == 0 {
+					endpointWeight = 1
+				}
+				resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: endpointWeight})
+			} else {
+				resolverEndpoint = weight.Set(resolverEndpoint, weight.EndpointInfo{Weight: locality.Weight * endpoint.Weight})
+			}
+			retEndpoints = append(retEndpoints, resolverEndpoint)
+		}
+	}
+	return &clusterimpl.LBConfig{
+		Cluster:     clusterUpdate.ClusterName,
+		ChildPolicy: xdsLBPolicy,
+	}, retEndpoints, nil
+}
+
+// fixedPointFractionalBits is the number of bits used for the fractional part
+// of normalized endpoint and locality weights.
+//
+// We use the UQ1.31 fixed-point format (Unsigned, 1 integer bit, 31 fractional bits).
+// This allows representing values in the range [0.0, 2.0) with a precision
+// of 2^-31.
+//
+// Bit Layout:
+// [ 31 ] [ 30 ................. 0 ]
+//
+//	|              |
+//	|              +--- Fractional Part (31 bits)
+//	+------------------ Integer Part (1 bit)
+//
+// See gRFC A113 for more details.
+const fixedPointFractionalBits = 31
+
+// fractionToFixedPoint converts a fraction represented by numerator and
+// denominator to a fixed-point number between 0 and 1 represented as a uint32.
+//
+// The xDS client guarantees that the sum of locality weights (within a
+// priority) will not exceed MaxUint32. TODO(i/8862): Once the xDS client
+// guarantees that the sum of endpoint weights does not exceed MaxUint32, we can
+// change the types of this function's arguments from uint64 to uint32.
+func fractionToFixedPoint(numerator, denominator uint64) uint32 {
+	return uint32(uint64(numerator) << fixedPointFractionalBits / uint64(denominator))
+}
+
+func fixedPointMultiply(a, b uint32) uint32 {
+	return uint32((uint64(a) * uint64(b)) >> fixedPointFractionalBits)
+}

internal/xds/balancer/cdsbalancer/configbuilder_childname.go

@@ -0,0 +1,111 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+)
+
+// nameGenerator generates a child name for a list of priorities (each priority
+// is a list of localities).
+//
+// The purpose of this generator is to reuse names between updates. So the
+// struct keeps state between generate() calls, and a later generate() might
+// return names returned by the previous call.
+type nameGenerator struct {
+	prevLocalitiesToChildNames map[clients.Locality]string // locality to child name mapping generated for the previous update
+	prevChildNames             []string                    // prioritized list of child names generated for the previous update
+	prefix                     uint64
+	nextID                     uint64
+}
+
+func newNameGenerator(prefix uint64) *nameGenerator {
+	return &nameGenerator{prefix: prefix}
+}
+
+// generate returns a list of names for the given list of priorities.
+//
+// Each priority is a list of localities. The name for the priority is picked as
+// - for each locality in this priority, if it exists in the existing names,
+// this priority will reuse the name
+// - if no reusable name is found for this priority, a new name is generated
+//
+// For example:
+// - update 1: [[L1], [L2], [L3]] --> ["0", "1", "2"]
+// - update 2: [[L1], [L2], [L3]] --> ["0", "1", "2"]
+// - update 3: [[L1, L2], [L3]] --> ["0", "2"]   (Two priorities were merged)
+// - update 4: [[L1], [L4]] --> ["0", "3",]      (A priority was split, and a new priority was added)
+func (ng *nameGenerator) generate(priorities [][]xdsresource.Locality) []string {
+	ret := make([]string, len(priorities))
+	usedNames := make(map[string]bool)
+	newNames := make(map[clients.Locality]string)
+
+	// Pass 1: Same priority index match.
+	for i, priority := range priorities {
+		if i >= len(ng.prevChildNames) {
+			continue
+		}
+		targetName := ng.prevChildNames[i]
+		for _, locality := range priority {
+			if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok && name == targetName {
+				ret[i] = targetName
+				usedNames[targetName] = true
+				break
+			}
+		}
+	}
+
+	// Pass 2: Greedy reuse.
+	for i, priority := range priorities {
+		if ret[i] != "" {
+			continue
+		}
+		for _, locality := range priority {
+			if name, ok := ng.prevLocalitiesToChildNames[locality.ID]; ok {
+				if !usedNames[name] {
+					ret[i] = name
+					usedNames[name] = true
+					break
+				}
+			}
+		}
+	}
+
+	// Pass 3: New name.
+	for i, name := range ret {
+		if name == "" {
+			newID := fmt.Sprintf("priority-%d-%d", ng.prefix, ng.nextID)
+			ng.nextID++
+			ret[i] = newID
+			usedNames[newID] = true
+		}
+	}
+
+	// Update state.
+	for i, priority := range priorities {
+		for _, l := range priority {
+			newNames[l.ID] = ret[i]
+		}
+	}
+	ng.prevLocalitiesToChildNames = newNames
+	ng.prevChildNames = ret
+	return ret
+}

internal/xds/balancer/cdsbalancer/configbuilder_childname_test.go

@@ -0,0 +1,311 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cdsbalancer
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+)
+
+func (s) Test_nameGenerator_generate(t *testing.T) {
+	tests := []struct {
+		name   string
+		prefix uint64
+		steps  []struct {
+			input [][]xdsresource.Locality
+			want  []string
+		}
+	}{
+		{
+			name:   "init, two new priorities",
+			prefix: 3,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-3-0", "priority-3-1"},
+				},
+			},
+		},
+		{
+			name:   "one new priority",
+			prefix: 1,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+					},
+					want: []string{"priority-1-0"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-1-0", "priority-1-1"},
+				},
+			},
+		},
+		{
+			name:   "merge two priorities",
+			prefix: 4,
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-4-0", "priority-4-1", "priority-4-2"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-4-0", "priority-4-2"},
+				},
+			},
+		},
+		{
+			name: "swap two priorities",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1", "priority-0-2"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-1", "priority-0-0", "priority-0-2"},
+				},
+			},
+		},
+		{
+			name: "split priority",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-2", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "priority index preference",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L0"}}},
+					},
+					want: []string{"priority-0-0"},
+				},
+			},
+		},
+		{
+			name: "merge partial",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "swap shift",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-1", "priority-0-0"},
+				},
+			},
+		},
+		{
+			name: "replace priority",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L2"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}},
+						{{ID: clients.Locality{Zone: "L5"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-2"},
+				},
+			},
+		},
+		{
+			name: "reordered merge",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L1"}}, {ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}, {ID: clients.Locality{Zone: "L3"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+		{
+			name: "three-step shift stability",
+			steps: []struct {
+				input [][]xdsresource.Locality
+				want  []string
+			}{
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L2"}}},
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-2", "priority-0-0", "priority-0-1"},
+				},
+				{
+					input: [][]xdsresource.Locality{
+						{{ID: clients.Locality{Zone: "L0"}}},
+						{{ID: clients.Locality{Zone: "L1"}}},
+					},
+					want: []string{"priority-0-0", "priority-0-1"},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ng := newNameGenerator(tt.prefix)
+			for i, step := range tt.steps {
+				got := ng.generate(step.input)
+				if diff := cmp.Diff(got, step.want); diff != "" {
+					t.Errorf("step %d: generate() = got: %v, want: %v, diff (-got +want): %s", i, got, step.want, diff)
+				}
+			}
+		})
+	}
+}

internal/xds/balancer/cdsbalancer/e2e_test/balancer_test.go

@@ -0,0 +1,420 @@
+/*
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/internal/xds/balancer/priority"
+	"google.golang.org/grpc/internal/xds/balancer/wrrlocality"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer" // Register the "cds_experimental" LB policy.
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router"    // Register the router filter
+	_ "google.golang.org/grpc/internal/xds/resolver"             // Register the xds resolver
+)
+
+// setupAndDial performs common setup across all tests
+//
+//   - creates an xDS client with the passed in bootstrap contents
+//   - creates a  xds resolver to be used
+//   - creates a ClientConn to talk to the test backends
+//
+// Returns a function to close the ClientConn and the xDS client.
+func setupAndDial(t *testing.T, bootstrapContents []byte) (*grpc.ClientConn, func()) {
+	t.Helper()
+	// Create an xDS resolver with the above bootstrap configuration.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("xDS resolver creation failed: %v", err)
+	}
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	return cc, func() {
+		cc.Close()
+	}
+}
+
+// TestErrorFromParentLB_ConnectionError tests the case where a connection error
+// is sent by the CDS LB Policy. The CDS LB policy sends a connection error when
+// the ADS stream to the management server breaks. The test verifies that there
+// is no perceivable effect because of this connection error, and that RPCs
+// continue to work (because the LB policies are expected to use previously
+// received xDS resources).
+func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) {
+	// Create a listener to be used by the management server. The test will
+	// close this listener to simulate ADS stream breakage.
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	// Start an xDS management server with the above restartable listener, and
+	// push a channel when the stream is closed.
+	streamClosedCh := make(chan struct{}, 1)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			select {
+			case streamClosedCh <- struct{}{}:
+			default:
+			}
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		NodeID:     nodeID,
+		DialTarget: serviceName,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	cc, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Close the listener and ensure that the ADS stream breaks.
+	lis.Close()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to close")
+	default:
+	}
+
+	// Ensure that RPCs continue to succeed for the next second.
+	for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+	}
+}
+
+// TestErrorFromParentLB_ResourceNotFound tests the case where a
+// resource-not-found error is received by the CDS LB policy for a cluster. The
+// resource-not-found error is received when the cluster resource associated
+// with these LB policies is removed by the management server. The test verifies
+// that the associated EDS is canceled and RPCs fail. It also ensures that when
+// the Cluster resource is added back, the EDS resource is re-requested and RPCs
+// being to succeed.
+func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) {
+	// Start an xDS management server that uses a couple of channels to
+	// notify the test about the following events:
+	// - an EDS requested with the expected resource name is requested
+	// - EDS resource is unrequested, i.e, an EDS request with no resource name
+	//   is received, which indicates that we are no longer interested in that
+	//   resource.
+	edsResourceRequestedCh := make(chan struct{}, 1)
+	edsResourceCanceledCh := make(chan struct{}, 1)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if req.GetTypeUrl() == version.V3EndpointsURL {
+				switch len(req.GetResourceNames()) {
+				case 0:
+					select {
+					case edsResourceCanceledCh <- struct{}{}:
+					default:
+					}
+				case 1:
+					if req.GetResourceNames()[0] == edsServiceName {
+						select {
+						case edsResourceRequestedCh <- struct{}{}:
+						default:
+						}
+					}
+				default:
+					t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames()))
+				}
+			}
+			return nil
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure cluster and endpoints resources in the management server.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	cc, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	// Wait for the EDS resource to be requested.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to be requested")
+	case <-edsResourceRequestedCh:
+	}
+
+	// Ensure that a successful RPC can be made.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	oldCluster := resources.Clusters
+	// Delete the cluster resource from the management server.
+	resources.Clusters = nil
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the EDS resource to be not requested anymore.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to not be requested")
+	case <-edsResourceCanceledCh:
+	}
+
+	// Ensure that RPCs start to fail with expected error.
+	wantErr := fmt.Sprintf("resource %q of type %q has been removed", clusterName, "ClusterResource")
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		_, err := client.EmptyCall(sCtx, &testpb.Empty{})
+		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
+			break
+		}
+		if err != nil {
+			t.Logf("EmptyCall failed: %v", err)
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("RPCs did not fail after removal of Cluster resource")
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Add the cluster resource back to the management server.
+	resources.Clusters = oldCluster
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the EDS resource to be requested again.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for EDS resource to be requested")
+	case <-edsResourceRequestedCh:
+	}
+
+	// Ensure that a successful RPC can be made.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil {
+			t.Logf("EmptyCall failed: %v", err)
+			continue
+		}
+		break
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("RPCs did not fail after removal of Cluster resource")
+	}
+}
+
+// Test verifies that when the received Cluster resource contains outlier
+// detection configuration, the LB config pushed to the priority policy contains
+// the appropriate configuration for the outlier detection LB policy.
+func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) {
+	// Unregister the priority balancer builder for the duration of this test,
+	// and register a policy under the same name that makes the LB config
+	// pushed to it available to the test.
+	priorityBuilder := balancer.Get(priority.Name)
+	internal.BalancerUnregister(priorityBuilder.Name())
+	lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1)
+	stub.Register(priority.Name, stub.BalancerFuncs{
+		Init: func(bd *stub.BalancerData) {
+			bd.ChildBalancer = priorityBuilder.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return priorityBuilder.(balancer.ConfigParser).ParseConfig(lbCfg)
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			select {
+			case lbCfgCh <- ccs.BalancerConfig:
+			default:
+			}
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+	})
+	defer balancer.Register(priorityBuilder)
+
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure cluster and endpoints resources in the management server.
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)
+	cluster.OutlierDetection = &v3clusterpb.OutlierDetection{
+		Interval:                 durationpb.New(10 * time.Second),
+		BaseEjectionTime:         durationpb.New(30 * time.Second),
+		MaxEjectionTime:          durationpb.New(300 * time.Second),
+		MaxEjectionPercent:       wrapperspb.UInt32(10),
+		SuccessRateStdevFactor:   wrapperspb.UInt32(2000),
+		EnforcingSuccessRate:     wrapperspb.UInt32(50),
+		SuccessRateMinimumHosts:  wrapperspb.UInt32(10),
+		SuccessRateRequestVolume: wrapperspb.UInt32(50),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, serviceName, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{cluster},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create xDS client, xdsResolver, and dial the test backends.
+	_, cleanup := setupAndDial(t, bootstrapContents)
+	defer cleanup()
+
+	// The priority configuration generated should have Outlier Detection as a
+	// direct child due to Outlier Detection being turned on.
+	wantCfg := &priority.LBConfig{
+		Children: map[string]*priority.Child{
+			"priority-0-0": {
+				Config: &iserviceconfig.BalancerConfig{
+					Name: outlierdetection.Name,
+					Config: &outlierdetection.LBConfig{
+						Interval:           iserviceconfig.Duration(10 * time.Second), // default interval
+						BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+						MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+						MaxEjectionPercent: 10,
+						SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+							StdevFactor:           2000,
+							EnforcementPercentage: 50,
+							MinimumHosts:          10,
+							RequestVolume:         50,
+						},
+						ChildPolicy: &iserviceconfig.BalancerConfig{
+							Name: clusterimpl.Name,
+							Config: &clusterimpl.LBConfig{
+								Cluster: clusterName,
+								ChildPolicy: &iserviceconfig.BalancerConfig{
+									Name: wrrlocality.Name,
+									Config: &wrrlocality.LBConfig{
+										ChildPolicy: &iserviceconfig.BalancerConfig{
+											Name: roundrobin.Name,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+				IgnoreReresolutionRequests: true,
+			},
+		},
+		Priorities: []string{"priority-0-0"},
+	}
+
+	select {
+	case lbCfg := <-lbCfgCh:
+		gotCfg := lbCfg.(*priority.LBConfig)
+		if diff := cmp.Diff(wantCfg, gotCfg); diff != "" {
+			t.Fatalf("Child policy received unexpected diff in config (-want +got):\n%s", diff)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for child policy to receive its configuration")
+	}
+}

internal/xds/balancer/cdsbalancer/e2e_test/dns_impl_test.go

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/priority" // Register priority LB policy.
+)
+
+// TestLogicalDNS_MultipleEndpoints tests the priority LB policy using a
+// LOGICAL_DNS discovery mechanism.
+//
+// The test verifies that multiple addresses returned by the DNS resolver are
+// grouped into a single endpoint (as per gRFC A61). Because the round_robin LB
+// policy sees only one endpoint, it should not rotate traffic between the
+// addresses. Instead, the single endpoint is picked, and connects to the first
+// address.
+func (s) TestLogicalDNS_MultipleEndpoints(t *testing.T) {
+	// Spin up a management server to receive xDS resources from.
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+	resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start backend servers which provide an implementation of the TestService.
+	server1 := stubserver.StartTestService(t, nil)
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+
+	// Register a manual resolver with the "dns" scheme to override DNS resolution.
+	// This global override is safe because connection to the xDS management
+	// server uses the passthrough scheme instead and therefore overriding
+	// the DNS resolver does not affect it in any way.
+	const dnsScheme = "dns"
+	dnsR := manual.NewBuilderWithScheme(dnsScheme)
+	originalDNS := resolver.Get("dns")
+	resolver.Register(dnsR)
+	t.Cleanup(func() { resolver.Register(originalDNS) })
+
+	// For LOGICAL_DNS, this updates the SINGLE endpoint to have 2 IPs.
+	dnsR.InitialState(resolver.State{
+		Endpoints: []resolver.Endpoint{{
+			Addresses: []resolver.Address{
+				{Addr: server1.Address},
+				{Addr: server2.Address},
+			}}},
+	})
+
+	const (
+		serviceName   = "test-xds-service"
+		clusterName   = "cluster-test-xds-service"
+		endpointsName = "endpoints-test-xds-service"
+		rdsName       = "route-test-xds-service"
+	)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+			ClusterName: clusterName,
+			ServiceName: endpointsName,
+			Type:        e2e.ClusterTypeLogicalDNS,
+			DNSHostName: "dns",
+			DNSPort:     uint32(8080),
+			Policy:      e2e.LoadBalancingPolicyRoundRobin,
+		})},
+		Endpoints: nil,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///"+serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("Failed to create new client for local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Ensure the RPC is routed to the first backend.
+	testClient := testgrpc.NewTestServiceClient(cc)
+	for i := 0; i < 10; i++ {
+		var peer peer.Peer
+		if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
+			t.Fatalf("RPC failed: %v", err)
+		}
+
+		if got, want := peer.Addr.String(), server1.Address; got != want {
+			t.Errorf("peer.Addr = %q, want = %q", got, want)
+		}
+	}
+}

internal/xds/balancer/cdsbalancer/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cdsbalancer
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[cds-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *cdsBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clusterimpl/balancer_test.go

@@ -0,0 +1,356 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/testutils/fakeclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultShortTestTimeout = 100 * time.Microsecond
+
+	testClusterName = "test-cluster"
+	testServiceName = "test-eds-service"
+)
+
+var (
+	testBackendEndpoints = []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}}
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func init() {
+	NewRandomWRR = testutils.NewTestWRR
+}
+
+// TestPickerUpdateAfterClose covers the case where a child policy sends a
+// picker update after the cluster_impl policy is closed. Because picker updates
+// are handled in the run() goroutine, which exits before Close() returns, we
+// expect the above picker update to be dropped.
+func (s) TestPickerUpdateAfterClose(t *testing.T) {
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a subConn state
+	// change).
+	closeCh := make(chan struct{})
+	const childPolicyName = "stubBalancer-TestPickerUpdateAfterClose"
+	stub.Register(childPolicyName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			// Create a subConn which will be used later on to test the race
+			// between StateListener() and Close().
+			sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(balancer.SubConnState) {
+					go func() {
+						// Wait for Close() to be called on the parent policy before
+						// sending the picker update.
+						<-closeCh
+						bd.ClientConn.UpdateState(balancer.State{
+							Picker: base.NewErrPicker(errors.New("dummy error picker")),
+						})
+					}()
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+
+	var maxRequest uint32 = 50
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+						MaxRequests:    &maxRequest,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: childPolicyName,
+			},
+		},
+	}); err != nil {
+		b.Close()
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Send a subConn state change to trigger a picker update. The stub balancer
+	// that we use as the child policy will not send a picker update until the
+	// parent policy is closed.
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	b.Close()
+	close(closeCh)
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("unexpected picker update after balancer is closed")
+	case <-time.After(defaultShortTestTimeout):
+	}
+}
+
+// TestClusterNameInAddressAttributes covers the case that cluster name is
+// attached to the subconn address attributes.
+func (s) TestClusterNameInAddressAttributes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendEndpoints[0].Addresses[0].Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	cn, ok := xdsinternal.GetXDSHandshakeClusterName(addrs1[0].Attributes)
+	if !ok || cn != testClusterName {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName)
+	}
+
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	const testClusterName2 = "test-cluster-2"
+	var addr2 = resolver.Address{Addr: "2.2.2.2"}
+	state2 := xdsclient.SetClient(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{addr2}}}}, xdsC)
+	state2 = xdsresource.SetXDSConfig(state2, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName2: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName2,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state2,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName2,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, addr2.Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	// New addresses should have the new cluster name.
+	cn2, ok := xdsinternal.GetXDSHandshakeClusterName(addrs2[0].Attributes)
+	if !ok || cn2 != testClusterName2 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
+	}
+}
+
+// Test verify that the case picker is updated synchronously on receipt of
+// configuration update.
+func (s) TestPickerUpdatedSynchronouslyOnConfigUpdate(t *testing.T) {
+	// Override the pickerUpdateHook to be notified that picker was updated.
+	pickerUpdated := make(chan struct{}, 1)
+	origNewPickerUpdated := pickerUpdateHook
+	pickerUpdateHook = func() {
+		pickerUpdated <- struct{}{}
+	}
+	defer func() { pickerUpdateHook = origNewPickerUpdated }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Override the clientConnUpdateHook to ensure client conn was updated.
+	clientConnUpdateDone := make(chan struct{}, 1)
+	origClientConnUpdateHook := clientConnUpdateHook
+	clientConnUpdateHook = func() {
+		// Verify that picker was updated before the completion of
+		// client conn update.
+		select {
+		case <-pickerUpdated:
+		case <-ctx.Done():
+			t.Fatal("Client conn update completed before picker update.")
+		}
+		clientConnUpdateDone <- struct{}{}
+	}
+	defer func() { clientConnUpdateHook = origClientConnUpdateHook }()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a resolver
+	// update).
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{
+				Picker: base.NewErrPicker(errors.New("dummy error picker")),
+			})
+			return nil
+		},
+	})
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: t.Name(),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("Unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	select {
+	case <-clientConnUpdateDone:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for client conn update to be completed.")
+	}
+}
+
+func (s) TestDropRequestsPerMillion(t *testing.T) {
+	tests := []struct {
+		name        string
+		numerator   uint32
+		denominator uint32
+		want        uint32
+	}{
+		{name: "zero", numerator: 0, denominator: million, want: 0},
+		{name: "five percent hundred", numerator: 5, denominator: 100, want: 50000},
+		// numerator*million overflows a uint32 once numerator reaches 4295 with
+		// the million denominator, which is only a 0.43% drop.
+		{name: "point four three percent million", numerator: 4295, denominator: million, want: 4295},
+		{name: "one percent million", numerator: 10000, denominator: million, want: 10000},
+		{name: "fifty percent million", numerator: 500000, denominator: million, want: 500000},
+		{name: "hundred percent million", numerator: million, denominator: million, want: million},
+		// A fraction above 100% is capped at a million.
+		{name: "over hundred percent", numerator: 150, denominator: 100, want: million},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := dropRequestsPerMillion(tt.numerator, tt.denominator); got != tt.want {
+				t.Errorf("dropRequestsPerMillion(%d, %d) = %d, want %d", tt.numerator, tt.denominator, got, tt.want)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/clusterimpl.go

@@ -0,0 +1,644 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clusterimpl implements the xds_cluster_impl balancing policy. It
+// handles the cluster features (e.g. circuit_breaking, RPC dropping).
+//
+// Note that it doesn't handle name resolution, which is done by policy
+// xds_cluster_resolver.
+package clusterimpl
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/weightedroundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/balancer/loadstore"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	// Name is the name of the cluster_impl balancer.
+	Name                   = "xds_cluster_impl_experimental"
+	defaultRequestCountMax = 1024
+	loadStoreStopTimeout   = 1 * time.Second
+)
+
+var (
+	// Below function is no-op in actual code, but can be overridden in
+	// tests to give tests visibility into exactly when certain events happen.
+	clientConnUpdateHook = func() {}
+	pickerUpdateHook     = func() {}
+	buildProvider        = buildProviderFunc
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &clusterImplBalancer{
+		ClientConn:      cc,
+		loadWrapper:     loadstore.NewWrapper(),
+		requestCountMax: defaultRequestCountMax,
+	}
+	b.xdsHIPtr.Store(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+	b.logger = prefixLogger(b)
+	b.child = gracefulswitch.NewBalancer(b, bOpts)
+	b.logger.Infof("Created")
+
+	var creds credentials.TransportCredentials
+	switch {
+	case bOpts.DialCreds != nil:
+		creds = bOpts.DialCreds
+	case bOpts.CredsBundle != nil:
+		creds = bOpts.CredsBundle.TransportCredentials()
+	}
+	if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+		b.xdsCredsInUse = true
+	}
+	b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse)
+	return b
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type clusterImplBalancer struct {
+	balancer.ClientConn
+
+	// The following fields are set at creation time, and are read-only after
+	// that, and therefore need not be protected by a mutex.
+	logger *grpclog.PrefixLogger
+	// TODO: #8366 -  Refactor usage of loadWrapper to easily plugin a test
+	// load reporter from tests.
+	loadWrapper *loadstore.Wrapper
+
+	// The following fields are only accessed from balancer API methods, which
+	// are guaranteed to be called serially by gRPC.
+	xdsClient        xdsclient.XDSClient     // Sent down in ResolverState attributes.
+	cancelLoadReport func(context.Context)   // To stop reporting load through the above xDS client.
+	edsServiceName   string                  // EDS service name to report load for.
+	lrsServer        *bootstrap.ServerConfig // Load reporting server configuration.
+	dropCategories   []DropConfig            // The categories for drops.
+	child            *gracefulswitch.Balancer
+	xdsHIPtr         atomic.Pointer[grpcsync.RefCounted[*xds.HandshakeInfo]] // Accessed atomically as it is shared between the balancer and the transport.
+	xdsCredsInUse    bool
+
+	// The active security configuration received from the control plane.
+	securityConfig *xdsresource.SecurityConfig
+
+	// The following fields are protected by mu, since they are accessed in
+	// balancer API methods and in methods called from the child policy.
+	mu                       sync.Mutex
+	clusterName              string                                      // The cluster name for credentials handshaking.
+	inhibitPickerUpdates     bool                                        // Inhibits state updates from child policy when processing an update from the parent.
+	pendingPickerUpdates     bool                                        // True if a picker update from the child policy was inhibited when processing an update from the parent.
+	childState               balancer.State                              // Most recent state update from the child policy.
+	drops                    []*dropper                                  // Drops implementation.
+	requestCounterCluster    string                                      // The cluster name for the request counter, from LB config.
+	requestCounterService    string                                      // The service name for the request counter, from LB config.
+	requestCountMax          uint32                                      // Max concurrent requests, from LB config.
+	requestCounter           *xdsclient.ClusterRequestsCounter           // Tracks total inflight requests for a given service.
+	telemetryLabels          map[string]string                           // Telemetry labels to set on picks, from LB config.
+	lrsReportEndpointMetrics *xdsresource.LRSReportEndpointMetricsConfig // LRS metrics to propagate.
+}
+
+// dropRequestsPerMillion scales a drop overload's numerator and denominator to
+// a number of requests to drop per million. numerator can be as large as a
+// million (a 100% drop), so the multiplication is done in uint64 to avoid
+// overflowing a uint32, and the result is capped at a million because a drop
+// ratio above 100% is treated as 100%.
+func dropRequestsPerMillion(numerator, denominator uint32) uint32 {
+	rpm := uint64(numerator) * million / uint64(denominator)
+	if rpm > million {
+		rpm = million
+	}
+	return uint32(rpm)
+}
+
+// handleClusterConfigLocked updates the internal state of the balancer with the
+// new cluster configuration. It returns true if a new picker needs to be
+// generated as a result of these changes. It must be called with b.mu held.
+func (b *clusterImplBalancer) handleClusterConfigLocked(clusterConfig xdsresource.ClusterConfig) bool {
+	clusterUpdate := clusterConfig.Cluster
+	var updatePicker bool
+
+	b.telemetryLabels = clusterUpdate.TelemetryLabels
+
+	if !b.lrsReportEndpointMetrics.Equal(clusterUpdate.LRSReportEndpointMetrics) {
+		b.lrsReportEndpointMetrics = clusterUpdate.LRSReportEndpointMetrics
+		updatePicker = true
+	}
+
+	var newDrops []DropConfig
+	if clusterUpdate.ClusterType == xdsresource.ClusterTypeEDS {
+		edsUpdate := clusterConfig.EndpointConfig.EDSUpdate
+		newDrops = make([]DropConfig, 0, len(edsUpdate.Drops))
+		for _, d := range edsUpdate.Drops {
+			newDrops = append(newDrops, DropConfig{
+				Category:           d.Category,
+				RequestsPerMillion: dropRequestsPerMillion(d.Numerator, d.Denominator),
+			})
+		}
+		if !slices.Equal(b.dropCategories, newDrops) {
+			b.dropCategories = newDrops
+			b.drops = make([]*dropper, 0, len(newDrops))
+			for _, c := range newDrops {
+				b.drops = append(b.drops, newDropper(c))
+			}
+			updatePicker = true
+		}
+	}
+
+	if b.requestCounterCluster != clusterUpdate.ClusterName || b.requestCounterService != clusterUpdate.EDSServiceName {
+		b.requestCounterCluster = clusterUpdate.ClusterName
+		b.requestCounterService = clusterUpdate.EDSServiceName
+		b.requestCounter = xdsclient.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName)
+		updatePicker = true
+	}
+	var newRequestCountMax uint32 = 1024
+	if clusterUpdate.MaxRequests != nil {
+		newRequestCountMax = *clusterUpdate.MaxRequests
+	}
+	if b.requestCountMax != newRequestCountMax {
+		b.requestCountMax = newRequestCountMax
+		updatePicker = true
+	}
+
+	return updatePicker
+}
+
+func (b *clusterImplBalancer) newPickerLocked() *picker {
+	return &picker{
+		drops:           b.drops,
+		s:               b.childState,
+		loadStore:       b.loadWrapper,
+		counter:         b.requestCounter,
+		countMax:        b.requestCountMax,
+		telemetryLabels: b.telemetryLabels,
+		clusterName:     b.clusterName,
+		metrics:         b.lrsReportEndpointMetrics,
+	}
+}
+
+// updateLoadStore checks the config for load store, and decides whether it
+// needs to restart the load reporting stream.
+func (b *clusterImplBalancer) updateLoadStore(clusterUpdate *xdsresource.ClusterUpdate) error {
+	var updateLoadClusterAndService bool
+
+	// ClusterName is different, restart. ClusterName is from ClusterName and
+	// EDSServiceName.
+	clusterName := b.getClusterName()
+	if clusterName != clusterUpdate.ClusterName {
+		updateLoadClusterAndService = true
+		b.setClusterName(clusterUpdate.ClusterName)
+		clusterName = clusterUpdate.ClusterName
+	}
+	if b.edsServiceName != clusterUpdate.EDSServiceName {
+		updateLoadClusterAndService = true
+		b.edsServiceName = clusterUpdate.EDSServiceName
+	}
+	if updateLoadClusterAndService {
+		// This updates the clusterName and serviceName that will be reported
+		// for the loads. The update here is too early, the perfect timing is
+		// when the picker is updated with the new connection. But from this
+		// balancer's point of view, it's impossible to tell.
+		//
+		// On the other hand, this will almost never happen. Each LRS policy
+		// shouldn't get updated config. The parent should do a graceful switch
+		// when the clusterName or serviceName is changed.
+		b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName)
+	}
+
+	var (
+		stopOldLoadReport  bool
+		startNewLoadReport bool
+	)
+
+	// Check if it's necessary to restart load report.
+	if b.lrsServer == nil {
+		if clusterUpdate.LRSServerConfig != nil {
+			// Old is nil, new is not nil, start new LRS.
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			startNewLoadReport = true
+		}
+		// Old is nil, new is nil, do nothing.
+	} else if clusterUpdate.LRSServerConfig == nil {
+		// Old is not nil, new is nil, stop old, don't start new.
+		b.lrsServer = nil
+		stopOldLoadReport = true
+	} else {
+		// Old is not nil, new is not nil, compare string values, if
+		// different, stop old and start new.
+		if !b.lrsServer.Equal(clusterUpdate.LRSServerConfig) {
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			stopOldLoadReport = true
+			startNewLoadReport = true
+		}
+	}
+
+	if stopOldLoadReport {
+		if b.cancelLoadReport != nil {
+			stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+			defer stopCancel()
+			b.cancelLoadReport(stopCtx)
+			b.cancelLoadReport = nil
+			if !startNewLoadReport {
+				// If a new LRS stream will be started later, no need to update
+				// it to nil here.
+				b.loadWrapper.UpdateLoadStore(nil)
+			}
+		}
+	}
+	if startNewLoadReport {
+		var loadStore *lrsclient.LoadStore
+		if b.xdsClient != nil {
+			loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(b.lrsServer)
+		}
+		b.loadWrapper.UpdateLoadStore(loadStore)
+	}
+
+	return nil
+}
+
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg := configs[instanceName]
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
+
+func (b *clusterImplBalancer) buildProviders(config *xdsresource.SecurityConfig) (certprovider.Provider, certprovider.Provider, error) {
+	cpc := b.xdsClient.BootstrapConfig().CertProviderConfigs()
+	var rootProvider certprovider.Provider
+	if config.UseSystemRootCerts {
+		rootProvider = systemRootCertsProvider{}
+	} else {
+		rp, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		rootProvider = rp
+	}
+
+	var identityProvider certprovider.Provider
+	if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" {
+		ip, err := buildProvider(cpc, name, cert, true, false)
+		if err != nil {
+			if rootProvider != nil {
+				rootProvider.Close()
+			}
+			return nil, nil, err
+		}
+		identityProvider = ip
+	}
+	return rootProvider, identityProvider, nil
+}
+
+// handleSecurityConfig processes the security configuration received from the
+// management server, creates appropriate certificate provider plugins, and
+// updates the HandshakeInfo which is added as an address attribute in
+// NewSubConn() calls.
+func (b *clusterImplBalancer) handleSecurityConfig(config *xdsresource.SecurityConfig) error {
+	// If xdsCredentials are not in use, i.e, the user did not want to get
+	// security configuration from an xDS server, we should not be acting on the
+	// received security config here. Doing so poses a security threat.
+	if !b.xdsCredsInUse || config.Equal(b.securityConfig) {
+		return nil
+	}
+
+	// Security config being nil is a valid case where the management server has
+	// not sent any security configuration. The xdsCredentials implementation
+	// handles this by delegating to its fallback credentials.
+	if config == nil {
+		b.securityConfig = nil
+		oldHI := b.xdsHIPtr.Swap(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+		if oldHI != nil {
+			oldHI.Decrement()
+		}
+		return nil
+	}
+
+	rootProvider, identityProvider, err := b.buildProviders(config)
+	if err != nil {
+		return err
+	}
+
+	newHI := xds.NewHandshakeInfo(rootProvider, identityProvider, config.SubjectAltNameMatchers, config.SNI, false, config.AutoSNISANValidation, config.UseAutoHostSNI)
+	b.securityConfig = config
+	oldHI := b.xdsHIPtr.Swap(newHI)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	return nil
+}
+
+func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	defer clientConnUpdateHook()
+
+	b.mu.Lock()
+	b.inhibitPickerUpdates = true
+	b.mu.Unlock()
+	if b.logger.V(2) {
+		b.logger.Infof("Received configuration: %s", pretty.ToJSON(s.BalancerConfig))
+	}
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	// Need to check for potential errors at the beginning of this function, so
+	// that on errors, we reject the whole config, instead of applying part of
+	// it.
+	bb := balancer.Get(newConfig.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("child policy %q not registered", newConfig.ChildPolicy.Name)
+	}
+
+	if b.xdsClient == nil {
+		c := xdsclient.FromResolverState(s.ResolverState)
+		if c == nil {
+			return balancer.ErrBadResolverState
+		}
+		b.xdsClient = c
+	}
+
+	xdsConfig := xdsresource.XDSConfigFromResolverState(s.ResolverState)
+	if xdsConfig == nil {
+		b.logger.Warningf("Received balancer config with no xDS config")
+		return balancer.ErrBadResolverState
+	}
+	clusterCfg := xdsConfig.Clusters[newConfig.Cluster]
+	clusterUpdate := clusterCfg.Config.Cluster
+	if err := b.handleSecurityConfig(clusterUpdate.SecurityCfg); err != nil {
+		// If the security config is invalid, for example, if the provider
+		// instance is not found in the bootstrap config, we need to put the
+		// channel in transient failure.
+		return fmt.Errorf("received Cluster resource that contains invalid security config: %v", err)
+
+	}
+	// Update load reporting config. This needs to be done before updating the
+	// child policy because we need the loadStore from the updated client to be
+	// passed to the ccWrapper, so that the next picker from the child policy
+	// will pick up the new loadStore.
+	if err := b.updateLoadStore(clusterUpdate); err != nil {
+		return err
+	}
+
+	// Build config for the gracefulswitch balancer. It is safe to ignore JSON
+	// marshaling errors here, since the config was already validated as part of
+	// ParseConfig().
+	cfg := []map[string]any{{newConfig.ChildPolicy.Name: newConfig.ChildPolicy.Config}}
+	cfgJSON, _ := json.Marshal(cfg)
+	parsedCfg, err := gracefulswitch.ParseConfig(cfgJSON)
+	if err != nil {
+		return err
+	}
+
+	// Addresses and sub-balancer config are sent to sub-balancer.
+	err = b.child.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  weightedroundrobin.SetBackendService(s.ResolverState, b.clusterName),
+		BalancerConfig: parsedCfg,
+	})
+
+	b.mu.Lock()
+	updatePicker := b.handleClusterConfigLocked(clusterCfg.Config)
+	// We want to send a picker update to the parent if one of the two
+	// conditions are met:
+	// - drop/request count config or LRS metrics config has changed *and* there
+	//   is already a picker from the child, or
+	// - there is a pending picker update from the child (and this covers the
+	//   case where the drop/request config has not changed, but the child sent
+	//   a picker update while we were still processing config from our parent).
+	if (updatePicker && b.childState.Picker != nil) || b.pendingPickerUpdates {
+		b.pendingPickerUpdates = false
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker:            b.newPickerLocked(),
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.mu.Unlock()
+	pickerUpdateHook()
+	return err
+}
+
+func (b *clusterImplBalancer) ResolverError(err error) {
+	b.child.ResolverError(err)
+}
+
+func (b *clusterImplBalancer) updateSubConnState(_ balancer.SubConn, s balancer.SubConnState, cb func(balancer.SubConnState)) {
+	// Trigger re-resolution when a SubConn turns transient failure. This is
+	// necessary for the LogicalDNS in cluster_resolver policy to re-resolve.
+	//
+	// Note that this happens not only for the addresses from DNS, but also for
+	// EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent
+	// knows). The parent priority policy is configured to ignore re-resolution
+	// signal from the EDS children.
+	if s.ConnectivityState == connectivity.TransientFailure {
+		b.ClientConn.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
+	if cb != nil {
+		cb(s)
+	}
+}
+
+func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)
+}
+
+func (b *clusterImplBalancer) Close() {
+	b.child.Close()
+	b.childState = balancer.State{}
+
+	if b.cancelLoadReport != nil {
+		stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+		defer stopCancel()
+		b.cancelLoadReport(stopCtx)
+		b.cancelLoadReport = nil
+	}
+	oldHI := b.xdsHIPtr.Swap(nil)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	b.securityConfig = nil
+	b.logger.Infof("Shutdown")
+}
+
+func (b *clusterImplBalancer) ExitIdle() {
+	b.child.ExitIdle()
+}
+
+// Override methods to accept updates from the child LB.
+
+func (b *clusterImplBalancer) UpdateState(state balancer.State) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Inhibit sending a picker update to our parent as part of handling new
+	// state from the child, if we are currently handling an update from our
+	// parent. Update the childState field regardless.
+	b.childState = state
+	if b.inhibitPickerUpdates {
+		b.pendingPickerUpdates = true
+		if b.logger.V(2) {
+			b.logger.Infof("Received a picker update from the child when processing an update from the parent")
+		}
+		return
+	}
+
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: state.ConnectivityState,
+		Picker:            b.newPickerLocked(),
+	})
+	pickerUpdateHook()
+}
+
+func (b *clusterImplBalancer) setClusterName(n string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.clusterName = n
+}
+
+func (b *clusterImplBalancer) getClusterName() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.clusterName
+}
+
+// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be
+// retrieved from the addresses when creating SubConn.
+//
+// All SubConns passed to the child policies are wrapped in this, so that the
+// picker can get the localityID from the picked SubConn, and do load reporting.
+//
+// After wrapping, all SubConns to and from the parent ClientConn (e.g. for
+// SubConn state update, update/remove SubConn) must be the original SubConns.
+// All SubConns to and from the child policy (NewSubConn, forwarding SubConn
+// state update) must be the wrapper. The balancer keeps a map from the original
+// SubConn to the wrapper for this purpose.
+type scWrapper struct {
+	balancer.SubConn
+
+	localityID clients.Locality
+
+	hostname string
+}
+
+func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	clusterName := b.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetXDSHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = xds.SetHandshakeInfo(newAddrs[i], &b.xdsHIPtr)
+
+		host := xdsresource.Hostname(addr)
+		// If the hostname contains a port, strip it. Per [RFC 6066, Section
+		// 3](https://www.rfc-editor.org/rfc/rfc6066.html#section-3), the SNI
+		// may only contain a qualified DNS hostname, which excludes port
+		// numbers.
+		h, _, err := net.SplitHostPort(host)
+		if err == nil {
+			host = h
+		}
+		// Store hostname in the address attributes, so that it can be used in
+		// the client handshake.
+		newAddrs[i] = xds.SetAddressHostname(newAddrs[i], host)
+	}
+	var sc balancer.SubConn
+	scw := &scWrapper{}
+	if len(addrs) > 0 {
+		scw.hostname = xdsresource.Hostname(addrs[0])
+		scw.localityID = xdsinternal.GetLocalityID(addrs[0])
+	}
+	oldListener := opts.StateListener
+	opts.StateListener = func(state balancer.SubConnState) {
+		b.updateSubConnState(sc, state, oldListener)
+	}
+	sc, err := b.ClientConn.NewSubConn(newAddrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	return scw, nil
+}
+
+func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// systemRootCertsProvider implements a certprovider.Provider that returns the
+// system default root certificates for validation.
+type systemRootCertsProvider struct{}
+
+func (systemRootCertsProvider) Close() {}
+
+func (systemRootCertsProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) {
+	rootCAs, err := internal.X509SystemCertPoolFunc()
+	if err != nil {
+		return nil, err
+	}
+	return &certprovider.KeyMaterial{Roots: rootCAs}, nil
+}

internal/xds/balancer/clusterimpl/config.go

@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// DropConfig contains the category, and drop ratio.
+type DropConfig struct {
+	Category           string
+	RequestsPerMillion uint32
+}
+
+// LBConfig is the balancer config for cluster_impl balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	Cluster     string                                `json:"cluster,omitempty"`
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}

internal/xds/balancer/clusterimpl/config_test.go

@@ -0,0 +1,132 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/roundrobin"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+const (
+	testJSONConfig = `{
+  "cluster": "test_cluster",
+  "edsServiceName": "test-eds",
+  "lrsLoadReportingServer": {
+    "server_uri": "trafficdirector.googleapis.com:443",
+    "channel_creds": [ { "type": "google_default" } ]
+  },
+  "maxConcurrentRequests": 123,
+  "dropCategories": [
+    {
+      "category": "drop-1",
+      "requestsPerMillion": 314
+    },
+    {
+      "category": "drop-2",
+      "requestsPerMillion": 159
+    }
+  ],
+  "childPolicy": [
+    {
+      "weighted_target_experimental": {
+        "targets": {
+          "wt-child-1": {
+            "weight": 75,
+            "childPolicy":[{"round_robin":{}}]
+          },
+          "wt-child-2": {
+            "weight": 25,
+            "childPolicy":[{"round_robin":{}}]
+          }
+        }
+      }
+    }
+  ]
+}`
+
+	wtName = "weighted_target_experimental"
+)
+
+var (
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON   = `{
+  "targets": {
+    "wt-child-1": {
+      "weight": 75,
+      "childPolicy":[{"round_robin":{}}]
+    },
+    "wt-child-2": {
+      "weight": 25,
+      "childPolicy":[{"round_robin":{}}]
+    }
+  }
+}`
+
+	wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON))
+)
+
+func (s) TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *LBConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "bad json",
+			js:      "{",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &LBConfig{
+				Cluster: "test_cluster",
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name:   wtName,
+					Config: wtConfig,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Errorf("parseConfig() got unexpected diff (-want, +got): %v", diff)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/internal/internal.go

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package internal contains code internal to the clusterimpl package.
+package internal
+
+import "crypto/x509"
+
+// X509SystemCertPoolFunc is used for overriding the system cert pool for
+// tests.
+var X509SystemCertPoolFunc = x509.SystemCertPool

internal/xds/balancer/clusterimpl/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-cluster-impl-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clusterimpl/picker.go

@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/wrr"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+// NewRandomWRR is used when calculating drops. It's exported so that tests can
+// override it.
+var NewRandomWRR = wrr.NewRandom
+
+const million = 1000000
+
+type dropper struct {
+	category string
+	w        wrr.WRR
+}
+
+// greatest common divisor (GCD) via Euclidean algorithm
+func gcd(a, b uint32) uint32 {
+	for b != 0 {
+		t := b
+		b = a % b
+		a = t
+	}
+	return a
+}
+
+func newDropper(c DropConfig) *dropper {
+	w := NewRandomWRR()
+	gcdv := gcd(c.RequestsPerMillion, million)
+	// Return true for RequestPerMillion, false for the rest.
+	w.Add(true, int64(c.RequestsPerMillion/gcdv))
+	w.Add(false, int64((million-c.RequestsPerMillion)/gcdv))
+
+	return &dropper{
+		category: c.Category,
+		w:        w,
+	}
+}
+
+func (d *dropper) drop() (ret bool) {
+	return d.w.Next().(bool)
+}
+
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallStarted(locality clients.Locality)
+	CallFinished(locality clients.Locality, err error)
+	CallServerLoad(locality clients.Locality, name string, val float64)
+	CallDropped(category string)
+}
+
+// Picker implements RPC drop, circuit breaking drop and load reporting.
+type picker struct {
+	drops           []*dropper
+	s               balancer.State
+	loadStore       loadReporter
+	counter         *xdsclient.ClusterRequestsCounter
+	countMax        uint32
+	telemetryLabels map[string]string
+	clusterName     string
+	metrics         *xdsresource.LRSReportEndpointMetricsConfig
+}
+
+func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// Unconditionally update labels if present, even dropped or queued RPC's can
+	// use these labels.
+	stats.UpdateLabels(info.Ctx, d.telemetryLabels)
+
+	// Don't drop unless the inner picker is READY. Similar to
+	// https://github.com/grpc/grpc-go/issues/2622.
+	if d.s.ConnectivityState == connectivity.Ready {
+		// Check if this RPC should be dropped by category.
+		for _, dp := range d.drops {
+			if dp.drop() {
+				if d.loadStore != nil {
+					d.loadStore.CallDropped(dp.category)
+				}
+				return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
+			}
+		}
+	}
+
+	// Check if this RPC should be dropped by circuit breaking.
+	if d.counter != nil {
+		if err := d.counter.StartRequest(d.countMax); err != nil {
+			// Drops by circuit breaking are reported with empty category. They
+			// will be reported only in total drops, but not in per category.
+			if d.loadStore != nil {
+				d.loadStore.CallDropped("")
+			}
+			return balancer.PickResult{}, status.Error(codes.Unavailable, err.Error())
+		}
+	}
+
+	var lID clients.Locality
+	pr, err := d.s.Picker.Pick(info)
+	if scw, ok := pr.SubConn.(*scWrapper); ok {
+		// This OK check also covers the case err!=nil, because SubConn will be
+		// nil.
+		pr.SubConn = scw.SubConn
+		// If locality ID isn't found in the wrapper, an empty locality ID will
+		// be used.
+		lID = scw.localityID
+
+		if scw.hostname != "" && autoHostRewriteEnabled(info.Ctx) {
+			if pr.Metadata == nil {
+				pr.Metadata = metadata.Pairs(":authority", scw.hostname)
+			} else {
+				pr.Metadata.Set(":authority", scw.hostname)
+			}
+		}
+	}
+
+	if err != nil {
+		if d.counter != nil {
+			// Release one request count if this pick fails.
+			d.counter.EndRequest()
+		}
+		return pr, err
+	}
+
+	stats.UpdateLabels(
+		info.Ctx,
+		map[string]string{
+			"grpc.lb.locality":        xdsinternal.LocalityString(lID),
+			"grpc.lb.backend_service": d.clusterName,
+		},
+	)
+
+	if d.loadStore != nil {
+		locality := clients.Locality{Region: lID.Region, Zone: lID.Zone, SubZone: lID.SubZone}
+		d.loadStore.CallStarted(locality)
+		oldDone := pr.Done
+		pr.Done = func(info balancer.DoneInfo) {
+			if oldDone != nil {
+				oldDone(info)
+			}
+			d.loadStore.CallFinished(locality, info.Err)
+
+			load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport)
+			if !ok || load == nil {
+				return
+			}
+
+			if envconfig.XDSORCAToLRSPropEnabled {
+				if d.metrics != nil {
+					if d.metrics.CPUUtilization {
+						d.loadStore.CallServerLoad(locality, "cpu_utilization", load.CpuUtilization)
+					}
+					if d.metrics.MemUtilization {
+						d.loadStore.CallServerLoad(locality, "mem_utilization", load.MemUtilization)
+					}
+					if d.metrics.ApplicationUtilization {
+						d.loadStore.CallServerLoad(locality, "application_utilization", load.ApplicationUtilization)
+					}
+					for n, c := range load.NamedMetrics {
+						_, ok := d.metrics.NamedMetrics[n]
+						if d.metrics.NamedMetricsAll || ok {
+							d.loadStore.CallServerLoad(locality, "named_metrics."+n, c)
+						}
+					}
+				}
+			} else {
+				for n, c := range load.NamedMetrics {
+					d.loadStore.CallServerLoad(locality, n, c)
+				}
+			}
+		}
+	}
+
+	if d.counter != nil {
+		// Update Done() so that when the RPC finishes, the request count will
+		// be released.
+		oldDone := pr.Done
+		pr.Done = func(doneInfo balancer.DoneInfo) {
+			d.counter.EndRequest()
+			if oldDone != nil {
+				oldDone(doneInfo)
+			}
+		}
+	}
+
+	return pr, err
+}
+
+// autoHostRewriteKey is the context key used to store the value of
+// route's autoHostRewrite in the RPC context.
+type autoHostRewriteKey struct{}
+
+// autoHostRewriteEnabled retrieves the autoHostRewrite value from the provided context.
+func autoHostRewriteEnabled(ctx context.Context) bool {
+	v, _ := ctx.Value(autoHostRewriteKey{}).(bool)
+	return v
+}
+
+// AutoHostRewriteEnabledForTesting returns the value of autoHostRewrite field;
+// to be used for testing only.
+func AutoHostRewriteEnabledForTesting(ctx context.Context) bool {
+	return autoHostRewriteEnabled(ctx)
+}
+
+// EnableAutoHostRewrite adds the autoHostRewrite value to the context for
+// the xds_cluster_impl LB policy to pick.
+func EnableAutoHostRewrite(ctx context.Context) context.Context {
+	return context.WithValue(ctx, autoHostRewriteKey{}, true)
+}

internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go

@@ -0,0 +1,783 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package clusterimpl_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	iclusterimpl "google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/testdata"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin.
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router"       // Register the router filter.
+	_ "google.golang.org/grpc/internal/xds/resolver"                // Register the xds resolver
+)
+
+// Common setup for security tests:
+//   - creates an xDS client with the specified bootstrap configuration
+//   - creates a gRPC channel that uses the passed in client creds
+//   - creates a test server that uses the passed in server creds
+//
+// Returns the following:
+// - a client channel to make RPCs
+// - address of the test backend server
+func setupForSecurityTests(t *testing.T, bootstrapContents []byte, clientCreds, serverCreds credentials.TransportCredentials) (*grpc.ClientConn, string) {
+	t.Helper()
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Create a ClientConn with the specified transport credentials.
+	cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	t.Cleanup(func() { cc.Close() })
+
+	// Start a test service backend with the specified transport credentials.
+	sOpts := []grpc.ServerOption{}
+	if serverCreds != nil {
+		sOpts = append(sOpts, grpc.Creds(serverCreds))
+	}
+	server := stubserver.StartTestService(t, nil, sOpts...)
+	t.Cleanup(server.Stop)
+
+	return cc, server.Address
+}
+
+// Creates transport credentials to be used on the client side that rely on xDS
+// to provide the security configuration. It falls back to insecure creds if no
+// security information is received from the management server.
+func xdsClientCredsWithInsecureFallback(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	xdsCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create xDS credentials: %v", err)
+	}
+	return xdsCreds
+}
+
+// Creates transport credentials to be used on the server side from certificate
+// files in testdata/x509.
+//
+// The certificate returned by this function has a CommonName of "test-server1".
+func tlsServerCreds(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("Failed to load server cert and key: %v", err)
+
+	}
+	pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("Failed to read client CA cert: %v", err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    roots,
+	}
+	return credentials.NewTLS(cfg)
+}
+
+// Checks the AuthInfo available in the peer if it matches the expected security
+// level of the connection.
+func verifySecurityInformationFromPeer(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel) {
+	// This is not a true helper in the Go sense, because it does not perform
+	// setup or cleanup tasks. Marking it a helper is to ensure that when the
+	// test fails, the line information of the caller is outputted instead of
+	// from here.
+	//
+	// And this function directly calls t.Fatalf() instead of returning an error
+	// and letting the caller decide what to do with it. This is also OK since
+	// all callers will simply end up calling t.Fatalf() with the returned
+	// error, and can't add any contextual information of value to the error
+	// message.
+	t.Helper()
+
+	switch wantSecLevel {
+	case e2e.SecurityLevelNone:
+		if pr.AuthInfo.AuthType() != "insecure" {
+			t.Fatalf("AuthType() is %s, want insecure", pr.AuthInfo.AuthType())
+		}
+	case e2e.SecurityLevelMTLS:
+		ai, ok := pr.AuthInfo.(credentials.TLSInfo)
+		if !ok {
+			t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{})
+		}
+		if len(ai.State.PeerCertificates) != 1 {
+			t.Fatalf("Number of peer certificates is %d, want 1", len(ai.State.PeerCertificates))
+		}
+		cert := ai.State.PeerCertificates[0]
+		const wantCommonName = "test-server1"
+		if cn := cert.Subject.CommonName; cn != wantCommonName {
+			t.Fatalf("Common name in peer certificate is %s, want %s", cn, wantCommonName)
+		}
+	}
+}
+
+// makeAggregateClusterResource returns an aggregate cluster resource with the
+// given name and list of child names.
+func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeAggregate,
+		ChildNames:  childNames,
+	})
+}
+
+// Tests the case where xDS credentials are not in use, but the cds LB policy
+// receives a Cluster update with security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with insecure creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, insecure.NewCredentials(), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where xDS credentials are in use, but the cds LB policy
+// receives a Cluster update without security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is not configured to return any security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the security config returned by the management server
+// cannot be resolved based on the contents of the bootstrap config. Verifies
+// that the cds LB policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server,
+	// and one that does not have certificate providers configuration.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       8080,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// A certificate provider builder that returns a nil Provider from the starter
+// func passed to certprovider.NewBuildableConfig().
+type errCertProviderBuilder struct{}
+
+const errCertProviderName = "err-cert-provider"
+
+func (e errCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) {
+	// Returning a nil Provider simulates the case where an error is encountered
+	// at the time of building the Provider.
+	bc := certprovider.NewBuildableConfig(errCertProviderName, nil, func(certprovider.BuildOptions) certprovider.Provider { return nil })
+	return bc, nil
+}
+
+func (e errCertProviderBuilder) Name() string {
+	return errCertProviderName
+}
+
+func init() {
+	certprovider.Register(errCertProviderBuilder{})
+}
+
+// Tests the case where the certprovider.Store returns an error when the cds LB
+// policy attempts to build a certificate provider. Verifies that the cds LB
+// policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestCertproviderStoreError(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration for
+	// errCertProviderBuilder.
+	nodeID := uuid.New().String()
+	providerCfg := json.RawMessage(fmt.Sprintf(`{
+		"plugin_name": "%s",
+		"config": {}
+	}`, errCertProviderName))
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+		CertificateProviders:               map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure.
+func (s) TestGoodSecurityConfig(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that contains a certificate provider instance
+// that is missing in the bootstrap file. Verifies that the channel moves to
+// TRANSIENT_FAILURE. Subsequently, the cds LB policy receives a cluster
+// resource that contains a certificate provider that is present in the
+// bootstrap file.  Verifies that the connection between the client and the
+// server is secure.
+func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
+	var (
+		target      = "test.service"
+		routeName   = "route1"
+		clusterName = "cluster1"
+		serviceName = "service1"
+	)
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)
+	cluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with initial resources: %v", err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Update the management server with a Cluster resource that contains a
+	// certificate provider instance that is present in the bootstrap
+	// configuration.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with valid resources: %v", err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource without security configuration. Verifies that this results in the
+// use of fallback credentials, which in this case is insecure creds.
+func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
+	target := "test.service"
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// Start a test service backend that does not expect a secure connection.
+	insecureServer := stubserver.StartTestService(t, nil)
+	t.Cleanup(insecureServer.Stop)
+
+	// Update the resources in the management server to contain no security
+	// configuration. This should result in the use of fallback credentials,
+	// which is insecure in our case.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, insecureServer.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to move to the new backend that expects
+	// connections without security.
+	for ctx.Err() == nil {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+			t.Logf("EmptyCall() failed: %v", err)
+		}
+		if peer.Addr.String() == insecureServer.Address {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timed out when waiting for connection to switch to second backend")
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource that is NACKed by the xDS client. Test verifies that the cds LB
+// policy continues to use the previous good configuration.
+func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	resources.Clusters[0] = e2e.DefaultCluster(resources.Clusters[0].Name, resources.Endpoints[0].ClusterName, e2e.SecurityLevelNone)
+	resources.Clusters[0].TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that specifies the use system root certs.
+// Verifies that the connection between the client and the server is secure.
+func (s) TestSystemRootCertsSecurityConfig(t *testing.T) {
+	origFlag := envconfig.XDSSystemRootCertsEnabled
+	origSRCF := iclusterimpl.X509SystemCertPoolFunc
+	defer func() {
+		envconfig.XDSSystemRootCertsEnabled = origFlag
+		iclusterimpl.X509SystemCertPoolFunc = origSRCF
+	}()
+	envconfig.XDSSystemRootCertsEnabled = true
+
+	systemRootCertsFuncCalled := false
+	iclusterimpl.X509SystemCertPoolFunc = func() (*x509.CertPool, error) {
+		certData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+		if err != nil {
+			return nil, fmt.Errorf("failed to read certificate file: %w", err)
+		}
+		certPool := x509.NewCertPool()
+
+		if ok := certPool.AppendCertsFromPEM(certData); !ok {
+			return nil, fmt.Errorf("failed to append certificate to cert pool")
+		}
+		systemRootCertsFuncCalled = true
+		return certPool, nil
+	}
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The cluster
+	// resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelTLSWithSystemRootCerts,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	if systemRootCertsFuncCalled != true {
+		t.Errorf("System root certs were not used during the test.")
+	}
+}
+
+// TestAggregateClusterSecurityConfig tests the case where the CDS LB policy
+// receives a top-level aggregate cluster pointing to a child EDS cluster. The
+// test verifies that the CDS LB policy correctly ignores the aggregate
+// cluster's security configuration and uses the child EDS cluster's security
+// configuration to establish the mTLS connection.
+func (s) TestAggregateClusterSecurityConfig(t *testing.T) {
+	var (
+		target         = "test.service"
+		routeName      = "route1"
+		clusterName    = "cluster1"
+		serviceName    = "service1"
+		edsClusterName = clusterName + "-eds"
+	)
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configures an aggregate cluster with a security config that is not NACKed
+	// by the xdsclient but would fail the handshake as it has an invalid SAN
+	// matcher.
+	aggCluster := makeAggregateClusterResource(clusterName, []string{edsClusterName})
+	aggCluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "bad-cn"}},
+								},
+							},
+							ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+						},
+					},
+				},
+			}),
+		},
+	}
+
+	// Create the child EDS cluster with a valid mTLS security configuration.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			aggCluster,
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelMTLS),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection since
+	// the EDS cluster has a valid mTLS configuration even though the top level
+	// cluster has an incorrect security configuration.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}

internal/xds/balancer/clustermanager/balancerstateaggregator.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/grpclog"
+)
+
+type subBalancerState struct {
+	state balancer.State
+	// stateToAggregate is the connectivity state used only for state
+	// aggregation. It could be different from state.ConnectivityState. For
+	// example when a sub-balancer transitions from TransientFailure to
+	// connecting, state.ConnectivityState is Connecting, but stateToAggregate
+	// is still TransientFailure.
+	stateToAggregate connectivity.State
+}
+
+func (s *subBalancerState) String() string {
+	return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate)
+}
+
+type balancerStateAggregator struct {
+	cc     balancer.ClientConn
+	logger *grpclog.PrefixLogger
+	csEval *balancer.ConnectivityStateEvaluator
+
+	mu sync.Mutex
+	// This field is used to ensure that no updates are forwarded to the parent
+	// CC once the aggregator is closed. A closed sub-balancer could still send
+	// pickers to this aggregator.
+	closed bool
+	// Map from child policy name to last reported state.
+	idToPickerState map[string]*subBalancerState
+	// Set when UpdateState call propagation is paused.
+	pauseUpdateState bool
+	// Set when UpdateState call propagation is paused and an UpdateState call
+	// is suppressed.
+	needUpdateStateOnResume bool
+}
+
+func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator {
+	return &balancerStateAggregator{
+		cc:              cc,
+		logger:          logger,
+		csEval:          &balancer.ConnectivityStateEvaluator{},
+		idToPickerState: make(map[string]*subBalancerState),
+	}
+}
+
+func (bsa *balancerStateAggregator) close() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.closed = true
+}
+
+// add adds a sub-balancer in CONNECTING state.
+//
+// This is called when there's a new child.
+func (bsa *balancerStateAggregator) add(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+
+	bsa.idToPickerState[id] = &subBalancerState{
+		// Start everything in CONNECTING, so if one of the sub-balancers
+		// reports TransientFailure, the RPCs will still wait for the other
+		// sub-balancers.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+		stateToAggregate: connectivity.Connecting,
+	}
+	bsa.csEval.RecordTransition(connectivity.Shutdown, connectivity.Connecting)
+	bsa.buildAndUpdateLocked()
+}
+
+// remove removes the sub-balancer state. Future updates from this sub-balancer,
+// if any, will be ignored.
+//
+// This is called when a child is removed.
+func (bsa *balancerStateAggregator) remove(id string) {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	if _, ok := bsa.idToPickerState[id]; !ok {
+		return
+	}
+	// Setting the state of the deleted sub-balancer to Shutdown will get
+	// csEvltr to remove the previous state for any aggregated state
+	// evaluations. Transitions to and from connectivity.Shutdown are ignored
+	// by csEvltr.
+	bsa.csEval.RecordTransition(bsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown)
+	// Remove id and picker from picker map. This also results in future updates
+	// for this ID to be ignored.
+	delete(bsa.idToPickerState, id)
+	bsa.buildAndUpdateLocked()
+}
+
+// pauseStateUpdates causes UpdateState calls to not propagate to the parent
+// ClientConn.  The last state will be remembered and propagated when
+// ResumeStateUpdates is called.
+func (bsa *balancerStateAggregator) pauseStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = true
+	bsa.needUpdateStateOnResume = false
+}
+
+// resumeStateUpdates will resume propagating UpdateState calls to the parent,
+// and call UpdateState on the parent if any UpdateState call was suppressed.
+func (bsa *balancerStateAggregator) resumeStateUpdates() {
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	bsa.pauseUpdateState = false
+	if bsa.needUpdateStateOnResume {
+		bsa.cc.UpdateState(bsa.buildLocked())
+	}
+}
+
+// UpdateState is called to report a balancer state change from sub-balancer.
+// It's usually called by the balancer group.
+//
+// It calls parent ClientConn's UpdateState with the new aggregated state.
+func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) {
+	bsa.logger.Infof("State update from sub-balancer %q: %+v", id, state)
+
+	bsa.mu.Lock()
+	defer bsa.mu.Unlock()
+	pickerSt, ok := bsa.idToPickerState[id]
+	if !ok {
+		// All state starts with an entry in pickStateMap. If ID is not in map,
+		// it's either removed, or never existed.
+		return
+	}
+	if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) {
+		// If old state is TransientFailure, and new state is Connecting, don't
+		// update the state, to prevent the aggregated state from being always
+		// CONNECTING. Otherwise, stateToAggregate is the same as
+		// state.ConnectivityState.
+		bsa.csEval.RecordTransition(pickerSt.stateToAggregate, state.ConnectivityState)
+		pickerSt.stateToAggregate = state.ConnectivityState
+	}
+	pickerSt.state = state
+	bsa.buildAndUpdateLocked()
+}
+
+// buildAndUpdateLocked combines the sub-state from each sub-balancer into one
+// state, and sends a picker update to the parent ClientConn.
+func (bsa *balancerStateAggregator) buildAndUpdateLocked() {
+	if bsa.closed {
+		return
+	}
+	if bsa.pauseUpdateState {
+		// If updates are paused, do not call UpdateState, but remember that we
+		// need to call it when they are resumed.
+		bsa.needUpdateStateOnResume = true
+		return
+	}
+	bsa.cc.UpdateState(bsa.buildLocked())
+}
+
+// buildLocked combines sub-states into one.
+func (bsa *balancerStateAggregator) buildLocked() balancer.State {
+	// The picker's return error might not be consistent with the
+	// aggregatedState. Because for this LB policy, we want to always build
+	// picker with all sub-pickers (not only ready sub-pickers), so even if the
+	// overall state is Ready, pick for certain RPCs can behave like Connecting
+	// or TransientFailure.
+	bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState)
+	return balancer.State{
+		ConnectivityState: bsa.csEval.CurrentState(),
+		Picker:            newPickerGroup(bsa.idToPickerState),
+	}
+}

internal/xds/balancer/clustermanager/clustermanager.go

@@ -0,0 +1,205 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clustermanager implements the cluster manager LB policy for xds.
+package clustermanager
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/balancergroup"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const balancerName = "xds_cluster_manager_experimental"
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	b := &bal{}
+	b.logger = prefixLogger(b)
+	b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
+	b.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               opts,
+		StateAggregator:         b.stateAggregator,
+		Logger:                  b.logger,
+		SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies
+	})
+	b.logger.Infof("Created")
+	return b
+}
+
+func (bb) Name() string {
+	return balancerName
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type bal struct {
+	logger          *internalgrpclog.PrefixLogger
+	bg              *balancergroup.BalancerGroup
+	stateAggregator *balancerStateAggregator
+
+	children map[string]childConfig
+}
+
+func (b *bal) setErrorPickerForChild(childName string, err error) {
+	b.stateAggregator.UpdateState(childName, balancer.State{
+		ConnectivityState: connectivity.TransientFailure,
+		Picker:            base.NewErrPicker(err),
+	})
+}
+
+func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) error {
+	endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints)
+
+	// Remove sub-balancers that are not in the new list from the aggregator and
+	// balancergroup.
+	for name := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			b.stateAggregator.remove(name)
+			b.bg.Remove(name)
+		}
+	}
+
+	var retErr error
+	for childName, childCfg := range newConfig.Children {
+		lbCfg := childCfg.ChildPolicy.Config
+		if _, ok := b.children[childName]; !ok {
+			// Add new sub-balancers to the aggregator and balancergroup.
+			b.stateAggregator.add(childName)
+			b.bg.Add(childName, balancer.Get(childCfg.ChildPolicy.Name))
+		} else {
+			// If the child policy type has changed for existing sub-balancers,
+			// parse the new config and send down the config update to the
+			// balancergroup, which will take care of gracefully switching the
+			// child over to the new policy.
+			//
+			// If we run into errors here, we need to ensure that RPCs to this
+			// child fail, while RPCs to other children with good configs
+			// continue to succeed.
+			newPolicyName, oldPolicyName := childCfg.ChildPolicy.Name, b.children[childName].ChildPolicy.Name
+			if newPolicyName != oldPolicyName {
+				var err error
+				var cfgJSON []byte
+				cfgJSON, err = childCfg.ChildPolicy.MarshalJSON()
+				if err != nil {
+					retErr = fmt.Errorf("failed to JSON marshal load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+				// This overwrites lbCfg to be in the format expected by the
+				// gracefulswitch balancer. So, when this config is pushed to
+				// the child (below), it will result in a graceful switch to the
+				// new child policy.
+				lbCfg, err = balancergroup.ParseConfig(cfgJSON)
+				if err != nil {
+					retErr = fmt.Errorf("failed to parse load balancing policy for child %q: %v", childName, err)
+					b.setErrorPickerForChild(childName, retErr)
+					continue
+				}
+			}
+		}
+
+		if err := b.bg.UpdateClientConnState(childName, balancer.ClientConnState{
+			ResolverState: resolver.State{
+				Endpoints:     endpointsSplit[childName],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			},
+			BalancerConfig: lbCfg,
+		}); err != nil {
+			retErr = fmt.Errorf("failed to push new configuration %v to child %q: %v", childCfg.ChildPolicy.Config, childName, err)
+			b.setErrorPickerForChild(childName, retErr)
+		}
+
+		// Picker update is sent to the parent ClientConn only after the
+		// new child policy returns a picker. So, there is no need to
+		// set needUpdateStateOnResume to true here.
+	}
+
+	b.children = newConfig.Children
+
+	// If multiple sub-balancers run into errors, we will return only the last
+	// one, which is still good enough, since the grpc channel will anyways
+	// return this error as balancer.ErrBadResolver to the name resolver,
+	// resulting in re-resolution attempts.
+	return retErr
+
+	// Adding or removing a sub-balancer will result in the
+	// needUpdateStateOnResume bit to true which results in a picker update once
+	// resumeStateUpdates() is called.
+}
+
+func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error {
+	if b.logger.V(2) {
+		b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
+	}
+
+	newConfig, ok := s.BalancerConfig.(*lbConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	b.stateAggregator.pauseStateUpdates()
+	defer b.stateAggregator.resumeStateUpdates()
+	return b.updateChildren(s, newConfig)
+}
+
+func (b *bal) ResolverError(err error) {
+	b.bg.ResolverError(err)
+}
+
+func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *bal) Close() {
+	b.stateAggregator.close()
+	b.bg.Close()
+	b.logger.Infof("Shutdown")
+}
+
+func (b *bal) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+const prefix = "[xds-cluster-manager-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *bal) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clustermanager/clustermanager_test.go

@@ -0,0 +1,775 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+	testBackendAddrsCount   = 12
+)
+
+var testBackendAddrStrs []string
+
+func init() {
+	for i := 0; i < testBackendAddrsCount; i++ {
+		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
+	}
+}
+
+func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) {
+	t.Helper()
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p.Pick(info)
+		if fmt.Sprint(err) != fmt.Sprint(wantErr) {
+			t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr)
+		}
+		if gotSCSt.SubConn != wantSC {
+			t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC)
+		}
+	}
+}
+
+func (s) TestClusterPicks(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	p1 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestConfigUpdateAddCluster covers the cases the balancer receives config
+// update with extra clusters.
+func (s) TestConfigUpdateAddCluster(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with different routes, and different actions. Expect a
+	// new subconn and a picker update.
+	configJSON2 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_3":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], BalancerAttributes: nil})
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[2]}}, []string{"cds:cluster_3"}),
+		}},
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect exactly one new subconn.
+	addrs := <-cc.NewSubConnAddrsCh
+	sc := <-cc.NewSubConnCh
+	// Clear the attributes before adding to map.
+	addrs[0].BalancerAttributes = nil
+	m1[addrs[0]] = sc
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	// Should have no more newSubConn.
+	select {
+	case <-time.After(time.Millisecond * 500):
+	case <-cc.NewSubConnCh:
+		addrs := <-cc.NewSubConnAddrsCh
+		t.Fatalf("unexpected NewSubConn with address %v", addrs)
+	}
+
+	p2 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_3"),
+			},
+			wantSC: m1[wantAddrs[2]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives
+// config update with no clusters. Pick should fail with details in error.
+func (s) TestRoutingConfigUpdateDeleteAll(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"round_robin":""}] },
+	"cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+		{Addr: testBackendAddrStrs[1], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m1 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m1[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p1 := <-cc.NewPickerCh
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m1[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m1[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+
+	// A config update with no clusters.
+	configJSON2 := `{}`
+	config2, err := parser.ParseConfig([]byte(configJSON2))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: config2,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Expect two removed subconns.
+	for range wantAddrs {
+		select {
+		case <-time.After(time.Millisecond * 500):
+			t.Fatalf("timeout waiting for remove subconn")
+		case <-cc.ShutdownSubConnCh:
+		}
+	}
+
+	p2 := <-cc.NewPickerCh
+	for i := 0; i < 5; i++ {
+		gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(ctx, "cds:notacluster")})
+		if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() {
+			t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`)
+		}
+	}
+
+	// Resend the previous config with clusters
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[1]}}, []string{"cds:cluster_2"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	m2 := make(map[resolver.Address]balancer.SubConn)
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		addrs := <-cc.NewSubConnAddrsCh
+		sc := <-cc.NewSubConnCh
+		// Clear the attributes before adding to map.
+		addrs[0].BalancerAttributes = nil
+		m2[addrs[0]] = sc
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	}
+
+	p3 := <-cc.NewPickerCh
+	for _, tt := range []struct {
+		pickInfo balancer.PickInfo
+		wantSC   balancer.SubConn
+		wantErr  error
+	}{
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_1"),
+			},
+			wantSC: m2[wantAddrs[0]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:cluster_2"),
+			},
+			wantSC: m2[wantAddrs[1]],
+		},
+		{
+			pickInfo: balancer.PickInfo{
+				Ctx: SetPickedCluster(ctx, "cds:notacluster"),
+			},
+			wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`),
+		},
+	} {
+		testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
+	}
+}
+
+func (s) TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) {
+	const (
+		userAgent          = "ua"
+		defaultTestTimeout = 1 * time.Second
+	)
+
+	// Setup the stub balancer such that we can read the build options passed to
+	// it in the UpdateClientConnState method.
+	ccsCh := testutils.NewChannel()
+	bOpts := balancer.BuildOptions{
+		DialCreds:       insecure.NewCredentials(),
+		CustomUserAgent: userAgent,
+	}
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			if !cmp.Equal(bd.BuildOptions, bOpts) {
+				err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
+				ccsCh.Send(err)
+				return err
+			}
+			ccsCh.Send(nil)
+			return nil
+		},
+	})
+
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, bOpts)
+
+	configJSON1 := fmt.Sprintf(`{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"%s":""}] }
+}
+}`, t.Name())
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	v, err := ccsCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timed out waiting for UpdateClientConnState result: %v", err)
+	}
+	if v != nil {
+		t.Fatal(v)
+	}
+}
+
+const initIdleBalancerName = "test-init-idle-balancer"
+
+var errTestInitIdle = fmt.Errorf("init Idle balancer error 0")
+
+func init() {
+	stub.Register(initIdleBalancerName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
+			sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(state balancer.SubConnState) {
+					err := fmt.Errorf("wrong picker error")
+					if state.ConnectivityState == connectivity.Idle {
+						err = errTestInitIdle
+					}
+					bd.ClientConn.UpdateState(balancer.State{
+						ConnectivityState: state.ConnectivityState,
+						Picker:            &testutils.TestConstPicker{Err: err},
+					})
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+}
+
+// TestInitialIdle covers the case that if the child reports Idle, the overall
+// state will be Idle.
+func (s) TestInitialIdle(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"test-init-idle-balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that a subconn is created with the address, and the hierarchy path
+	// in the address is cleared.
+	for range wantAddrs {
+		sc := <-cc.NewSubConnCh
+		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
+	}
+
+	if state1 := <-cc.NewStateCh; state1 != connectivity.Idle {
+		t.Fatalf("Received aggregated state: %v, want Idle", state1)
+	}
+}
+
+// TestClusterGracefulSwitch tests the graceful switch functionality for a child
+// of the cluster_manager. The test performs the following steps:
+//  1. Configures a single child for the cluster_manager with a round_robin LB
+//     policy. Ensures the child sends a picker update when the subchannel moves
+//     to READY.
+//  2. Updates the exiting child with a different LB policy (pick_first), and a
+//     different endpoint. This should trigger the graceful switch process, and
+//     cluster_manager should not send a picker update until the child has
+//     completed the graceful switch process.
+//  3. The child should complete the graceful switch process once it receives an
+//     update that moves its subchannel to READY. At this point, the cluster_manager
+//     should send a picker update reflecting the new pick_first balancer's picker.
+func (s) TestClusterGracefulSwitch(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	// Push round_robin config with one endpoint to the cluster_manager.
+	rrConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"round_robin": {}}] }
+		}
+	}`
+	rrConfig, err := parser.ParseConfig([]byte(rrConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse round_robin config: %v", err)
+	}
+	rrEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "rr-backend"}}}, []string{"csp:cluster"})
+	ccs := balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{rrEndpoint}},
+		BalancerConfig: rrConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send round_robin config update to cluster_manager: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Wait for a subchannel to be created.
+	var sc1 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the round_robin LB policy")
+	case sc1 = <-cc.NewSubConnCh:
+	}
+
+	// Update the subchannel's state to READY, and ensure we get a picker update.
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	var p1 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p1 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	pi := balancer.PickInfo{Ctx: SetPickedCluster(ctx, "csp:cluster")}
+	testPick(t, p1, pi, sc1, nil)
+
+	// Update the config to use pick_first, and a different endpoint.
+	pfConfigJSON := `
+	{
+		"children": {
+			"csp:cluster":{ "childPolicy": [{"pick_first": {}}] }
+		}
+	}`
+	pfConfig, err := parser.ParseConfig([]byte(pfConfigJSON))
+	if err != nil {
+		t.Fatalf("Failed to parse pick_first config: %v", err)
+	}
+	pfEndpoint := hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{{Addr: "pf-backend"}}}, []string{"csp:cluster"})
+	ccs = balancer.ClientConnState{
+		ResolverState:  resolver.State{Endpoints: []resolver.Endpoint{pfEndpoint}},
+		BalancerConfig: pfConfig,
+	}
+	if err := bal.UpdateClientConnState(ccs); err != nil {
+		t.Fatalf("Failed to send pick_first config update to cluster_manager: %v", err)
+	}
+
+	// Wait for a subchannel to be created.
+	var sc2 *testutils.TestSubConn
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannel to be created by the pick_first LB policy")
+	case sc2 = <-cc.NewSubConnCh:
+	}
+
+	// Move the pick_first LB policy's subchannel to CONNECTING. This will cause
+	// the pick_first LB policy to send a connecting picker. But this should not
+	// result in the cluster_manager sending a picker update to the channel as
+	// the graceful switch process is not yet complete.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("Received picker from cluster_manager when none expected during graceful switch process")
+	case <-sCtx.Done():
+	}
+
+	// Move the pick_first LB policy's subchannel to READY. This will cause the
+	// pick_first LB policy to send a ready picker and complete the graceful
+	// switch process.
+	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+
+	var p2 balancer.Picker
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for picker update from cluster_manager LB policy")
+	case p2 = <-cc.NewPickerCh:
+	}
+
+	// Verify the picker picks the expected subchannel.
+	testPick(t, p2, pi, sc2, nil)
+
+	// Completion of the graceful switch process should result in the old
+	// subchannel being removed.
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for subchannels belonging to old child to be deleted")
+	case rsc := <-cc.ShutdownSubConnCh:
+		if rsc != sc1 {
+			t.Fatalf("Unexpected subchannel %v being shutdown, want %v", rsc, sc1)
+		}
+	}
+}
+
+// tcc wraps a testutils.TestClientConn but stores all state transitions in a
+// slice.
+type tcc struct {
+	*testutils.BalancerClientConn
+	states []balancer.State
+}
+
+func (t *tcc) UpdateState(bs balancer.State) {
+	t.states = append(t.states, bs)
+	t.BalancerClientConn.UpdateState(bs)
+}
+
+func (s) TestUpdateStatePauses(t *testing.T) {
+	cc := &tcc{BalancerClientConn: testutils.NewBalancerClientConn(t)}
+
+	balFuncs := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil})
+			bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil})
+			return nil
+		},
+	}
+	stub.Register("update_state_balancer", balFuncs)
+
+	builder := balancer.Get(balancerName)
+	parser := builder.(balancer.ConfigParser)
+	bal := builder.Build(cc, balancer.BuildOptions{})
+	defer bal.Close()
+
+	configJSON1 := `{
+"children": {
+	"cds:cluster_1":{ "childPolicy": [{"update_state_balancer":""}] }
+}
+}`
+	config1, err := parser.ParseConfig([]byte(configJSON1))
+	if err != nil {
+		t.Fatalf("failed to parse balancer config: %v", err)
+	}
+
+	// Send the config, and an address with hierarchy path ["cluster_1"].
+	wantAddrs := []resolver.Address{
+		{Addr: testBackendAddrStrs[0], BalancerAttributes: nil},
+	}
+	if err := bal.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: resolver.State{Endpoints: []resolver.Endpoint{
+			hierarchy.SetInEndpoint(resolver.Endpoint{Addresses: []resolver.Address{wantAddrs[0]}}, []string{"cds:cluster_1"}),
+		}},
+		BalancerConfig: config1,
+	}); err != nil {
+		t.Fatalf("failed to update ClientConn state: %v", err)
+	}
+
+	// Verify that the only state update is the second one called by the child.
+	if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready {
+		t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states)
+	}
+}

internal/xds/balancer/clustermanager/config.go

@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type childConfig struct {
+	// ChildPolicy is the child policy and it's config.
+	ChildPolicy *internalserviceconfig.BalancerConfig
+}
+
+// lbConfig is the balancer config for xds routing policy.
+type lbConfig struct {
+	serviceconfig.LoadBalancingConfig
+	Children map[string]childConfig
+}
+
+func parseConfig(c json.RawMessage) (*lbConfig, error) {
+	cfg := &lbConfig{}
+	if err := json.Unmarshal(c, cfg); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}

internal/xds/balancer/clustermanager/config_test.go

@@ -0,0 +1,144 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	_ "google.golang.org/grpc/internal/xds/balancer/cdsbalancer"
+)
+
+const (
+	testJSONConfig = `{
+      "children":{
+        "cds:cluster_1":{
+          "childPolicy":[{
+            "cds_experimental":{"cluster":"cluster_1"}
+          }]
+        },
+        "weighted:cluster_1_cluster_2_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1" : {
+                  "weight":75,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_2" : {
+                  "weight":25,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}]
+                }
+              }
+            }
+          }]
+        },
+        "weighted:cluster_1_cluster_3_1":{
+          "childPolicy":[{
+            "weighted_target_experimental":{
+              "targets": {
+                "cluster_1": {
+                  "weight":99,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}]
+                },
+                "cluster_3": {
+                  "weight":1,
+                  "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}]
+                }
+              }
+            }
+          }]
+        }
+      }
+}
+`
+
+	cdsName = "cds_experimental"
+	wtName  = "weighted_target_experimental"
+)
+
+var (
+	cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser)
+	cdsConfigJSON1  = `{"cluster":"cluster_1"}`
+	cdsConfig1, _   = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1))
+
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON1  = `{
+	"targets": {
+	  "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+	  "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] }
+	} }`
+	wtConfig1, _  = wtConfigParser.ParseConfig([]byte(wtConfigJSON1))
+	wtConfigJSON2 = `{
+    "targets": {
+      "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] },
+      "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] }
+    } }`
+	wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2))
+)
+
+func (s) Test_parseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *lbConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &lbConfig{
+				Children: map[string]childConfig{
+					"cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: cdsName, Config: cdsConfig1},
+					},
+					"weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig1},
+					},
+					"weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{
+						Name: wtName, Config: wtConfig2},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" {
+				t.Errorf("parseConfig() got unexpected result, diff: %v", d)
+			}
+		})
+	}
+}

internal/xds/balancer/clustermanager/e2e_test/clustermanager_test.go

@@ -0,0 +1,342 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3"
+	"github.com/google/uuid"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/xds" // Register the xDS name resolver and related LB policies.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+func makeEmptyCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("EmptyCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func makeUnaryCallRPCAndVerifyPeer(ctx context.Context, client testgrpc.TestServiceClient, wantPeer string) error {
+	peer := &peer.Peer{}
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		return fmt.Errorf("UnaryCall() failed: %v", err)
+	}
+	if gotPeer := peer.Addr.String(); gotPeer != wantPeer {
+		return fmt.Errorf("EmptyCall() routed to %q, want to be routed to: %q", gotPeer, wantPeer)
+	}
+	return nil
+}
+
+func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	testutils.CreateBootstrapFileForTesting(t, bc)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	var err error
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Configure client side xDS resources on the management server.
+	const (
+		serviceName     = "my-service-client-side-xds"
+		routeConfigName = "route-" + serviceName
+		clusterName1    = "cluster1-" + serviceName
+		clusterName2    = "cluster2-" + serviceName
+		endpointsName1  = "endpoints1-" + serviceName
+		endpointsName2  = "endpoints2-" + serviceName
+		endpointsName3  = "endpoints3-" + serviceName
+	)
+	// A single Listener resource pointing to the following Route
+	// configuration:
+	//   - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	//   - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+			},
+		}},
+	}}
+	// Two cluster resources corresponding to the ones mentioned in the above
+	// route configuration resource. These are configured with round_robin as
+	// their endpoint picking policy.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelNone),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+	}
+	// Spin up two test backends, one for each cluster below.
+	server1 := stubserver.StartTestService(t, nil)
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	// Two endpoints resources, each with one backend from above.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2.
+	if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a wrapped pickfirst LB policy. When the endpoint picking policy on
+	// the cluster resource is changed to pickfirst, this will allow us to
+	// verify that load balancing configuration is pushed to it.
+	pfBuilder := balancer.Get(pickfirst.Name)
+	internal.BalancerUnregister(pfBuilder.Name())
+
+	lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1)
+	stub.Register(pfBuilder.Name(), stub.BalancerFuncs{
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return pfBuilder.(balancer.ConfigParser).ParseConfig(lbCfg)
+		},
+		Init: func(bd *stub.BalancerData) {
+			bd.ChildBalancer = pfBuilder.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			select {
+			case lbCfgCh <- ccs.BalancerConfig:
+			default:
+			}
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+	})
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to pickfirst. The endpoints resource is also changed here
+	// to ensure that we can verify that the new child policy
+	cluster2 := &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						TypedConfig: testutils.MarshalAny(t, &v3pickfirstpb.PickFirst{
+							ShuffleAddressList: true,
+						}),
+					},
+				},
+			},
+		},
+	}
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+	endpoints3 := e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)})
+	resources.Clusters = append(resources.Clusters, cluster2)
+	resources.Endpoints = append(resources.Endpoints, endpoints3)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for configuration to be pushed to the new pickfirst child policy")
+	case <-lbCfgCh:
+	}
+
+	// Ensure RPCs are still succeeding.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it is routed to cluster2, and the
+	// new endpoints resource.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		if err := makeUnaryCallRPCAndVerifyPeer(ctx, client, server3.Address); err == nil {
+			break
+		}
+		t.Log(err)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to be routed to the new endpoints resource")
+	}
+
+	// Send a config update that changes the child policy configuration for one
+	// of the clusters to an unsupported LB policy. This should result in
+	// failure of RPCs to that cluster.
+	cluster2 = &v3clusterpb.Cluster{
+		Name:                 clusterName2,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: endpointsName3,
+		},
+		LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						// The type not registered in gRPC Policy registry.
+						TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+							TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist",
+							Value:   &structpb.Struct{},
+						}),
+					},
+				},
+			},
+		},
+	}
+	resources.Clusters[1] = cluster2
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// At this point, RPCs to cluster1 should continue to succeed, while RPCs to
+	// cluster2 should start to fail.
+
+	// Make an EmptyCall RPC and verify that it is routed to cluster1.
+	if err := makeEmptyCallRPCAndVerifyPeer(ctx, client, server1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make a UnaryCall RPC and verify that it starts to fail.
+	for ; ctx.Err() != nil; <-time.After(defaultTestShortTimeout) {
+		_, err := client.UnaryCall(ctx, &testpb.SimpleRequest{})
+		got := status.Code(err)
+		if got == codes.Unavailable {
+			break
+		}
+		t.Logf("UnaryCall() returned code: %v, want %v", got, codes.Unavailable)
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timeout when waiting for RPCs to cluster2 to start failing")
+	}
+
+	// Channel should still be READY.
+	if got, want := cc.GetState(), connectivity.Ready; got != want {
+		t.Fatalf("grpc.ClientConn in state %v, want %v", got, want)
+	}
+}

internal/xds/balancer/clustermanager/picker.go

@@ -0,0 +1,65 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clustermanager
+
+import (
+	"context"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// pickerGroup contains a list of pickers. If the picker isn't ready, the pick
+// will be queued.
+type pickerGroup struct {
+	pickers map[string]balancer.Picker
+}
+
+func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup {
+	pickers := make(map[string]balancer.Picker)
+	for id, st := range idToPickerState {
+		pickers[id] = st.state.Picker
+	}
+	return &pickerGroup{
+		pickers: pickers,
+	}
+}
+
+func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	cluster := PickedCluster(info.Ctx)
+	if p := pg.pickers[cluster]; p != nil {
+		return p.Pick(info)
+	}
+	return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster)
+}
+
+type clusterKey struct{}
+
+// PickedCluster returns the cluster name stored in the context.
+func PickedCluster(ctx context.Context) string {
+	cluster, _ := ctx.Value(clusterKey{}).(string)
+	return cluster
+}
+
+// SetPickedCluster adds the selected cluster to the context for the
+// xds_cluster_manager LB policy to pick.
+func SetPickedCluster(ctx context.Context, cluster string) context.Context {
+	return context.WithValue(ctx, clusterKey{}, cluster)
+}

internal/xds/balancer/loadstore/load_store_wrapper.go

@@ -0,0 +1,128 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package loadstore contains the loadStoreWrapper shared by the balancers.
+package loadstore
+
+import (
+	"sync"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+)
+
+// NewWrapper creates a Wrapper.
+func NewWrapper() *Wrapper {
+	return &Wrapper{}
+}
+
+// Wrapper wraps a load store with cluster and edsService.
+//
+// It's store and cluster/edsService can be updated separately. And it will
+// update its internal perCluster store so that new stats will be added to the
+// correct perCluster.
+//
+// Note that this struct is a temporary workaround before we implement graceful
+// switch for EDS. Any update to the clusterName and serviceName is too early,
+// the perfect timing is when the picker is updated with the new connection.
+// This early update could cause picks for the old SubConn being reported to the
+// new services.
+//
+// When the graceful switch in EDS is done, there should be no need for this
+// struct. The policies that record/report load shouldn't need to handle update
+// of lrsServerName/cluster/edsService. Its parent should do a graceful switch
+// of the whole tree when one of that changes.
+type Wrapper struct {
+	mu         sync.RWMutex
+	cluster    string
+	edsService string
+	// store and perCluster are initialized as nil. They are only set by the
+	// balancer when LRS is enabled. Before that, all functions to record loads
+	// are no-op.
+	store      *lrsclient.LoadStore
+	perCluster *lrsclient.PerClusterReporter
+}
+
+// UpdateClusterAndService updates the cluster name and eds service for this
+// wrapper. If any one of them is changed from before, the perCluster store in
+// this wrapper will also be updated.
+func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if cluster == lsw.cluster && edsService == lsw.edsService {
+		return
+	}
+	lsw.cluster = cluster
+	lsw.edsService = edsService
+	if lsw.store == nil {
+		return
+	}
+	lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService)
+}
+
+// UpdateLoadStore updates the load store for this wrapper. If it is changed
+// from before, the perCluster store in this wrapper will also be updated.
+func (lsw *Wrapper) UpdateLoadStore(store *lrsclient.LoadStore) {
+	lsw.mu.Lock()
+	defer lsw.mu.Unlock()
+	if store == lsw.store {
+		return
+	}
+	lsw.store = store
+	if lsw.store == nil {
+		lsw.perCluster = nil
+		return
+	}
+	lsw.perCluster = lsw.store.ReporterForCluster(lsw.cluster, lsw.edsService)
+}
+
+// CallStarted records a call started in the store.
+func (lsw *Wrapper) CallStarted(locality clients.Locality) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallStarted(locality)
+	}
+}
+
+// CallFinished records a call finished in the store.
+func (lsw *Wrapper) CallFinished(locality clients.Locality, err error) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallFinished(locality, err)
+	}
+}
+
+// CallServerLoad records the server load in the store.
+func (lsw *Wrapper) CallServerLoad(locality clients.Locality, name string, val float64) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallServerLoad(locality, name, val)
+	}
+}
+
+// CallDropped records a call dropped in the store.
+func (lsw *Wrapper) CallDropped(category string) {
+	lsw.mu.RLock()
+	defer lsw.mu.RUnlock()
+	if lsw.perCluster != nil {
+		lsw.perCluster.CallDropped(category)
+	}
+}

internal/xds/balancer/outlierdetection/balancer.go

@@ -0,0 +1,927 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package outlierdetection provides an implementation of the outlier detection
+// LB policy, as defined in
+// https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md.
+package outlierdetection
+
+import (
+	"encoding/json"
+	"fmt"
+	"math"
+	rand "math/rand/v2"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Globals to stub out in tests.
+var (
+	afterFunc = time.AfterFunc
+	now       = time.Now
+)
+
+// Name is the name of the outlier detection balancer.
+const Name = "outlier_detection_experimental"
+
+var (
+	ejectionsEnforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_enforced",
+		Description: "EXPERIMENTAL. Number of outlier ejections enforced by detection method",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method"},
+		Default:     false,
+	})
+
+	ejectionsUnenforcedMetric = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:        "grpc.lb.outlier_detection.ejections_unenforced",
+		Description: "EXPERIMENTAL. Number of unenforced outlier ejections due to either `max_ejection_percentage` or `enforcement_percentage`",
+		Unit:        "{ejection}",
+		Labels:      []string{"grpc.target", "grpc.lb.outlier_detection.detection_method", "grpc.lb.outlier_detection.unenforced_reason"},
+		Default:     false,
+	})
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &outlierDetectionBalancer{
+		ClientConn:      cc,
+		closed:          grpcsync.NewEvent(),
+		done:            grpcsync.NewEvent(),
+		addrs:           make(map[string]*endpointInfo),
+		scUpdateCh:      buffer.NewUnbounded[any](),
+		pickerUpdateCh:  buffer.NewUnbounded[any](),
+		channelzParent:  bOpts.ChannelzParent,
+		endpoints:       resolver.NewEndpointMap[*endpointInfo](),
+		metricsRecorder: cc.MetricsRecorder(), // we use an explicit field instead of using cc.MetricsRecorder() so we can override the metric recorder in tests.
+		target:          bOpts.Target.String(),
+	}
+	b.logger = prefixLogger(b)
+	b.logger.Infof("Created")
+	b.child = synchronizingBalancerWrapper{lb: gracefulswitch.NewBalancer(b, bOpts)}
+	go b.run()
+	return b
+}
+
+func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	lbCfg := &LBConfig{
+		// Default top layer values as documented in A50.
+		Interval:           iserviceconfig.Duration(10 * time.Second),
+		BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+		MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+		MaxEjectionPercent: 10,
+	}
+
+	// This unmarshalling handles underlying layers sre and fpe which have their
+	// own defaults for their fields if either sre or fpe are present.
+	if err := json.Unmarshal(s, lbCfg); err != nil { // Validates child config if present as well.
+		return nil, fmt.Errorf("xds: unable to unmarshal LBconfig: %s, error: %v", string(s), err)
+	}
+
+	// Note: in the xds flow, these validations will never fail. The xdsclient
+	// performs the same validations as here on the xds Outlier Detection
+	// resource before parsing resource into JSON which this function gets
+	// called with. A50 defines two separate places for these validations to
+	// take place, the xdsclient and this ParseConfig method. "When parsing a
+	// config from JSON, if any of these requirements is violated, that should
+	// be treated as a parsing error." - A50
+	switch {
+	// "The google.protobuf.Duration fields interval, base_ejection_time, and
+	// max_ejection_time must obey the restrictions in the
+	// google.protobuf.Duration documentation and they must have non-negative
+	// values." - A50
+	// Approximately 290 years is the maximum time that time.Duration (int64)
+	// can represent. The restrictions on the protobuf.Duration field are to be
+	// within +-10000 years. Thus, just check for negative values.
+	case lbCfg.Interval < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.interval = %s; must be >= 0", lbCfg.Interval)
+	case lbCfg.BaseEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.base_ejection_time = %s; must be >= 0", lbCfg.BaseEjectionTime)
+	case lbCfg.MaxEjectionTime < 0:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_time = %s; must be >= 0", lbCfg.MaxEjectionTime)
+
+	// "The fields max_ejection_percent,
+	// success_rate_ejection.enforcement_percentage,
+	// failure_percentage_ejection.threshold, and
+	// failure_percentage.enforcement_percentage must have values less than or
+	// equal to 100." - A50
+	case lbCfg.MaxEjectionPercent > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_percent = %v; must be <= 100", lbCfg.MaxEjectionPercent)
+	case lbCfg.SuccessRateEjection != nil && lbCfg.SuccessRateEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = %v; must be <= 100", lbCfg.SuccessRateEjection.EnforcementPercentage)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.Threshold > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = %v; must be <= 100", lbCfg.FailurePercentageEjection.Threshold)
+	case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.EnforcementPercentage > 100:
+		return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = %v; must be <= 100", lbCfg.FailurePercentageEjection.EnforcementPercentage)
+	}
+	return lbCfg, nil
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+// scUpdate wraps a subConn update to be sent to the child balancer.
+type scUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type ejectionUpdate struct {
+	scw       *subConnWrapper
+	isEjected bool // true for ejected, false for unejected
+}
+
+type lbCfgUpdate struct {
+	lbCfg *LBConfig
+	// to make sure picker is updated synchronously.
+	done chan struct{}
+}
+
+type scHealthUpdate struct {
+	scw   *subConnWrapper
+	state balancer.SubConnState
+}
+
+type outlierDetectionBalancer struct {
+	balancer.ClientConn
+	// These fields are safe to be accessed without holding any mutex because
+	// they are synchronized in run(), which makes these field accesses happen
+	// serially.
+	//
+	// childState is the latest balancer state received from the child.
+	childState balancer.State
+	// recentPickerNoop represents whether the most recent picker sent upward to
+	// the balancer.ClientConn is a noop picker, which doesn't count RPC's. Used
+	// to suppress redundant picker updates.
+	recentPickerNoop bool
+
+	closed          *grpcsync.Event
+	done            *grpcsync.Event
+	logger          *grpclog.PrefixLogger
+	channelzParent  channelz.Identifier
+	metricsRecorder estats.MetricsRecorder
+	target          string
+
+	child synchronizingBalancerWrapper
+
+	// mu guards access to the following fields. It also helps to synchronize
+	// behaviors of the following events: config updates, firing of the interval
+	// timer, SubConn State updates, SubConn address updates, and child state
+	// updates.
+	//
+	// For example, when we receive a config update in the middle of the
+	// interval timer algorithm, which uses knobs present in the config, the
+	// balancer will wait for the interval timer algorithm to finish before
+	// persisting the new configuration.
+	//
+	// Another example would be the updating of the endpoints or addrs map, such
+	// as from a SubConn address update in the middle of the interval timer
+	// algorithm which uses endpoints. This balancer waits for the interval
+	// timer algorithm to finish before making the update to the endpoints map.
+	//
+	// This mutex is never held when calling methods on the child policy
+	// (within the context of a single goroutine).
+	mu sync.Mutex
+	// endpoints stores pointers to endpointInfo objects for each endpoint.
+	endpoints *resolver.EndpointMap[*endpointInfo]
+	// addrs stores pointers to endpointInfo objects for each address. Addresses
+	// belonging to the same endpoint point to the same object.
+	addrs                 map[string]*endpointInfo
+	cfg                   *LBConfig
+	timerStartTime        time.Time
+	intervalTimer         *time.Timer
+	inhibitPickerUpdates  bool
+	updateUnconditionally bool
+	numEndpointsEjected   int // For fast calculations of percentage of endpoints ejected
+
+	scUpdateCh     *buffer.Unbounded[any]
+	pickerUpdateCh *buffer.Unbounded[any]
+}
+
+// noopConfig returns whether this balancer is configured with a logical no-op
+// configuration or not.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) noopConfig() bool {
+	return b.cfg.SuccessRateEjection == nil && b.cfg.FailurePercentageEjection == nil
+}
+
+// onIntervalConfig handles logic required specifically on the receipt of a
+// configuration which specifies to count RPC's and periodically perform passive
+// health checking based on heuristics defined in configuration every configured
+// interval.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onIntervalConfig() {
+	var interval time.Duration
+	if b.timerStartTime.IsZero() {
+		b.timerStartTime = time.Now()
+		for _, epInfo := range b.endpoints.All() {
+			epInfo.callCounter.clear()
+		}
+		interval = time.Duration(b.cfg.Interval)
+	} else {
+		interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime)
+		if interval < 0 {
+			interval = 0
+		}
+	}
+	b.intervalTimer = afterFunc(interval, b.intervalTimerAlgorithm)
+}
+
+// onNoopConfig handles logic required specifically on the receipt of a
+// configuration which specifies the balancer to be a noop.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) onNoopConfig() {
+	// "If a config is provided with both the `success_rate_ejection` and
+	// `failure_percentage_ejection` fields unset, skip starting the timer and
+	// do the following:"
+	// "Unset the timer start timestamp."
+	b.timerStartTime = time.Time{}
+	for _, epInfo := range b.endpoints.All() {
+		// "Uneject all currently ejected endpoints."
+		if !epInfo.latestEjectionTimestamp.IsZero() {
+			b.unejectEndpoint(epInfo)
+		}
+		// "Reset each endpoint's ejection time multiplier to 0."
+		epInfo.ejectionTimeMultiplier = 0
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	lbCfg, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		b.logger.Errorf("received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+
+	// Reject whole config if child policy doesn't exist, don't persist it for
+	// later.
+	bb := balancer.Get(lbCfg.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("outlier detection: child balancer %q not registered", lbCfg.ChildPolicy.Name)
+	}
+
+	// It is safe to read b.cfg here without holding the mutex, as the only
+	// write to b.cfg happens later in this function. This function is part of
+	// the balancer.Balancer API, so it is guaranteed to be called in a
+	// synchronous manner, so it cannot race with this read.
+	if b.cfg == nil || b.cfg.ChildPolicy.Name != lbCfg.ChildPolicy.Name {
+		if err := b.child.switchTo(bb); err != nil {
+			return fmt.Errorf("outlier detection: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err)
+		}
+	}
+
+	b.mu.Lock()
+	// Inhibit child picker updates until this UpdateClientConnState() call
+	// completes. If needed, a picker update containing the no-op config bit
+	// determined from this config and most recent state from the child will be
+	// sent synchronously upward at the end of this UpdateClientConnState()
+	// call.
+	b.inhibitPickerUpdates = true
+	b.updateUnconditionally = false
+	b.cfg = lbCfg
+
+	newEndpoints := resolver.NewEndpointMap[bool]()
+	for _, ep := range s.ResolverState.Endpoints {
+		newEndpoints.Set(ep, true)
+		if _, ok := b.endpoints.Get(ep); !ok {
+			b.endpoints.Set(ep, newEndpointInfo())
+		}
+	}
+
+	for ep := range b.endpoints.All() {
+		if _, ok := newEndpoints.Get(ep); !ok {
+			b.endpoints.Delete(ep)
+		}
+	}
+
+	// populate the addrs map.
+	b.addrs = map[string]*endpointInfo{}
+	for _, ep := range s.ResolverState.Endpoints {
+		epInfo, _ := b.endpoints.Get(ep)
+		for _, addr := range ep.Addresses {
+			if _, ok := b.addrs[addr.Addr]; ok {
+				b.logger.Errorf("Endpoints contain duplicate address %q", addr.Addr)
+				continue
+			}
+			b.addrs[addr.Addr] = epInfo
+		}
+	}
+
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+
+	if b.noopConfig() {
+		b.onNoopConfig()
+	} else {
+		b.onIntervalConfig()
+	}
+	b.mu.Unlock()
+
+	err := b.child.updateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: b.cfg.ChildPolicy.Config,
+	})
+
+	done := make(chan struct{})
+	b.pickerUpdateCh.Put(lbCfgUpdate{
+		lbCfg: lbCfg,
+		done:  done,
+	})
+	<-done
+
+	return err
+}
+
+func (b *outlierDetectionBalancer) ResolverError(err error) {
+	b.child.resolverError(err)
+}
+
+func (b *outlierDetectionBalancer) updateSubConnState(scw *subConnWrapper, state balancer.SubConnState) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	scw.setLatestConnectivityState(state.ConnectivityState)
+	b.scUpdateCh.Put(&scUpdate{
+		scw:   scw,
+		state: state,
+	})
+}
+
+func (b *outlierDetectionBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *outlierDetectionBalancer) Close() {
+	b.closed.Fire()
+	<-b.done.Done()
+	b.child.closeLB()
+
+	b.scUpdateCh.Close()
+	b.pickerUpdateCh.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+}
+
+func (b *outlierDetectionBalancer) ExitIdle() {
+	b.child.exitIdle()
+}
+
+// wrappedPicker delegates to the child policy's picker, and when the request
+// finishes, it increments the corresponding counter in the map entry referenced
+// by the subConnWrapper that was picked. If both the `success_rate_ejection`
+// and `failure_percentage_ejection` fields are unset in the configuration, this
+// picker will not count.
+type wrappedPicker struct {
+	childPicker balancer.Picker
+	noopPicker  bool
+}
+
+func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	pr, err := wp.childPicker.Pick(info)
+	if err != nil {
+		return balancer.PickResult{}, err
+	}
+
+	done := func(di balancer.DoneInfo) {
+		if !wp.noopPicker {
+			incrementCounter(pr.SubConn, di)
+		}
+		if pr.Done != nil {
+			pr.Done(di)
+		}
+	}
+	scw, ok := pr.SubConn.(*subConnWrapper)
+	if !ok {
+		// This can never happen, but check is present for defensive
+		// programming.
+		logger.Errorf("Picked SubConn from child picker is not a SubConnWrapper")
+		return balancer.PickResult{
+			SubConn:  pr.SubConn,
+			Done:     done,
+			Metadata: pr.Metadata,
+		}, nil
+	}
+	return balancer.PickResult{
+		SubConn:  scw.SubConn,
+		Done:     done,
+		Metadata: pr.Metadata,
+	}, nil
+}
+
+func incrementCounter(sc balancer.SubConn, info balancer.DoneInfo) {
+	scw, ok := sc.(*subConnWrapper)
+	if !ok {
+		// Shouldn't happen, as comes from child
+		return
+	}
+
+	// After reading callCounter.activeBucket in this picker a swap call can
+	// concurrently change what activeBucket points to. A50 says to swap the
+	// pointer, which will cause this race to write to deprecated memory the
+	// interval timer algorithm will never read, which makes this race alright.
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	ab := epInfo.callCounter.activeBucket.Load()
+
+	if info.Err == nil {
+		atomic.AddUint32(&ab.numSuccesses, 1)
+	} else {
+		atomic.AddUint32(&ab.numFailures, 1)
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateState(s balancer.State) {
+	b.pickerUpdateCh.Put(s)
+}
+
+func (b *outlierDetectionBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	oldListener := opts.StateListener
+	scw := &subConnWrapper{
+		addresses:         addrs,
+		scUpdateCh:        b.scUpdateCh,
+		listener:          oldListener,
+		latestHealthState: balancer.SubConnState{ConnectivityState: connectivity.Connecting},
+	}
+	opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(scw, state) }
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	sc, err := b.ClientConn.NewSubConn(addrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	if len(addrs) != 1 {
+		return scw, nil
+	}
+	epInfo, ok := b.addrs[addrs[0].Addr]
+	if !ok {
+		return scw, nil
+	}
+	epInfo.sws = append(epInfo.sws, scw)
+	scw.endpointInfo = epInfo
+	if !epInfo.latestEjectionTimestamp.IsZero() {
+		scw.eject()
+	}
+	return scw, nil
+}
+
+func (b *outlierDetectionBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+// removeSubConnFromEndpointMapEntry removes the scw from its map entry if
+// present.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) removeSubConnFromEndpointMapEntry(scw *subConnWrapper) {
+	epInfo := scw.endpointInfo
+	if epInfo == nil {
+		return
+	}
+	for i, sw := range epInfo.sws {
+		if scw == sw {
+			epInfo.sws = append(epInfo.sws[:i], epInfo.sws[i+1:]...)
+			return
+		}
+	}
+}
+
+func (b *outlierDetectionBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// handleSubConnUpdate stores the recent state and forward the update.
+func (b *outlierDetectionBalancer) handleSubConnUpdate(u *scUpdate) {
+	scw := u.scw
+	scw.clearHealthListener()
+	b.child.updateSubConnState(scw, u.state)
+	if u.state.ConnectivityState == connectivity.Shutdown {
+		b.mu.Lock()
+		b.removeSubConnFromEndpointMapEntry(scw)
+		b.mu.Unlock()
+	}
+}
+
+func (b *outlierDetectionBalancer) handleSubConnHealthUpdate(u *scHealthUpdate) {
+	b.child.updateSubConnHealthState(u.scw, u.state)
+}
+
+// handleEjectedUpdate handles any SubConns that get ejected/unejected, and
+// forwards the appropriate corresponding subConnState to the child policy.
+func (b *outlierDetectionBalancer) handleEjectedUpdate(u *ejectionUpdate) {
+	b.child.handleEjectionUpdate(u)
+}
+
+// handleChildStateUpdate forwards the picker update wrapped in a wrapped picker
+// with the noop picker bit present.
+func (b *outlierDetectionBalancer) handleChildStateUpdate(u balancer.State) {
+	b.childState = u
+	b.mu.Lock()
+	if b.inhibitPickerUpdates {
+		// If a child's state is updated during the suppression of child
+		// updates, the synchronous handleLBConfigUpdate function with respect
+		// to UpdateClientConnState should return a picker unconditionally.
+		b.updateUnconditionally = true
+		b.mu.Unlock()
+		return
+	}
+	noopCfg := b.noopConfig()
+	b.mu.Unlock()
+	b.recentPickerNoop = noopCfg
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: b.childState.ConnectivityState,
+		Picker: &wrappedPicker{
+			childPicker: b.childState.Picker,
+			noopPicker:  noopCfg,
+		},
+	})
+}
+
+// handleLBConfigUpdate compares whether the new config is a noop config or not,
+// to the noop bit in the picker if present. It updates the picker if this bit
+// changed compared to the picker currently in use.
+func (b *outlierDetectionBalancer) handleLBConfigUpdate(u lbCfgUpdate) {
+	lbCfg := u.lbCfg
+	noopCfg := lbCfg.SuccessRateEjection == nil && lbCfg.FailurePercentageEjection == nil
+	// If the child has sent its first update and this config flips the noop
+	// bit compared to the most recent picker update sent upward, then a new
+	// picker with this updated bit needs to be forwarded upward. If a child
+	// update was received during the suppression of child updates within
+	// UpdateClientConnState(), then a new picker needs to be forwarded with
+	// this updated state, irregardless of whether this new configuration flips
+	// the bit.
+	if b.childState.Picker != nil && noopCfg != b.recentPickerNoop || b.updateUnconditionally {
+		b.recentPickerNoop = noopCfg
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker: &wrappedPicker{
+				childPicker: b.childState.Picker,
+				noopPicker:  noopCfg,
+			},
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.updateUnconditionally = false
+	close(u.done)
+}
+
+func (b *outlierDetectionBalancer) run() {
+	defer b.done.Fire()
+	for {
+		select {
+		case update, ok := <-b.scUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.scUpdateCh.Load()
+			if b.closed.HasFired() { // don't send SubConn updates to child after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case *scUpdate:
+				b.handleSubConnUpdate(u)
+			case *ejectionUpdate:
+				b.handleEjectedUpdate(u)
+			case *scHealthUpdate:
+				b.handleSubConnHealthUpdate(u)
+			}
+		case update, ok := <-b.pickerUpdateCh.Get():
+			if !ok {
+				return
+			}
+			b.pickerUpdateCh.Load()
+			if b.closed.HasFired() { // don't send picker updates to grpc after the balancer has been closed
+				return
+			}
+			switch u := update.(type) {
+			case balancer.State:
+				b.handleChildStateUpdate(u)
+			case lbCfgUpdate:
+				b.handleLBConfigUpdate(u)
+			}
+		case <-b.closed.Done():
+			return
+		}
+	}
+}
+
+// intervalTimerAlgorithm ejects and unejects endpoints based on the Outlier
+// Detection configuration and data about each endpoint from the previous
+// interval.
+func (b *outlierDetectionBalancer) intervalTimerAlgorithm() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.timerStartTime = time.Now()
+
+	for _, epInfo := range b.endpoints.All() {
+		epInfo.callCounter.swap()
+	}
+
+	if b.cfg.SuccessRateEjection != nil {
+		b.successRateAlgorithm()
+	}
+
+	if b.cfg.FailurePercentageEjection != nil {
+		b.failurePercentageAlgorithm()
+	}
+
+	for _, epInfo := range b.endpoints.All() {
+		if epInfo.latestEjectionTimestamp.IsZero() && epInfo.ejectionTimeMultiplier > 0 {
+			epInfo.ejectionTimeMultiplier--
+			continue
+		}
+		if epInfo.latestEjectionTimestamp.IsZero() {
+			// Endpoint is already not ejected, so no need to check for whether
+			// to uneject the endpoint below.
+			continue
+		}
+		et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(epInfo.ejectionTimeMultiplier)
+		met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime))
+		uet := epInfo.latestEjectionTimestamp.Add(min(et, met))
+		if now().After(uet) {
+			b.unejectEndpoint(epInfo)
+		}
+	}
+
+	// This conditional only for testing (since the interval timer algorithm is
+	// called manually), will never hit in production.
+	if b.intervalTimer != nil {
+		b.intervalTimer.Stop()
+	}
+	b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm)
+}
+
+// endpointsWithAtLeastRequestVolume returns a slice of endpoint information of
+// all endpoints with at least request volume passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) endpointsWithAtLeastRequestVolume(requestVolume uint32) []*endpointInfo {
+	var endpoints []*endpointInfo
+	for _, epInfo := range b.endpoints.All() {
+		bucket1 := epInfo.callCounter.inactiveBucket
+		rv := bucket1.numSuccesses + bucket1.numFailures
+		if rv >= requestVolume {
+			endpoints = append(endpoints, epInfo)
+		}
+	}
+	return endpoints
+}
+
+// meanAndStdDev returns the mean and std dev of the fractions of successful
+// requests of the endpoints passed in.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) meanAndStdDev(endpoints []*endpointInfo) (float64, float64) {
+	var totalFractionOfSuccessfulRequests float64
+	var mean float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		totalFractionOfSuccessfulRequests += float64(bucket.numSuccesses) / float64(rv)
+	}
+	mean = totalFractionOfSuccessfulRequests / float64(len(endpoints))
+	var sumOfSquares float64
+	for _, epInfo := range endpoints {
+		bucket := epInfo.callCounter.inactiveBucket
+		rv := bucket.numSuccesses + bucket.numFailures
+		devFromMean := (float64(bucket.numSuccesses) / float64(rv)) - mean
+		sumOfSquares += devFromMean * devFromMean
+	}
+	variance := sumOfSquares / float64(len(endpoints))
+	return mean, math.Sqrt(variance)
+}
+
+// successRateAlgorithm ejects any endpoints where the success rate falls below
+// the other endpoints according to mean and standard deviation, and if overall
+// applicable from other set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) successRateAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.SuccessRateEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.SuccessRateEjection.MinimumHosts) {
+		return
+	}
+	mean, stddev := b.meanAndStdDev(endpointsToConsider)
+	ejectionCfg := b.cfg.SuccessRateEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures)
+		requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000)
+		if successRate < requiredSuccessRate {
+			channelz.Infof(logger, b.channelzParent, "SuccessRate algorithm detected outlier: %s. Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", epInfo, successRate, mean, stddev, requiredSuccessRate)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "success_rate")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "success_rate", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// failurePercentageAlgorithm ejects any endpoints where the failure percentage
+// rate exceeds a set enforcement percentage, if overall applicable from other
+// set heuristics.
+//
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) failurePercentageAlgorithm() {
+	endpointsToConsider := b.endpointsWithAtLeastRequestVolume(b.cfg.FailurePercentageEjection.RequestVolume)
+	if len(endpointsToConsider) < int(b.cfg.FailurePercentageEjection.MinimumHosts) {
+		return
+	}
+
+	ejectionCfg := b.cfg.FailurePercentageEjection
+	for _, epInfo := range endpointsToConsider {
+		bucket := epInfo.callCounter.inactiveBucket
+		failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100
+		if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) {
+			channelz.Infof(logger, b.channelzParent, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", epInfo, failurePercentage)
+			// Check if max ejection percentage would prevent ejection.
+			if float64(b.numEndpointsEjected)/float64(b.endpoints.Len())*100 >= float64(b.cfg.MaxEjectionPercent) {
+				// Record unenforced ejection due to max ejection percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "max_ejection_overflow")
+				continue
+			}
+			if uint32(rand.Int32N(100)) < ejectionCfg.EnforcementPercentage {
+				b.ejectEndpoint(epInfo, "failure_percentage")
+			} else {
+				// Record unenforced ejection due to enforcement percentage.
+				ejectionsUnenforcedMetric.Record(b.metricsRecorder, 1, b.target, "failure_percentage", "enforcement_percentage")
+			}
+		}
+	}
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) ejectEndpoint(epInfo *endpointInfo, detectionMethod string) {
+	b.numEndpointsEjected++
+	epInfo.latestEjectionTimestamp = b.timerStartTime
+	epInfo.ejectionTimeMultiplier++
+	for _, sbw := range epInfo.sws {
+		sbw.eject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel ejected: %s", sbw)
+	}
+
+	// Record the enforced ejection metric.
+	ejectionsEnforcedMetric.Record(b.metricsRecorder, 1, b.target, detectionMethod)
+}
+
+// Caller must hold b.mu.
+func (b *outlierDetectionBalancer) unejectEndpoint(epInfo *endpointInfo) {
+	b.numEndpointsEjected--
+	epInfo.latestEjectionTimestamp = time.Time{}
+	for _, sbw := range epInfo.sws {
+		sbw.uneject()
+		channelz.Infof(logger, b.channelzParent, "Subchannel unejected: %s", sbw)
+	}
+}
+
+// synchronizingBalancerWrapper serializes calls into balancer (to uphold the
+// balancer.Balancer API guarantee of synchronous calls). It also ensures a
+// consistent order of locking mutexes when using SubConn listeners to avoid
+// deadlocks.
+type synchronizingBalancerWrapper struct {
+	// mu should not be used directly from outside this struct, instead use
+	// methods defined on the struct.
+	mu sync.Mutex
+	lb *gracefulswitch.Balancer
+}
+
+func (sbw *synchronizingBalancerWrapper) switchTo(builder balancer.Builder) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.SwitchTo(builder)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateClientConnState(state balancer.ClientConnState) error {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	return sbw.lb.UpdateClientConnState(state)
+}
+
+func (sbw *synchronizingBalancerWrapper) resolverError(err error) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ResolverError(err)
+}
+
+func (sbw *synchronizingBalancerWrapper) closeLB() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.Close()
+}
+
+func (sbw *synchronizingBalancerWrapper) exitIdle() {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	sbw.lb.ExitIdle()
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnHealthState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnHealthState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) updateSubConnState(scw *subConnWrapper, scs balancer.SubConnState) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	scw.updateSubConnConnectivityState(scs)
+}
+
+func (sbw *synchronizingBalancerWrapper) handleEjectionUpdate(u *ejectionUpdate) {
+	sbw.mu.Lock()
+	defer sbw.mu.Unlock()
+	if u.isEjected {
+		u.scw.handleEjection()
+	} else {
+		u.scw.handleUnejection()
+	}
+}
+
+// endpointInfo contains the runtime information about an endpoint that pertains
+// to Outlier Detection. This struct and all of its fields is protected by
+// outlierDetectionBalancer.mu in the case where it is accessed through the
+// address or endpoint map. In the case of Picker callbacks, the writes to the
+// activeBucket of callCounter are protected by atomically loading and storing
+// unsafe.Pointers (see further explanation in incrementCounter()).
+type endpointInfo struct {
+	// The call result counter object.
+	callCounter *callCounter
+
+	// The latest ejection timestamp, or zero if the endpoint is currently not
+	// ejected.
+	latestEjectionTimestamp time.Time
+
+	// The current ejection time multiplier, starting at 0.
+	ejectionTimeMultiplier int64
+
+	// A list of subchannel wrapper objects that correspond to this endpoint.
+	sws []*subConnWrapper
+}
+
+func (a *endpointInfo) String() string {
+	var res strings.Builder
+	res.WriteString("[")
+	for _, sw := range a.sws {
+		res.WriteString(sw.String())
+	}
+	res.WriteString("]")
+	return res.String()
+}
+
+func newEndpointInfo() *endpointInfo {
+	return &endpointInfo{
+		callCounter: newCallCounter(),
+	}
+}

internal/xds/balancer/outlierdetection/balancer_ext_test.go

@@ -0,0 +1,518 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection_test
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl"
+	"google.golang.org/grpc/internal/xds/balancer/outlierdetection"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestParseConfig verifies the ParseConfig() method in the Outlier Detection
+// Balancer.
+func (s) TestParseConfig(t *testing.T) {
+	const errParseConfigName = "errParseConfigBalancer"
+	stub.Register(errParseConfigName, stub.BalancerFuncs{
+		ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return nil, errors.New("some error")
+		},
+	})
+
+	parser := balancer.Get(outlierdetection.Name).(balancer.ConfigParser)
+	const (
+		defaultInterval                       = iserviceconfig.Duration(10 * time.Second)
+		defaultBaseEjectionTime               = iserviceconfig.Duration(30 * time.Second)
+		defaultMaxEjectionTime                = iserviceconfig.Duration(300 * time.Second)
+		defaultMaxEjectionPercent             = 10
+		defaultSuccessRateStdevFactor         = 1900
+		defaultEnforcingSuccessRate           = 100
+		defaultSuccessRateMinimumHosts        = 5
+		defaultSuccessRateRequestVolume       = 100
+		defaultFailurePercentageThreshold     = 85
+		defaultEnforcingFailurePercentage     = 0
+		defaultFailurePercentageMinimumHosts  = 5
+		defaultFailurePercentageRequestVolume = 50
+	)
+	tests := []struct {
+		name    string
+		input   string
+		wantCfg serviceconfig.LoadBalancingConfig
+		wantErr string
+	}{
+		{
+			name: "Default",
+			input: `{
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+
+		{
+			name: "TopLevelSet",
+			input: `{
+				"interval": "15s",
+				"maxEjectionTime": "350s",
+				"childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for unset fields.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(15 * time.Second),
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    iserviceconfig.Duration(350 * time.Second),
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Defaults",
+			input: `{
+				"successRateEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of success-rate-ejection struct.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           defaultSuccessRateStdevFactor,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          defaultSuccessRateMinimumHosts,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Partial",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"minimumHosts": 5
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: defaultEnforcingSuccessRate,
+					MinimumHosts:          5,
+					RequestVolume:         defaultSuccessRateRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "SuccessRate_Full",
+			input: `{
+				"successRateEjection": {
+					"stdevFactor": 1000,
+					"enforcementPercentage": 50,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1000,
+					EnforcementPercentage: 50,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Defaults",
+			input: `{
+				"failurePercentageEjection": {},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get defaults of failure percentage ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             defaultFailurePercentageThreshold,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          defaultFailurePercentageMinimumHosts,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Partial",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"minimumHosts": 10
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			// Should get set fields + defaults for others in success rate
+			// ejection layer.
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: defaultEnforcingFailurePercentage,
+					MinimumHosts:          10,
+					RequestVolume:         defaultFailurePercentageRequestVolume,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "FailurePercentage_Full",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 80,
+					"enforcementPercentage": 100,
+					"minimumHosts": 10,
+					"requestVolume": 40
+                },
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           defaultInterval,
+				BaseEjectionTime:   defaultBaseEjectionTime,
+				MaxEjectionTime:    defaultMaxEjectionTime,
+				MaxEjectionPercent: defaultMaxEjectionPercent,
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             80,
+					EnforcementPercentage: 100,
+					MinimumHosts:          10,
+					RequestVolume:         40,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{ // to make sure zero values aren't overwritten by defaults
+			name: "AllFields_Zero",
+			input: `{
+				"interval": "0s",
+				"baseEjectionTime": "0s",
+				"maxEjectionTime": "0s",
+				"maxEjectionPercent": 0,
+				"successRateEjection": {
+					"stdevFactor": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+				"failurePercentageEjection": {
+					"threshold": 0,
+					"enforcementPercentage": 0,
+					"minimumHosts": 0,
+					"requestVolume": 0
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				SuccessRateEjection:       &outlierdetection.SuccessRateEjection{},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name: "AllFields_Set",
+			input: `{
+				"interval": "10s",
+				"baseEjectionTime": "30s",
+				"maxEjectionTime": "300s",
+				"maxEjectionPercent": 10,
+				"successRateEjection": {
+					"stdevFactor": 1900,
+					"enforcementPercentage": 100,
+					"minimumHosts": 5,
+					"requestVolume": 100
+				},
+				"failurePercentageEjection": {
+					"threshold": 85,
+					"enforcementPercentage": 5,
+					"minimumHosts": 5,
+					"requestVolume": 50
+				},
+                "childPolicy": [
+				{
+					"xds_cluster_impl_experimental": {
+						"cluster": "test_cluster"
+					}
+				}
+				]
+			}`,
+			wantCfg: &outlierdetection.LBConfig{
+				Interval:           iserviceconfig.Duration(10 * time.Second),
+				BaseEjectionTime:   iserviceconfig.Duration(30 * time.Second),
+				MaxEjectionTime:    iserviceconfig.Duration(300 * time.Second),
+				MaxEjectionPercent: 10,
+				SuccessRateEjection: &outlierdetection.SuccessRateEjection{
+					StdevFactor:           1900,
+					EnforcementPercentage: 100,
+					MinimumHosts:          5,
+					RequestVolume:         100,
+				},
+				FailurePercentageEjection: &outlierdetection.FailurePercentageEjection{
+					Threshold:             85,
+					EnforcementPercentage: 5,
+					MinimumHosts:          5,
+					RequestVolume:         50,
+				},
+				ChildPolicy: &iserviceconfig.BalancerConfig{
+					Name: "xds_cluster_impl_experimental",
+					Config: &clusterimpl.LBConfig{
+						Cluster: "test_cluster",
+					},
+				},
+			},
+		},
+		{
+			name:    "Interval_Negative",
+			input:   `{"interval": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0",
+		},
+		{
+			name:    "BaseEjectionTime_Negative",
+			input:   `{"baseEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionTime_Negative",
+			input:   `{"maxEjectionTime": "-10s"}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0",
+		},
+		{
+			name:    "MaxEjectionPercent_TooHigh",
+			input:   `{"maxEjectionPercent": 150}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_percent = 150; must be <= 100",
+		},
+		{
+			name: "SuccessRate_Enforcement_TooHigh",
+			input: `{
+				"successRateEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Threshold_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"threshold": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = 150; must be <= 100",
+		},
+		{
+			name: "FailurePercentage_Enforcement_TooHigh",
+			input: `{
+				"failurePercentageEjection": {
+					"enforcementPercentage": 150
+				}
+			}`,
+			wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = 150; must be <= 100",
+		},
+		{
+			name: "ChildPolicy_ParseError",
+			input: `{
+				"childPolicy": [
+				{
+					"errParseConfigBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"",
+		},
+		{
+			name: "ChildPolicy_NotSupported",
+			input: `{
+				"childPolicy": [
+				{
+					"doesNotExistBalancer": {
+						"cluster": "test_cluster"
+					}
+				}
+			]
+			}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input))
+			if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if (gotErr != nil) != (test.wantErr != "") {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" {
+				t.Fatalf("parseConfig(%v) got unexpected output, diff (-got +want): %v", string(test.input), diff)
+			}
+		})
+	}
+}

internal/xds/balancer/outlierdetection/callcounter.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"sync/atomic"
+)
+
+type bucket struct {
+	numSuccesses uint32
+	numFailures  uint32
+}
+
+func newCallCounter() *callCounter {
+	cc := &callCounter{
+		inactiveBucket: &bucket{},
+	}
+	cc.activeBucket.Store(&bucket{})
+	return cc
+}
+
+// callCounter has two buckets, which each count successful and failing RPC's.
+// The activeBucket is used to actively count any finished RPC's, and the
+// inactiveBucket is populated with this activeBucket's data every interval for
+// use by the Outlier Detection algorithm.
+type callCounter struct {
+	// activeBucket updates every time a call finishes (from picker passed to
+	// Client Conn), so protect pointer read with atomic load of the pointer
+	// so picker does not have to grab a mutex per RPC, the critical path.
+	activeBucket   atomic.Pointer[bucket]
+	inactiveBucket *bucket
+}
+
+func (cc *callCounter) clear() {
+	cc.activeBucket.Store(&bucket{})
+	cc.inactiveBucket = &bucket{}
+}
+
+// "When the timer triggers, the inactive bucket is zeroed and swapped with the
+// active bucket. Then the inactive bucket contains the number of successes and
+// failures since the last time the timer triggered. Those numbers are used to
+// evaluate the ejection criteria." - A50.
+func (cc *callCounter) swap() {
+	ib := cc.inactiveBucket
+	*ib = bucket{}
+	ab := cc.activeBucket.Swap(ib)
+	cc.inactiveBucket = &bucket{
+		numSuccesses: atomic.LoadUint32(&ab.numSuccesses),
+		numFailures:  atomic.LoadUint32(&ab.numFailures),
+	}
+}

internal/xds/balancer/outlierdetection/callcounter_test.go

@@ -0,0 +1,92 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func (b1 *bucket) Equal(b2 *bucket) bool {
+	if b1 == nil && b2 == nil {
+		return true
+	}
+	if (b1 != nil) != (b2 != nil) {
+		return false
+	}
+	if b1.numSuccesses != b2.numSuccesses {
+		return false
+	}
+	return b1.numFailures == b2.numFailures
+}
+
+func (cc *callCounter) Equal(cc2 *callCounter) bool {
+	if cc == nil && cc2 == nil {
+		return true
+	}
+	if (cc != nil) != (cc2 != nil) {
+		return false
+	}
+	ab1 := cc.activeBucket.Load()
+	ab2 := cc2.activeBucket.Load()
+	if !ab1.Equal(ab2) {
+		return false
+	}
+	return cc.inactiveBucket.Equal(cc2.inactiveBucket)
+}
+
+// TestClear tests that clear on the call counter clears (everything set to 0)
+// the active and inactive buckets.
+func (s) TestClear(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	cc.clear()
+	// Both the active and inactive buckets should be cleared.
+	ccWant := newCallCounter()
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}
+
+// TestSwap tests that swap() on the callCounter successfully has the desired
+// end result of inactive bucket containing the previous active buckets data,
+// and the active bucket being cleared.
+func (s) TestSwap(t *testing.T) {
+	cc := newCallCounter()
+	ab := cc.activeBucket.Load()
+	ab.numSuccesses = 1
+	ab.numFailures = 2
+	cc.inactiveBucket.numSuccesses = 4
+	cc.inactiveBucket.numFailures = 5
+	ib := cc.inactiveBucket
+	cc.swap()
+	// Inactive should pick up active's data, active should be swapped to zeroed
+	// inactive.
+	ccWant := newCallCounter()
+	ccWant.inactiveBucket.numSuccesses = 1
+	ccWant.inactiveBucket.numFailures = 2
+	ccWant.activeBucket.Store(ib)
+	if diff := cmp.Diff(cc, ccWant); diff != "" {
+		t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff)
+	}
+}

internal/xds/balancer/outlierdetection/config.go

@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"encoding/json"
+	"time"
+
+	iserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// SuccessRateEjection is parameters for the success rate ejection algorithm.
+// This algorithm monitors the request success rate for all endpoints and ejects
+// individual endpoints whose success rates are statistical outliers.
+type SuccessRateEjection struct {
+	// StddevFactor is used to determine the ejection threshold for
+	// success rate outlier ejection. The ejection threshold is the difference
+	// between the mean success rate, and the product of this factor and the
+	// standard deviation of the mean success rate: mean - (stdev *
+	// success_rate_stdev_factor). This factor is divided by a thousand to get a
+	// double. That is, if the desired factor is 1.9, the runtime value should
+	// be 1900. Defaults to 1900.
+	StdevFactor uint32 `json:"stdevFactor,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually ejected
+	// when an outlier status is detected through success rate statistics. This
+	// setting can be used to disable ejection or to ramp it up slowly. Defaults
+	// to 100.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the number of hosts in a cluster that must have enough
+	// request volume to detect success rate outliers. If the number of hosts is
+	// less than this setting, outlier detection via success rate statistics is
+	// not performed for any host in the cluster. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// include this host in success rate based outlier detection. If the volume
+	// is lower than this setting, outlier detection via success rate statistics
+	// is not performed for that host. Defaults to 100.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type successRateEjection SuccessRateEjection
+
+// UnmarshalJSON unmarshals JSON into SuccessRateEjection. If a
+// SuccessRateEjection field is not set, that field will get its default value.
+func (sre *SuccessRateEjection) UnmarshalJSON(j []byte) error {
+	sre.StdevFactor = 1900
+	sre.EnforcementPercentage = 100
+	sre.MinimumHosts = 5
+	sre.RequestVolume = 100
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*successRateEjection)(sre))
+}
+
+// Equal returns whether the SuccessRateEjection is the same with the parameter.
+func (sre *SuccessRateEjection) Equal(sre2 *SuccessRateEjection) bool {
+	if sre == nil && sre2 == nil {
+		return true
+	}
+	if (sre != nil) != (sre2 != nil) {
+		return false
+	}
+	if sre.StdevFactor != sre2.StdevFactor {
+		return false
+	}
+	if sre.EnforcementPercentage != sre2.EnforcementPercentage {
+		return false
+	}
+	if sre.MinimumHosts != sre2.MinimumHosts {
+		return false
+	}
+	return sre.RequestVolume == sre2.RequestVolume
+}
+
+// FailurePercentageEjection is parameters for the failure percentage algorithm.
+// This algorithm ejects individual endpoints whose failure rate is greater than
+// some threshold, independently of any other endpoint.
+type FailurePercentageEjection struct {
+	// Threshold is the failure percentage to use when determining failure
+	// percentage-based outlier detection. If the failure percentage of a given
+	// host is greater than or equal to this value, it will be ejected. Defaults
+	// to 85.
+	Threshold uint32 `json:"threshold,omitempty"`
+	// EnforcementPercentage is the % chance that a host will be actually
+	// ejected when an outlier status is detected through failure percentage
+	// statistics. This setting can be used to disable ejection or to ramp it up
+	// slowly. Defaults to 0.
+	EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"`
+	// MinimumHosts is the minimum number of hosts in a cluster in order to
+	// perform failure percentage-based ejection. If the total number of hosts
+	// in the cluster is less than this value, failure percentage-based ejection
+	// will not be performed. Defaults to 5.
+	MinimumHosts uint32 `json:"minimumHosts,omitempty"`
+	// RequestVolume is the minimum number of total requests that must be
+	// collected in one interval (as defined by the interval duration above) to
+	// perform failure percentage-based ejection for this host. If the volume is
+	// lower than this setting, failure percentage-based ejection will not be
+	// performed for this host. Defaults to 50.
+	RequestVolume uint32 `json:"requestVolume,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type failurePercentageEjection FailurePercentageEjection
+
+// UnmarshalJSON unmarshals JSON into FailurePercentageEjection. If a
+// FailurePercentageEjection field is not set, that field will get its default
+// value.
+func (fpe *FailurePercentageEjection) UnmarshalJSON(j []byte) error {
+	fpe.Threshold = 85
+	fpe.EnforcementPercentage = 0
+	fpe.MinimumHosts = 5
+	fpe.RequestVolume = 50
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*failurePercentageEjection)(fpe))
+}
+
+// Equal returns whether the FailurePercentageEjection is the same with the
+// parameter.
+func (fpe *FailurePercentageEjection) Equal(fpe2 *FailurePercentageEjection) bool {
+	if fpe == nil && fpe2 == nil {
+		return true
+	}
+	if (fpe != nil) != (fpe2 != nil) {
+		return false
+	}
+	if fpe.Threshold != fpe2.Threshold {
+		return false
+	}
+	if fpe.EnforcementPercentage != fpe2.EnforcementPercentage {
+		return false
+	}
+	if fpe.MinimumHosts != fpe2.MinimumHosts {
+		return false
+	}
+	return fpe.RequestVolume == fpe2.RequestVolume
+}
+
+// LBConfig is the config for the outlier detection balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+	// Interval is the time interval between ejection analysis sweeps. This can
+	// result in both new ejections as well as addresses being returned to
+	// service. Defaults to 10s.
+	Interval iserviceconfig.Duration `json:"interval,omitempty"`
+	// BaseEjectionTime is the base time that a host is ejected for. The real
+	// time is equal to the base time multiplied by the number of times the host
+	// has been ejected and is capped by MaxEjectionTime. Defaults to 30s.
+	BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"`
+	// MaxEjectionTime is the maximum time that an endpoint is ejected for. If
+	// not specified, the default value (300s) or the BaseEjectionTime value is
+	// applied, whichever is larger.
+	MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"`
+	// MaxEjectionPercent is the maximum % of an upstream cluster that can be
+	// ejected due to outlier detection. Defaults to 10% but will eject at least
+	// one host regardless of the value.
+	MaxEjectionPercent uint32 `json:"maxEjectionPercent,omitempty"`
+	// SuccessRateEjection is the parameters for the success rate ejection
+	// algorithm. If set, success rate ejections will be performed.
+	SuccessRateEjection *SuccessRateEjection `json:"successRateEjection,omitempty"`
+	// FailurePercentageEjection is the parameters for the failure percentage
+	// algorithm. If set, failure rate ejections will be performed.
+	FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"`
+	// ChildPolicy is the config for the child policy.
+	ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+// For UnmarshalJSON to work correctly and set defaults without infinite
+// recursion.
+type lbConfig LBConfig
+
+// UnmarshalJSON unmarshals JSON into LBConfig. If a top level LBConfig field
+// (i.e. not next layer sre or fpe) is not set, that field will get its default
+// value. If sre or fpe is not set, it will stay unset, otherwise it will
+// unmarshal on those types populating with default values for their fields if
+// needed.
+func (lbc *LBConfig) UnmarshalJSON(j []byte) error {
+	// Default top layer values as documented in A50.
+	lbc.Interval = iserviceconfig.Duration(10 * time.Second)
+	lbc.BaseEjectionTime = iserviceconfig.Duration(30 * time.Second)
+	lbc.MaxEjectionTime = iserviceconfig.Duration(300 * time.Second)
+	lbc.MaxEjectionPercent = 10
+	// Unmarshal JSON on a type with zero values for methods, including
+	// UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to
+	// avoid infinite recursion by not recalling this function and causing stack
+	// overflow.
+	return json.Unmarshal(j, (*lbConfig)(lbc))
+}
+
+// EqualIgnoringChildPolicy returns whether the LBConfig is same with the
+// parameter outside of the child policy, only comparing the Outlier Detection
+// specific configuration.
+func (lbc *LBConfig) EqualIgnoringChildPolicy(lbc2 *LBConfig) bool {
+	if lbc == nil && lbc2 == nil {
+		return true
+	}
+	if (lbc != nil) != (lbc2 != nil) {
+		return false
+	}
+	if lbc.Interval != lbc2.Interval {
+		return false
+	}
+	if lbc.BaseEjectionTime != lbc2.BaseEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionTime != lbc2.MaxEjectionTime {
+		return false
+	}
+	if lbc.MaxEjectionPercent != lbc2.MaxEjectionPercent {
+		return false
+	}
+	if !lbc.SuccessRateEjection.Equal(lbc2.SuccessRateEjection) {
+		return false
+	}
+	return lbc.FailurePercentageEjection.Equal(lbc2.FailurePercentageEjection)
+}

internal/xds/balancer/outlierdetection/config_test.go

@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSuccessRateEjection(t *testing.T) {
+	fields := map[string]bool{
+		"StdevFactor":           true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(SuccessRateEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in SuccessRateEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsFailurePercentageEjection(t *testing.T) {
+	fields := map[string]bool{
+		"Threshold":             true,
+		"EnforcementPercentage": true,
+		"MinimumHosts":          true,
+		"RequestVolume":         true,
+	}
+	typ := reflect.TypeOf(FailurePercentageEjection{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in FailurePercentageEjection %q, update this test and Equal", n)
+		}
+	}
+}
+
+func TestEqualFieldsLBConfig(t *testing.T) {
+	fields := map[string]bool{
+		"LoadBalancingConfig":       true,
+		"Interval":                  true,
+		"BaseEjectionTime":          true,
+		"MaxEjectionTime":           true,
+		"MaxEjectionPercent":        true,
+		"SuccessRateEjection":       true,
+		"FailurePercentageEjection": true,
+		"ChildPolicy":               true,
+	}
+	typ := reflect.TypeOf(LBConfig{})
+	for i := 0; i < typ.NumField(); i++ {
+		if n := typ.Field(i).Name; !fields[n] {
+			t.Errorf("New field in LBConfig %q, update this test and EqualIgnoringChildPolicy", n)
+		}
+	}
+}

internal/xds/balancer/outlierdetection/e2e_test/outlierdetection_test.go

@@ -0,0 +1,425 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e_test contains e2e test cases for the Outlier Detection LB Policy.
+package e2e_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/balancer/outlierdetection" // To register helper functions which register/unregister Outlier Detection LB Policy.
+)
+
+var (
+	defaultTestTimeout = 5 * time.Second
+	leafPolicyName     = "round_robin"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// Setup spins up three test backends, each listening on a port on localhost.
+// Two of the backends are configured to always reply with an empty response and
+// no error and one is configured to always return an error.
+func setupBackends(t *testing.T) ([]string, func()) {
+	t.Helper()
+
+	backends := make([]*stubserver.StubServer, 3)
+	addresses := make([]string, 3)
+	// Construct and start 2 working backends.
+	for i := 0; i < 2; i++ {
+		backend := &stubserver.StubServer{
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+				return &testpb.Empty{}, nil
+			},
+		}
+		if err := backend.StartServer(); err != nil {
+			t.Fatalf("Failed to start backend: %v", err)
+		}
+		t.Logf("Started good TestService backend at: %q", backend.Address)
+		backends[i] = backend
+		addresses[i] = backend.Address
+	}
+
+	// Construct and start a failing backend.
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started bad TestService backend at: %q", backend.Address)
+	backends[2] = backend
+	addresses[2] = backend.Address
+	cancel := func() {
+		for _, backend := range backends {
+			backend.Stop()
+		}
+	}
+	return addresses, cancel
+}
+
+// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn,
+// connected to a server exposing the test.grpc_testing.TestService, are
+// roundrobined across the given backend addresses.
+//
+// Returns a non-nil error if context deadline expires before RPCs start to get
+// roundrobined across the given backends.
+func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error {
+	wantAddrCount := make(map[string]int)
+	for _, addr := range addrs {
+		wantAddrCount[addr.Addr]++
+	}
+	gotAddrCount := make(map[string]int)
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		gotAddrCount = make(map[string]int)
+		// Perform 3 iterations.
+		var iterations [][]string
+		for i := 0; i < 3; i++ {
+			iteration := make([]string, len(addrs))
+			for c := 0; c < len(addrs); c++ {
+				var peer peer.Peer
+				client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer))
+				if peer.Addr != nil {
+					iteration[c] = peer.Addr.String()
+				}
+			}
+			iterations = append(iterations, iteration)
+		}
+		// Ensure the first iteration contains all addresses in addrs.
+		for _, addr := range iterations[0] {
+			gotAddrCount[addr]++
+		}
+		if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" {
+			continue
+		}
+		// Ensure all three iterations contain the same addresses.
+		if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) {
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount)
+}
+
+// TestPickFirstHealthListenerDisabled verifies that outlier detection doesn't
+// eject subchannels created by pick_first when it is not under a petiole
+// policy. In that configuration, pick_first does not register a health
+// listener and ejection updates must not affect connectivity.
+func (s) TestPickFirstHealthListenerDisabled(t *testing.T) {
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, errors.New("some error")
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	defer backend.Stop()
+
+	const serviceConfig = `{
+	  "loadBalancingConfig": [{
+		"outlier_detection_experimental": {
+		  "interval": "0.050s",
+		  "failurePercentageEjection": {
+			"threshold": 50,
+			"enforcementPercentage": 100,
+			"minimumHosts": 0,
+			"requestVolume": 2
+		  },
+		  "childPolicy": [{"pick_first": {}}]
+		}
+	  }]
+	}`
+	cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(serviceConfig))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	client.EmptyCall(ctx, &testpb.Empty{})
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	for range 10 {
+		client.EmptyCall(ctx, &testpb.Empty{})
+	}
+
+	// Wait for three outlier detection intervals to verify that the subchannel is not ejected.
+	sCtx, sCancel := context.WithTimeout(ctx, 150*time.Millisecond)
+	defer sCancel()
+	testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready)
+}
+
+// TestOutlierDetectionAlgorithmsE2E tests the Outlier Detection Success Rate
+// and Failure Percentage algorithms in an e2e fashion. The Outlier Detection
+// Balancer is configured as the top level LB Policy of the channel with a Round
+// Robin child, and connects to three upstreams. Two of the upstreams are healthy and
+// one is unhealthy. The two algorithms should at some point eject the failing
+// upstream, causing RPC's to not be routed to that upstream, and only be
+// Round Robined across the two healthy upstreams. Other than the intervals the
+// unhealthy upstream is ejected, RPC's should regularly round robin
+// across all three upstreams.
+func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) {
+	tests := []struct {
+		name     string
+		odscJSON string
+	}{
+		{
+			name: "Success Rate Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"successRateEjection": {
+						"stdevFactor": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+		{
+			name: "Failure Percentage Algorithm",
+			odscJSON: fmt.Sprintf(`
+			{
+			  "loadBalancingConfig": [
+				{
+				  "outlier_detection_experimental": {
+					"interval": "0.050s",
+					"baseEjectionTime": "0.100s",
+					"maxEjectionTime": "300s",
+					"maxEjectionPercent": 33,
+					"failurePercentageEjection": {
+						"threshold": 50,
+						"enforcementPercentage": 100,
+						"minimumHosts": 3,
+						"requestVolume": 5
+					},
+					"childPolicy": [{"%s": {}}
+					]
+				  }
+				}
+			  ]
+			}`, leafPolicyName),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			addresses, cancel := setupBackends(t)
+			defer cancel()
+
+			mr := manual.NewBuilderWithScheme("od-e2e")
+			defer mr.Close()
+
+			sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(test.odscJSON)
+			// The full list of addresses.
+			fullAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+				{Addr: addresses[2]},
+			}
+			mr.InitialState(resolver.State{
+				Addresses:     fullAddresses,
+				ServiceConfig: sc,
+			})
+
+			cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+			// At first, due to no statistics on each of the backends, the 3
+			// upstreams should all be round robined across.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The addresses which don't return errors.
+			okAddresses := []resolver.Address{
+				{Addr: addresses[0]},
+				{Addr: addresses[1]},
+			}
+			// After calling the three upstreams, one of them constantly error
+			// and should eventually be ejected for a period of time. This
+			// period of time should cause the RPC's to be round robined only
+			// across the two that are healthy.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+
+			// The failing upstream isn't ejected indefinitely, and eventually
+			// should be unejected in subsequent iterations of the interval
+			// algorithm as per the spec for the two specific algorithms.
+			if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+				t.Fatalf("error in expected round robin: %v", err)
+			}
+		})
+	}
+}
+
+// TestNoopConfiguration tests the Outlier Detection Balancer configured with a
+// noop configuration. The noop configuration should cause the Outlier Detection
+// Balancer to not count RPC's, and thus never eject any upstreams and continue
+// to route to every upstream connected to, even if they continuously error.
+// Once the Outlier Detection Balancer gets reconfigured with configuration
+// requiring counting RPC's, the Outlier Detection Balancer should start
+// ejecting any upstreams as specified in the configuration.
+func (s) TestNoopConfiguration(t *testing.T) {
+	addresses, cancel := setupBackends(t)
+	defer cancel()
+
+	mr := manual.NewBuilderWithScheme("od-e2e")
+	defer mr.Close()
+
+	noopODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(noopODServiceConfigJSON)
+	// The full list of addresses.
+	fullAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+		{Addr: addresses[2]},
+	}
+	mr.InitialState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testServiceClient := testgrpc.NewTestServiceClient(cc)
+
+	for i := 0; i < 2; i++ {
+		// Since the Outlier Detection Balancer starts with a noop
+		// configuration, it shouldn't count RPCs or eject any upstreams. Thus,
+		// even though an upstream it connects to constantly errors, it should
+		// continue to Round Robin across every upstream.
+		if err := checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+			t.Fatalf("error in expected round robin: %v", err)
+		}
+	}
+
+	// Reconfigure the Outlier Detection Balancer with a configuration that
+	// specifies to count RPC's and eject upstreams. Due to the balancer no
+	// longer being a noop, it should eject any unhealthy addresses as specified
+	// by the failure percentage portion of the configuration.
+	countingODServiceConfigJSON := fmt.Sprintf(`
+	{
+	  "loadBalancingConfig": [
+		{
+		  "outlier_detection_experimental": {
+			"interval": "0.050s",
+			"baseEjectionTime": "0.100s",
+			"maxEjectionTime": "300s",
+			"maxEjectionPercent": 33,
+			"failurePercentageEjection": {
+				"threshold": 50,
+				"enforcementPercentage": 100,
+				"minimumHosts": 3,
+				"requestVolume": 5
+			},
+			"childPolicy": [{"%s": {}}]
+		  }
+		}
+	  ]
+	}`, leafPolicyName)
+	sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(countingODServiceConfigJSON)
+
+	mr.UpdateState(resolver.State{
+		Addresses:     fullAddresses,
+		ServiceConfig: sc,
+	})
+
+	// At first on the reconfigured balancer, the balancer has no stats
+	// collected about upstreams. Thus, it should at first route across the full
+	// upstream list.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: addresses[0]},
+		{Addr: addresses[1]},
+	}
+	// Now that the reconfigured balancer has data about the failing upstream,
+	// it should eject the upstream and only route across the two healthy
+	// upstreams.
+	if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}

internal/xds/balancer/outlierdetection/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[outlier-detection-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *outlierDetectionBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/outlierdetection/subconn_wrapper.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package outlierdetection
+
+import (
+	"fmt"
+	"sync"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/resolver"
+)
+
+// subConnWrapper wraps every created SubConn in the Outlier Detection Balancer,
+// to help track the latest state update from the underlying SubConn, and also
+// whether or not this SubConn is ejected.
+type subConnWrapper struct {
+	balancer.SubConn
+	// endpointInfo is a pointer to the subConnWrapper's corresponding endpoint
+	// map entry, if the map entry exists.
+	endpointInfo *endpointInfo
+	// The following fields are set during object creation and read-only after
+	// that.
+
+	listener func(balancer.SubConnState)
+
+	scUpdateCh *buffer.Unbounded[any]
+
+	// The following fields are only referenced in the context of a work
+	// serializing buffer and don't need to be protected by a mutex.
+
+	// These two pieces of state will reach eventual consistency due to sync in
+	// run(), and child will always have the correctly updated SubConnState.
+
+	ejected bool
+
+	// addresses is the list of address(es) this SubConn was created with to
+	// help support any change in address(es)
+	addresses []resolver.Address
+	// latestHealthState is tracked to update the child policy during
+	// unejection.
+	latestHealthState balancer.SubConnState
+
+	// Access to the following fields are protected by a mutex. These fields
+	// should not be accessed from outside this file, instead use methods
+	// defined on the struct.
+	mu             sync.Mutex
+	healthListener func(balancer.SubConnState)
+	// latestReceivedConnectivityState is the SubConn's most recent connectivity
+	// state received. It may not be delivered to the child balancer yet. It is
+	// used to ensure a health listener is registered with the SubConn only when
+	// the SubConn is READY.
+	latestReceivedConnectivityState connectivity.State
+}
+
+// eject causes the wrapper to report a state update with the TRANSIENT_FAILURE
+// state, and to stop passing along updates from the underlying subchannel.
+func (scw *subConnWrapper) eject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: true,
+	})
+}
+
+// uneject causes the wrapper to report a state update with the latest update
+// from the underlying subchannel, and resume passing along updates from the
+// underlying subchannel.
+func (scw *subConnWrapper) uneject() {
+	scw.scUpdateCh.Put(&ejectionUpdate{
+		scw:       scw,
+		isEjected: false,
+	})
+}
+
+func (scw *subConnWrapper) String() string {
+	return fmt.Sprintf("%+v", scw.addresses)
+}
+
+func (scw *subConnWrapper) RegisterHealthListener(listener func(balancer.SubConnState)) {
+	// gRPC currently supports two mechanisms that provide a health signal for
+	// a connection: client-side health checking and outlier detection. Earlier
+	// both these mechanisms signaled unhealthiness by setting the subchannel
+	// state to TRANSIENT_FAILURE. As part of the dualstack changes to make
+	// pick_first the universal leaf policy (see A61), both these mechanisms
+	// started using the new health listener to make health signal visible to
+	// the petiole policies without affecting the underlying connectivity
+	// management of the pick_first policy.
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+
+	if scw.latestReceivedConnectivityState != connectivity.Ready {
+		return
+	}
+	scw.healthListener = listener
+	if listener == nil {
+		scw.SubConn.RegisterHealthListener(nil)
+		return
+	}
+
+	scw.SubConn.RegisterHealthListener(func(scs balancer.SubConnState) {
+		scw.scUpdateCh.Put(&scHealthUpdate{
+			scw:   scw,
+			state: scs,
+		})
+	})
+}
+
+// updateSubConnHealthState stores the latest health state for unejection and
+// sends updates the health listener.
+func (scw *subConnWrapper) updateSubConnHealthState(scs balancer.SubConnState) {
+	scw.latestHealthState = scs
+	if scw.ejected {
+		return
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(scs)
+	}
+}
+
+// updateSubConnConnectivityState stores the latest connectivity state for
+// unejection and updates the raw connectivity listener.
+func (scw *subConnWrapper) updateSubConnConnectivityState(scs balancer.SubConnState) {
+	if scw.listener != nil {
+		scw.listener(scs)
+	}
+}
+
+func (scw *subConnWrapper) clearHealthListener() {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.healthListener = nil
+}
+
+func (scw *subConnWrapper) handleUnejection() {
+	scw.ejected = false
+	// If scw.latestHealthState has never been written to will use the health
+	// state CONNECTING set during object creation.
+	scw.updateSubConnHealthState(scw.latestHealthState)
+}
+
+func (scw *subConnWrapper) handleEjection() {
+	scw.ejected = true
+	stateToUpdate := balancer.SubConnState{
+		ConnectivityState: connectivity.TransientFailure,
+	}
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	if scw.healthListener != nil {
+		scw.healthListener(stateToUpdate)
+	}
+}
+
+func (scw *subConnWrapper) setLatestConnectivityState(state connectivity.State) {
+	scw.mu.Lock()
+	defer scw.mu.Unlock()
+	scw.latestReceivedConnectivityState = state
+}

internal/xds/balancer/priority/balancer.go

@@ -0,0 +1,292 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package priority implements the priority balancer.
+//
+// This balancer will be kept in internal until we use it in the xds balancers,
+// and are confident its functionalities are stable. It will then be exported
+// for more users.
+package priority
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancergroup"
+	"google.golang.org/grpc/internal/buffer"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/hierarchy"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Name is the name of the priority balancer.
+const Name = "priority_experimental"
+
+// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for
+// testing.
+var DefaultSubBalancerCloseTimeout = 15 * time.Minute
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &priorityBalancer{
+		cc:                       cc,
+		done:                     grpcsync.NewEvent(),
+		children:                 make(map[string]*childBalancer),
+		childBalancerStateUpdate: buffer.NewUnbounded[any](),
+	}
+
+	b.logger = prefixLogger(b)
+	b.bg = balancergroup.New(balancergroup.Options{
+		CC:                      cc,
+		BuildOpts:               bOpts,
+		StateAggregator:         b,
+		Logger:                  b.logger,
+		SubBalancerCloseTimeout: DefaultSubBalancerCloseTimeout,
+	})
+	go b.run()
+	b.logger.Infof("Created")
+	return b
+}
+
+func (b bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(s)
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+// timerWrapper wraps a timer with a boolean. So that when a race happens
+// between AfterFunc and Stop, the func is guaranteed to not execute.
+type timerWrapper struct {
+	stopped bool
+	timer   *time.Timer
+}
+
+type priorityBalancer struct {
+	logger                   *grpclog.PrefixLogger
+	cc                       balancer.ClientConn
+	bg                       *balancergroup.BalancerGroup
+	done                     *grpcsync.Event
+	childBalancerStateUpdate *buffer.Unbounded[any]
+
+	mu         sync.Mutex
+	childInUse string
+	// priorities is a list of child names from higher to lower priority.
+	priorities []string
+	// children is a map from child name to sub-balancers.
+	children map[string]*childBalancer
+
+	// Set during UpdateClientConnState when calling into sub-balancers.
+	// Prevents child updates from recomputing the active priority or sending
+	// an update of the aggregated picker to the parent.  Cleared after all
+	// sub-balancers have finished UpdateClientConnState, after which
+	// syncPriority is called manually.
+	inhibitPickerUpdates bool
+}
+
+func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	if b.logger.V(2) {
+		b.logger.Infof("Received an update with balancer config: %+v", pretty.ToJSON(s.BalancerConfig))
+	}
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+	endpointsSplit := hierarchy.Group(s.ResolverState.Endpoints)
+
+	b.mu.Lock()
+	// Create and remove children, since we know all children from the config
+	// are used by some priority.
+	for name, newSubConfig := range newConfig.Children {
+		bb := balancer.Get(newSubConfig.Config.Name)
+		if bb == nil {
+			b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name)
+			continue
+		}
+
+		currentChild, ok := b.children[name]
+		if !ok {
+			// This is a new child, add it to the children list. But note that
+			// the balancer isn't built, because this child can be a low
+			// priority. If necessary, it will be built when syncing priorities.
+			cb := newChildBalancer(name, b, bb.Name(), b.cc)
+			cb.updateConfig(newSubConfig, resolver.State{
+				Endpoints:     endpointsSplit[name],
+				ServiceConfig: s.ResolverState.ServiceConfig,
+				Attributes:    s.ResolverState.Attributes,
+			})
+			b.children[name] = cb
+			continue
+		}
+
+		// This is not a new child. But the config/addresses could change.
+
+		// The balancing policy name is changed, close the old child. But don't
+		// rebuild, rebuild will happen when syncing priorities.
+		if currentChild.balancerName != bb.Name() {
+			currentChild.stop(true)
+			currentChild.updateBalancerName(bb.Name())
+		}
+
+		// Update config and address, but note that this doesn't send the
+		// updates to non-started child balancers (the child balancer might not
+		// be built, if it's a low priority).
+		currentChild.updateConfig(newSubConfig, resolver.State{
+			Endpoints:     endpointsSplit[name],
+			ServiceConfig: s.ResolverState.ServiceConfig,
+			Attributes:    s.ResolverState.Attributes,
+		})
+	}
+	// Cleanup resources used by children removed from the config.
+	for name, oldChild := range b.children {
+		if _, ok := newConfig.Children[name]; !ok {
+			oldChild.stop(!envconfig.EnablePriorityLBChildPolicyCache)
+			delete(b.children, name)
+		}
+	}
+
+	// Update priorities and handle priority changes.
+	b.priorities = newConfig.Priorities
+
+	// Everything was removed by the update.
+	if len(b.priorities) == 0 {
+		b.childInUse = ""
+		b.cc.UpdateState(balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(ErrAllPrioritiesRemoved),
+		})
+		b.mu.Unlock()
+		return nil
+	}
+
+	// This will sync the states of all children to the new updated
+	// priorities. Includes starting/stopping child balancers when necessary.
+	// Block picker updates until all children have had a chance to call
+	// UpdateState to prevent races where, e.g., the active priority reports
+	// transient failure but a higher priority may have reported something that
+	// made it active, and if the transient failure update is handled first,
+	// RPCs could fail.
+	b.inhibitPickerUpdates = true
+	// Add an item to queue to notify us when the current items in the queue
+	// are done and syncPriority has been called.
+	done := make(chan struct{})
+	b.childBalancerStateUpdate.Put(resumePickerUpdates{done: done})
+	b.mu.Unlock()
+	<-done
+
+	return nil
+}
+
+func (b *priorityBalancer) ResolverError(err error) {
+	if b.logger.V(2) {
+		b.logger.Infof("Received error from the resolver: %v", err)
+	}
+	b.bg.ResolverError(err)
+}
+
+func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *priorityBalancer) Close() {
+	b.bg.Close()
+	b.childBalancerStateUpdate.Close()
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.done.Fire()
+	// Clear states of the current child in use, so if there's a race in picker
+	// update, it will be dropped.
+	b.childInUse = ""
+	// Stop the child policies, this is necessary to stop the init timers in the
+	// children.
+	for _, child := range b.children {
+		child.stop(true)
+	}
+}
+
+func (b *priorityBalancer) ExitIdle() {
+	b.bg.ExitIdle()
+}
+
+// UpdateState implements balancergroup.BalancerStateAggregator interface. The
+// balancer group sends new connectivity state and picker here.
+func (b *priorityBalancer) UpdateState(childName string, state balancer.State) {
+	b.childBalancerStateUpdate.Put(childBalancerState{
+		name: childName,
+		s:    state,
+	})
+}
+
+type childBalancerState struct {
+	name string
+	s    balancer.State
+}
+
+type resumePickerUpdates struct {
+	done chan struct{}
+}
+
+// run handles child update in a separate goroutine, so if the child sends
+// updates inline (when called by parent), it won't cause deadlocks (by trying
+// to hold the same mutex).
+func (b *priorityBalancer) run() {
+	for {
+		select {
+		case u, ok := <-b.childBalancerStateUpdate.Get():
+			if !ok {
+				return
+			}
+			b.childBalancerStateUpdate.Load()
+			// Needs to handle state update in a goroutine, because each state
+			// update needs to start/close child policy, could result in
+			// deadlock.
+			b.mu.Lock()
+			if b.done.HasFired() {
+				b.mu.Unlock()
+				return
+			}
+			switch s := u.(type) {
+			case childBalancerState:
+				b.handleChildStateUpdate(s.name, s.s)
+			case resumePickerUpdates:
+				b.inhibitPickerUpdates = false
+				b.syncPriority(b.childInUse)
+				close(s.done)
+			}
+			b.mu.Unlock()
+		case <-b.done.Done():
+			return
+		}
+	}
+}

internal/xds/balancer/priority/balancer_child.go

@@ -0,0 +1,186 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+var timeAfterFunc = time.AfterFunc
+
+type childBalancer struct {
+	name         string
+	parent       *priorityBalancer
+	parentCC     balancer.ClientConn
+	balancerName string
+	cc           *ignoreResolveNowClientConn
+
+	ignoreReresolutionRequests bool
+	config                     serviceconfig.LoadBalancingConfig
+	rState                     resolver.State
+
+	started bool
+	// This is set when the child reports TransientFailure, and unset when it
+	// reports Ready or Idle. It is used to decide whether the failover timer
+	// should start when the child is transitioning into Connecting. The timer
+	// will be restarted if the child has not reported TF more recently than it
+	// reported Ready or Idle.
+	reportedTF bool
+	// The latest state the child balancer provided.
+	state balancer.State
+	// The timer to give a priority some time to connect. And if the priority
+	// doesn't go into Ready/Failure, the next priority will be started.
+	initTimer *timerWrapper
+}
+
+// newChildBalancer creates a child balancer place holder, but doesn't
+// build/start the child balancer.
+func newChildBalancer(name string, parent *priorityBalancer, balancerName string, cc balancer.ClientConn) *childBalancer {
+	return &childBalancer{
+		name:         name,
+		parent:       parent,
+		parentCC:     cc,
+		balancerName: balancerName,
+		cc:           newIgnoreResolveNowClientConn(cc, false),
+		started:      false,
+		// Start with the connecting state and picker with re-pick error, so
+		// that when a priority switch causes this child picked before it's
+		// balancing policy is created, a re-pick will happen.
+		state: balancer.State{
+			ConnectivityState: connectivity.Connecting,
+			Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+		},
+	}
+}
+
+// updateBalancerName updates balancer name for the child, but doesn't build a
+// new one. The parent priority LB always closes the child policy before
+// updating the balancer name, and the new balancer is built when it gets added
+// to the balancergroup as part of start().
+func (cb *childBalancer) updateBalancerName(balancerName string) {
+	cb.balancerName = balancerName
+	cb.cc = newIgnoreResolveNowClientConn(cb.parentCC, cb.ignoreReresolutionRequests)
+}
+
+// updateConfig sets childBalancer's config and state, but doesn't send update to
+// the child balancer unless it is started.
+func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) {
+	cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests
+	cb.config = child.Config.Config
+	cb.rState = rState
+	if cb.started {
+		cb.sendUpdate()
+	}
+}
+
+// start builds the child balancer if it's not already started.
+//
+// It doesn't do it directly. It asks the balancer group to build it.
+func (cb *childBalancer) start() {
+	if cb.started {
+		return
+	}
+	cb.started = true
+	cb.parent.bg.AddWithClientConn(cb.name, cb.balancerName, cb.cc)
+	cb.startInitTimer()
+	cb.sendUpdate()
+}
+
+// sendUpdate sends the addresses and config to the child balancer.
+func (cb *childBalancer) sendUpdate() {
+	cb.cc.updateIgnoreResolveNow(cb.ignoreReresolutionRequests)
+	// TODO: return and aggregate the returned error in the parent.
+	err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{
+		ResolverState:  cb.rState,
+		BalancerConfig: cb.config,
+	})
+	// Report TF if update to the child fails.
+	if err != nil {
+		cb.parent.logger.Warningf("Failed to update state for child policy %q: %v", cb.name, err)
+		cb.reportedTF = true
+		cb.state = balancer.State{
+			ConnectivityState: connectivity.TransientFailure,
+			Picker:            base.NewErrPicker(err),
+		}
+		cb.parent.handleChildStateUpdate(cb.name, cb.state)
+	}
+}
+
+// stop stops the child balancer and resets the state.
+//
+// It doesn't do it directly. It asks the balancer group to remove it.
+//
+// Note that the underlying balancer group could keep the child in a cache.
+func (cb *childBalancer) stop(immediate bool) {
+	if !cb.started {
+		return
+	}
+	cb.stopInitTimer()
+	if immediate {
+		cb.parent.bg.RemoveImmediately(cb.name)
+	} else {
+		cb.parent.bg.Remove(cb.name)
+	}
+	cb.started = false
+	cb.state = balancer.State{
+		ConnectivityState: connectivity.Connecting,
+		Picker:            base.NewErrPicker(balancer.ErrNoSubConnAvailable),
+	}
+	// Clear child.reportedTF, so that if this child is started later, it will
+	// be given time to connect.
+	cb.reportedTF = false
+}
+
+func (cb *childBalancer) startInitTimer() {
+	if cb.initTimer != nil {
+		return
+	}
+	// Need this local variable to capture timerW in the AfterFunc closure
+	// to check the stopped boolean.
+	timerW := &timerWrapper{}
+	cb.initTimer = timerW
+	timerW.timer = timeAfterFunc(DefaultPriorityInitTimeout, func() {
+		cb.parent.mu.Lock()
+		defer cb.parent.mu.Unlock()
+		if timerW.stopped {
+			return
+		}
+		cb.initTimer = nil
+		// Re-sync the priority. This will switch to the next priority if
+		// there's any. Note that it's important sync() is called after setting
+		// initTimer to nil.
+		cb.parent.syncPriority("")
+	})
+}
+
+func (cb *childBalancer) stopInitTimer() {
+	timerW := cb.initTimer
+	if timerW == nil {
+		return
+	}
+	cb.initTimer = nil
+	timerW.stopped = true
+	timerW.timer.Stop()
+}

internal/xds/balancer/priority/balancer_priority.go

@@ -0,0 +1,223 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"errors"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/connectivity"
+)
+
+var (
+	// ErrAllPrioritiesRemoved is returned by the picker when there's no priority available.
+	ErrAllPrioritiesRemoved = errors.New("no priority is provided, all priorities are removed")
+	// DefaultPriorityInitTimeout is the timeout after which if a priority is
+	// not READY, the next will be started. It's exported to be overridden by
+	// tests.
+	DefaultPriorityInitTimeout = 10 * time.Second
+)
+
+// syncPriority handles priority after a config update or a child balancer
+// connectivity state update. It makes sure the balancer state (started or not)
+// is in sync with the priorities (even in tricky cases where a child is moved
+// from a priority to another).
+//
+// It's guaranteed that after this function returns:
+//
+//	If some child is READY, it is childInUse, and all lower priorities are
+//	closed.
+//
+//	If some child is newly started(in Connecting for the first time), it is
+//	childInUse, and all lower priorities are closed.
+//
+//	Otherwise, the lowest priority is childInUse (none of the children is
+//	ready, and the overall state is not ready).
+//
+// Steps:
+//
+//	If all priorities were deleted, unset childInUse (to an empty string), and
+//	set parent ClientConn to TransientFailure
+//
+//	Otherwise, Scan all children from p0, and check balancer stats:
+//
+//	  For any of the following cases:
+//
+//	    If balancer is not started (not built), this is either a new child with
+//	    high priority, or a new builder for an existing child.
+//
+//	    If balancer is Connecting and has non-nil initTimer (meaning it
+//	    transitioned from Ready or Idle to connecting, not from TF, so we
+//	    should give it init-time to connect).
+//
+//	    If balancer is READY or IDLE
+//
+//	    If this is the lowest priority
+//
+//	 do the following:
+//
+//	    if this is not the old childInUse, override picker so old picker is no
+//	    longer used.
+//
+//	    switch to it (because all higher priorities are neither new or Ready)
+//
+//	    forward the new addresses and config
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) syncPriority(childUpdating string) {
+	if b.inhibitPickerUpdates {
+		if b.logger.V(2) {
+			b.logger.Infof("Skipping update from child policy %q", childUpdating)
+		}
+		return
+	}
+	for p, name := range b.priorities {
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Warningf("Priority name %q is not found in list of child policies", name)
+			continue
+		}
+
+		if !child.started ||
+			child.state.ConnectivityState == connectivity.Ready ||
+			child.state.ConnectivityState == connectivity.Idle ||
+			(child.state.ConnectivityState == connectivity.Connecting && child.initTimer != nil) ||
+			p == len(b.priorities)-1 {
+			if b.childInUse != child.name || child.name == childUpdating {
+				if b.logger.V(2) {
+					b.logger.Infof("childInUse, childUpdating: %q, %q", b.childInUse, child.name)
+				}
+				// If we switch children or the child in use just updated its
+				// picker, push the child's picker to the parent.
+				b.cc.UpdateState(child.state)
+			}
+			if b.logger.V(2) {
+				b.logger.Infof("Switching to (%q, %v) in syncPriority", child.name, p)
+			}
+			b.switchToChild(child, p)
+			break
+		}
+	}
+}
+
+// Stop priorities [p+1, lowest].
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) {
+	for i := p + 1; i < len(b.priorities); i++ {
+		name := b.priorities[i]
+		child, ok := b.children[name]
+		if !ok {
+			b.logger.Warningf("Priority name %q is not found in list of child policies", name)
+			continue
+		}
+		child.stop(false)
+	}
+}
+
+// switchToChild does the following:
+// - stop all child with lower priorities
+// - if childInUse is not this child
+//   - set childInUse to this child
+//   - if this child is not started, start it
+//
+// Note that it does NOT send the current child state (picker) to the parent
+// ClientConn. The caller needs to send it if necessary.
+//
+// this can be called when
+// 1. first update, start p0
+// 2. an update moves a READY child from a lower priority to higher
+// 2. a different builder is updated for this child
+// 3. a high priority goes Failure, start next
+// 4. a high priority init timeout, start next
+//
+// Caller must hold b.mu.
+func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) {
+	// Stop lower priorities even if childInUse is same as this child. It's
+	// possible this child was moved from a priority to another.
+	b.stopSubBalancersLowerThanPriority(priority)
+
+	// If this child is already in use, do nothing.
+	//
+	// This can happen:
+	// - all priorities are not READY, an config update always triggers switch
+	// to the lowest. In this case, the lowest child could still be connecting,
+	// so we don't stop the init timer.
+	// - a high priority is READY, an config update always triggers switch to
+	// it.
+	if b.childInUse == child.name && child.started {
+		return
+	}
+	b.childInUse = child.name
+
+	if !child.started {
+		child.start()
+	}
+}
+
+// handleChildStateUpdate start/close priorities based on the connectivity
+// state.
+func (b *priorityBalancer) handleChildStateUpdate(childName string, newState balancer.State) {
+	// Update state in child. The updated picker will be sent to parent later if
+	// necessary.
+	child, ok := b.children[childName]
+	if !ok {
+		b.logger.Warningf("Child policy not found for %q", childName)
+		return
+	}
+	if !child.started {
+		b.logger.Warningf("Ignoring update from child policy %q which is not in started state: %+v", childName, newState)
+		return
+	}
+	oldState := child.state
+	child.state = newState
+
+	// We start/stop the init timer of this child based on the new connectivity
+	// state. syncPriority() later will need the init timer (to check if it's
+	// nil or not) to decide which child to switch to.
+	switch newState.ConnectivityState {
+	case connectivity.Ready, connectivity.Idle:
+		child.reportedTF = false
+		child.stopInitTimer()
+	case connectivity.TransientFailure:
+		child.reportedTF = true
+		child.stopInitTimer()
+	case connectivity.Connecting:
+		// The init timer is created when the child is created and is reset when
+		// it reports Ready or Idle. Most child policies start off in
+		// Connecting, but ring_hash starts off in Idle and moves to Connecting
+		// when a request comes in. To support such cases, we restart the init
+		// timer when we see Connecting, but only if the child has not reported
+		// TransientFailure more recently than it reported Ready or Idle. See
+		// gRFC A42 for details on why ring_hash is special and what provisions
+		// are required to make it work as a child of the priority LB policy.
+		//
+		// We don't want to restart the timer if the child was already in
+		// Connecting, because we want failover to happen once the timer elapses
+		// even when the child is still in Connecting.
+		if !child.reportedTF && oldState.ConnectivityState != connectivity.Connecting {
+			child.startInitTimer()
+		}
+	default:
+		// New state is Shutdown, should never happen. Don't forward.
+	}
+
+	child.parent.syncPriority(childName)
+}

internal/xds/balancer/priority/config.go

@@ -0,0 +1,67 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"encoding/json"
+	"fmt"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Child is a child of priority balancer.
+type Child struct {
+	Config                     *internalserviceconfig.BalancerConfig `json:"config,omitempty"`
+	IgnoreReresolutionRequests bool                                  `json:"ignoreReresolutionRequests,omitempty"`
+}
+
+// LBConfig represents priority balancer's config.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	// Children is a map from the child balancer names to their configs. Child
+	// names can be found in field Priorities.
+	Children map[string]*Child `json:"children,omitempty"`
+	// Priorities is a list of child balancer names. They are sorted from
+	// highest priority to low. The type/config for each child can be found in
+	// field Children, with the balancer name as the key.
+	Priorities []string `json:"priorities,omitempty"`
+}
+
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+
+	prioritiesSet := make(map[string]bool)
+	for _, name := range cfg.Priorities {
+		if _, ok := cfg.Children[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children)
+		}
+		prioritiesSet[name] = true
+	}
+	for name := range cfg.Children {
+		if _, ok := prioritiesSet[name]; !ok {
+			return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities)
+		}
+	}
+	return &cfg, nil
+}

internal/xds/balancer/priority/config_test.go

@@ -0,0 +1,108 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer/roundrobin"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+func TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *LBConfig
+		wantErr bool
+	}{
+		{
+			name: "child not found",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "child not used",
+			js: `{
+  "priorities": ["child-1", "child-2"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}]},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			wantErr: true,
+		},
+		{
+			name: "good",
+			js: `{
+  "priorities": ["child-1", "child-2", "child-3"],
+  "children": {
+    "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true},
+    "child-2": {"config": [{"round_robin":{}}]},
+    "child-3": {"config": [{"round_robin":{}}]}
+  }
+}
+			`,
+			want: &LBConfig{
+				Children: map[string]*Child{
+					"child-1": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+						IgnoreReresolutionRequests: true,
+					},
+					"child-2": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+					"child-3": {
+						Config: &internalserviceconfig.BalancerConfig{
+							Name: roundrobin.Name,
+						},
+					},
+				},
+				Priorities: []string{"child-1", "child-2", "child-3"},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if diff := cmp.Diff(got, tt.want); diff != "" {
+				t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff)
+			}
+		})
+	}
+}

internal/xds/balancer/priority/ignore_resolve_now.go

@@ -0,0 +1,50 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"sync/atomic"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/resolver"
+)
+
+// ignoreResolveNowClientConn wraps a balancer.ClientConn and overrides the
+// ResolveNow() method to ignore those calls if the ignoreResolveNow bit is set.
+type ignoreResolveNowClientConn struct {
+	balancer.ClientConn
+	ignoreResolveNow atomic.Bool
+}
+
+func newIgnoreResolveNowClientConn(cc balancer.ClientConn, ignore bool) *ignoreResolveNowClientConn {
+	ret := &ignoreResolveNowClientConn{ClientConn: cc}
+	ret.updateIgnoreResolveNow(ignore)
+	return ret
+}
+
+func (i *ignoreResolveNowClientConn) updateIgnoreResolveNow(b bool) {
+	i.ignoreResolveNow.Store(b)
+}
+
+func (i *ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) {
+	if i.ignoreResolveNow.Load() {
+		return
+	}
+	i.ClientConn.ResolveNow(o)
+}

internal/xds/balancer/priority/ignore_resolve_now_test.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+)
+
+func (s) TestIgnoreResolveNowClientConn(t *testing.T) {
+	cc := testutils.NewBalancerClientConn(t)
+	ignoreCC := newIgnoreResolveNowClientConn(cc, false)
+
+	// Call ResolveNow() on the CC, it should be forwarded.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for ResolveNow()")
+	}
+
+	// Update ignoreResolveNow to true, call ResolveNow() on the CC, they should
+	// all be ignored.
+	ignoreCC.updateIgnoreResolveNow(true)
+	for i := 0; i < 5; i++ {
+		ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	}
+	select {
+	case <-cc.ResolveNowCh:
+		t.Fatalf("got unexpected ResolveNow() call")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Update ignoreResolveNow to false, new ResolveNow() calls should be
+	// forwarded.
+	ignoreCC.updateIgnoreResolveNow(false)
+	ignoreCC.ResolveNow(resolver.ResolveNowOptions{})
+	select {
+	case <-cc.ResolveNowCh:
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for ResolveNow()")
+	}
+}

internal/xds/balancer/priority/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package priority
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[priority-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/wrrlocality/balancer.go

@@ -0,0 +1,201 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package wrrlocality provides an implementation of the wrr locality LB policy,
+// as defined in [A52 - xDS Custom LB Policies].
+//
+// [A52 - xDS Custom LB Policies]: https://github.com/grpc/proposal/blob/master/A52-xds-custom-lb-policies.md
+package wrrlocality
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/weightedtarget"
+	"google.golang.org/grpc/internal/grpclog"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// Name is the name of wrr_locality balancer.
+const Name = "xds_wrr_locality_experimental"
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Name() string {
+	return Name
+}
+
+// LBConfig is the config for the wrr locality balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+	// ChildPolicy is the config for the child policy.
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+// To plumb in a different child in tests.
+var weightedTargetName = weightedtarget.Name
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	builder := balancer.Get(weightedTargetName)
+	if builder == nil {
+		// Shouldn't happen, registered through imported weighted target,
+		// defensive programming.
+		return nil
+	}
+
+	// Doesn't need to intercept any balancer.ClientConn operations; pass
+	// through by just giving cc to child balancer.
+	wtb := builder.Build(cc, bOpts)
+	if wtb == nil {
+		// shouldn't happen, defensive programming.
+		return nil
+	}
+	wtbCfgParser, ok := builder.(balancer.ConfigParser)
+	if !ok {
+		// Shouldn't happen, imported weighted target builder has this method.
+		return nil
+	}
+	wrrL := &wrrLocalityBalancer{
+		child:       wtb,
+		childParser: wtbCfgParser,
+	}
+
+	wrrL.logger = prefixLogger(wrrL)
+	wrrL.logger.Infof("Created")
+	return wrrL
+}
+
+func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	var lbCfg *LBConfig
+	if err := json.Unmarshal(s, &lbCfg); err != nil {
+		return nil, fmt.Errorf("xds_wrr_locality: invalid LBConfig: %s, error: %v", string(s), err)
+	}
+	if lbCfg == nil || lbCfg.ChildPolicy == nil {
+		return nil, errors.New("xds_wrr_locality: invalid LBConfig: child policy field must be set")
+	}
+	return lbCfg, nil
+}
+
+type attributeKey struct{}
+
+// Equal allows the values to be compared by Attributes.Equal.
+func (a AddrInfo) Equal(o any) bool {
+	oa, ok := o.(AddrInfo)
+	return ok && oa.LocalityWeight == a.LocalityWeight
+}
+
+// AddrInfo is the locality weight of the locality an address is a part of.
+type AddrInfo struct {
+	LocalityWeight uint32
+}
+
+// SetAddrInfo returns a copy of endpoint in which the Attributes field is
+// updated with AddrInfo.
+func SetAddrInfo(endpoint resolver.Endpoint, addrInfo AddrInfo) resolver.Endpoint {
+	endpoint.Attributes = endpoint.Attributes.WithValue(attributeKey{}, addrInfo)
+	return endpoint
+}
+
+func (a AddrInfo) String() string {
+	return fmt.Sprintf("Locality Weight: %d", a.LocalityWeight)
+}
+
+// getAddrInfo returns the AddrInfo stored in the Attributes field of
+// ep. Returns false if no AddrInfo found.
+func getAddrInfo(ep resolver.Endpoint) (AddrInfo, bool) {
+	v := ep.Attributes.Value(attributeKey{})
+	ai, ok := v.(AddrInfo)
+	return ai, ok
+}
+
+// wrrLocalityBalancer wraps a weighted target balancer, and builds
+// configuration for the weighted target once it receives configuration
+// specifying the weighted target child balancer and locality weight
+// information.
+type wrrLocalityBalancer struct {
+	// child will be a weighted target balancer, and will be built it at
+	// wrrLocalityBalancer build time. Other than preparing configuration, other
+	// balancer operations are simply pass through.
+	child balancer.Balancer
+
+	childParser balancer.ConfigParser
+
+	logger *grpclog.PrefixLogger
+}
+
+func (b *wrrLocalityBalancer) ExitIdle() {
+	b.child.ExitIdle()
+}
+
+func (b *wrrLocalityBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	lbCfg, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		b.logger.Errorf("Received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig)
+		return balancer.ErrBadResolverState
+	}
+
+	weightedTargets := make(map[string]weightedtarget.Target)
+	for _, ep := range s.ResolverState.Endpoints {
+		// This get of LocalityID could potentially return a zero value. This
+		// shouldn't happen though (this attribute that is set actually gets
+		// used to build localities in the first place), and thus don't error
+		// out, and just build a weighted target with undefined behavior.
+		locality := xdsinternal.LocalityString(xdsinternal.LocalityIDFromEndpoint(ep))
+		ai, ok := getAddrInfo(ep)
+		if !ok {
+			return fmt.Errorf("xds_wrr_locality: missing locality weight information in endpoint %q", ep)
+		}
+		weightedTargets[locality] = weightedtarget.Target{Weight: ai.LocalityWeight, ChildPolicy: lbCfg.ChildPolicy}
+	}
+	wtCfg := &weightedtarget.LBConfig{Targets: weightedTargets}
+	wtCfgJSON, err := json.Marshal(wtCfg)
+	if err != nil {
+		// Shouldn't happen.
+		return fmt.Errorf("xds_wrr_locality: error marshalling prepared config: %v", wtCfg)
+	}
+	var sc serviceconfig.LoadBalancingConfig
+	if sc, err = b.childParser.ParseConfig(wtCfgJSON); err != nil {
+		return fmt.Errorf("xds_wrr_locality: config generated %v is invalid: %v", wtCfgJSON, err)
+	}
+
+	return b.child.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  s.ResolverState,
+		BalancerConfig: sc,
+	})
+}
+
+func (b *wrrLocalityBalancer) ResolverError(err error) {
+	b.child.ResolverError(err)
+}
+
+func (b *wrrLocalityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)
+}
+
+func (b *wrrLocalityBalancer) Close() {
+	b.child.Close()
+}

internal/xds/balancer/wrrlocality/balancer_test.go

@@ -0,0 +1,250 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package wrrlocality
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/balancer/weightedtarget"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	defaultTestTimeout = 5 * time.Second
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestParseConfig(t *testing.T) {
+	const errParseConfigName = "errParseConfigBalancer"
+	stub.Register(errParseConfigName, stub.BalancerFuncs{
+		ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			return nil, errors.New("some error")
+		},
+	})
+
+	parser := bb{}
+	tests := []struct {
+		name    string
+		input   string
+		wantCfg serviceconfig.LoadBalancingConfig
+		wantErr string
+	}{
+		{
+			name:  "happy-case-round robin-child",
+			input: `{"childPolicy": [{"round_robin": {}}]}`,
+			wantCfg: &LBConfig{
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: roundrobin.Name,
+				},
+			},
+		},
+		{
+			name:    "invalid-json",
+			input:   "{{invalidjson{{",
+			wantErr: "invalid character",
+		},
+
+		{
+			name:    "child-policy-field-isn't-set",
+			input:   `{}`,
+			wantErr: "child policy field must be set",
+		},
+		{
+			name:    "child-policy-type-is-empty",
+			input:   `{"childPolicy": []}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in []",
+		},
+		{
+			name:    "child-policy-empty-config",
+			input:   `{"childPolicy": [{"": {}}]}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in []",
+		},
+		{
+			name:    "child-policy-type-isn't-registered",
+			input:   `{"childPolicy": [{"doesNotExistBalancer": {"cluster": "test_cluster"}}]}`,
+			wantErr: "invalid loadBalancingConfig: no supported policies found in [doesNotExistBalancer]",
+		},
+		{
+			name:    "child-policy-config-is-invalid",
+			input:   `{"childPolicy": [{"errParseConfigBalancer": {"cluster": "test_cluster"}}]}`,
+			wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input))
+			// Substring match makes this very tightly coupled to the
+			// internalserviceconfig.BalancerConfig error strings. However, it
+			// is important to distinguish the different types of error messages
+			// possible as the parser has a few defined buckets of ways it can
+			// error out.
+			if (gotErr != nil) != (test.wantErr != "") {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) {
+				t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr)
+			}
+			if test.wantErr != "" {
+				return
+			}
+			if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" {
+				t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff)
+			}
+		})
+	}
+}
+
+// TestUpdateClientConnState tests the UpdateClientConnState method of the
+// wrr_locality_experimental balancer. This UpdateClientConn operation should
+// take the localities and their weights in the addresses passed in, alongside
+// the endpoint picking policy defined in the Balancer Config and construct a
+// weighted target configuration corresponding to these inputs.
+func (s) TestUpdateClientConnState(t *testing.T) {
+	// Configure the stub balancer defined below as the child policy of
+	// wrrLocalityBalancer.
+	cfgCh := testutils.NewChannel()
+	oldWeightedTargetName := weightedTargetName
+	defer func() {
+		weightedTargetName = oldWeightedTargetName
+	}()
+	weightedTargetName = "fake_weighted_target"
+	stub.Register("fake_weighted_target", stub.BalancerFuncs{
+		ParseConfig: func(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			var cfg weightedtarget.LBConfig
+			if err := json.Unmarshal(c, &cfg); err != nil {
+				return nil, err
+			}
+			return &cfg, nil
+		},
+		UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error {
+			wtCfg, ok := ccs.BalancerConfig.(*weightedtarget.LBConfig)
+			if !ok {
+				return errors.New("child received config that was not a weighted target config")
+			}
+			defer cfgCh.Send(wtCfg)
+			return nil
+		},
+	})
+
+	builder := balancer.Get(Name)
+	if builder == nil {
+		t.Fatalf("balancer.Get(%q) returned nil", Name)
+	}
+	tcc := testutils.NewBalancerClientConn(t)
+	bal := builder.Build(tcc, balancer.BuildOptions{})
+	defer bal.Close()
+	wrrL := bal.(*wrrLocalityBalancer)
+
+	// Create the addresses with two localities with certain locality weights.
+	// This represents what addresses the wrr_locality balancer will receive in
+	// UpdateClientConnState.
+	ep1 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-1"}}}
+	ep1 = xdsinternal.SetLocalityIDInEndpoint(ep1, clients.Locality{
+		Region:  "region-1",
+		Zone:    "zone-1",
+		SubZone: "subzone-1",
+	})
+	ep1 = SetAddrInfo(ep1, AddrInfo{LocalityWeight: 2})
+
+	ep2 := resolver.Endpoint{Addresses: []resolver.Address{{Addr: "locality-2"}}}
+	ep2 = xdsinternal.SetLocalityIDInEndpoint(ep2, clients.Locality{
+		Region:  "region-2",
+		Zone:    "zone-2",
+		SubZone: "subzone-2",
+	})
+	ep2 = SetAddrInfo(ep2, AddrInfo{LocalityWeight: 1})
+
+	eps := []resolver.Endpoint{ep1, ep2}
+
+	err := wrrL.UpdateClientConnState(balancer.ClientConnState{
+		BalancerConfig: &LBConfig{
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: "round_robin",
+			},
+		},
+		ResolverState: resolver.State{
+			Endpoints: eps,
+		},
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Note that these inline strings declared as the key in Targets built from
+	// Locality ID are not exactly what is shown in the example in the gRFC.
+	// However, this is an implementation detail that does not affect
+	// correctness (confirmed with Java team). The important thing is to get
+	// those three pieces of information region, zone, and subzone down to the
+	// child layer.
+	wantWtCfg := &weightedtarget.LBConfig{
+		Targets: map[string]weightedtarget.Target{
+			"{region=\"region-1\", zone=\"zone-1\", sub_zone=\"subzone-1\"}": {
+				Weight: 2,
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: "round_robin",
+				},
+			},
+			"{region=\"region-2\", zone=\"zone-2\", sub_zone=\"subzone-2\"}": {
+				Weight: 1,
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name: "round_robin",
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cfg, err := cfgCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("No signal received from UpdateClientConnState() on the child: %v", err)
+	}
+
+	gotWtCfg, ok := cfg.(*weightedtarget.LBConfig)
+	if !ok {
+		// Shouldn't happen - only sends a config on this channel.
+		t.Fatalf("Unexpected config type: %T", gotWtCfg)
+	}
+
+	if diff := cmp.Diff(gotWtCfg, wantWtCfg); diff != "" {
+		t.Fatalf("Child received unexpected config, diff (-got, +want): %v", diff)
+	}
+}

internal/xds/balancer/wrrlocality/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package wrrlocality
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[wrrlocality-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *wrrLocalityBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/bootstrap/jwtcreds/call_creds.go

@@ -0,0 +1,54 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package jwtcreds implements JWT CallCredentials for XDS, configured via xDS
+// Bootstrap File. For more details, see gRFC A97:
+// https://github.com/grpc/proposal/blob/master/A97-xds-jwt-call-creds.md
+package jwtcreds
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/jwt"
+)
+
+// NewCallCredentials returns a new JWT token based call credentials. The input
+// config must match the structure specified in gRFC A97.
+//
+// The caller is expected to invoke the cancel function when they are done using
+// the returned call creds. This cancel function is idempotent.
+func NewCallCredentials(configJSON json.RawMessage) (c credentials.PerRPCCredentials, cancel func(), err error) {
+	var cfg struct {
+		JWTTokenFile string `json:"jwt_token_file"`
+	}
+	emptyFn := func() {}
+
+	if err := json.Unmarshal(configJSON, &cfg); err != nil {
+		return nil, emptyFn, fmt.Errorf("failed to unmarshal JWT call credentials config: %v", err)
+	}
+	if cfg.JWTTokenFile == "" {
+		return nil, emptyFn, fmt.Errorf("jwt_token_file is required in JWT call credentials config")
+	}
+	callCreds, err := jwt.NewTokenFileCallCredentials(cfg.JWTTokenFile)
+	if err != nil {
+		return nil, emptyFn, fmt.Errorf("failed to create JWT call credentials: %v", err)
+	}
+	return callCreds, emptyFn, nil
+}

internal/xds/bootstrap/jwtcreds/call_creds_test.go

@@ -0,0 +1,168 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package jwtcreds
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestNewCallCredentialsWithInvalidConfig(t *testing.T) {
+	tests := []struct {
+		name   string
+		config string
+	}{
+		{
+			name:   "empty_file",
+			config: `""`,
+		},
+		{
+			name:   "empty_config",
+			config: `{}`,
+		},
+		{
+			name:   "empty_path",
+			config: `{"jwt_token_file": ""}`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			callCreds, cleanup, err := NewCallCredentials(json.RawMessage(tt.config))
+			if err == nil {
+				t.Fatalf("NewCallCredentials(%s): got nil, want error", tt.config)
+			}
+			if callCreds != nil {
+				t.Errorf("NewCallCredentials(%s): returned non-nil call credentials", tt.config)
+			}
+			if cleanup == nil {
+				t.Errorf("NewCallCredentials(%s): returned nil cleanup function", tt.config)
+			}
+		})
+	}
+}
+
+func (s) TestNewCallCredentialsWithValidConfig(t *testing.T) {
+	token := createTestJWT(t)
+	tokenFile := writeTempFile(t, token)
+	config := `{"jwt_token_file": "` + tokenFile + `"}`
+
+	callCreds, cleanup, err := NewCallCredentials(json.RawMessage(config))
+	if err != nil {
+		t.Fatalf("NewCallCredentials(%s) failed: %v", config, err)
+	}
+	if callCreds == nil {
+		t.Fatalf("NewCallCredentials(%s): returned nil credentials", config)
+	}
+	if cleanup == nil {
+		t.Errorf("NewCallCredentials(%s): returned nil cleanup function", config)
+	} else {
+		defer cleanup()
+	}
+
+	// Test that call credentials get used.
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	ctx = credentials.NewContextWithRequestInfo(ctx, credentials.RequestInfo{
+		AuthInfo: &testAuthInfo{secLevel: credentials.PrivacyAndIntegrity},
+	})
+	metadata, err := callCreds.GetRequestMetadata(ctx)
+	if err != nil {
+		t.Fatalf("GetRequestMetadata failed: %v", err)
+	}
+	if len(metadata) == 0 {
+		t.Fatal("GetRequestMetadata: returned empty metadata")
+	}
+	authHeader, ok := metadata["authorization"]
+	if !ok {
+		t.Fatal("GetRequestMetadata: returned empty authorization header in metadata")
+	}
+	if !strings.HasPrefix(authHeader, "Bearer ") {
+		t.Errorf("GetRequestMetadata: Authorization header should start with 'Bearer ', got %q", authHeader)
+	}
+}
+
+func (s) TestCallCredentials_Cleanup(t *testing.T) {
+	token := createTestJWT(t)
+	tokenFile := writeTempFile(t, token)
+	config := `{"jwt_token_file": "` + tokenFile + `"}`
+	_, cleanup, err := NewCallCredentials(json.RawMessage(config))
+	if err != nil {
+		t.Fatalf("NewCallCredentials(%s) failed: %v", config, err)
+	}
+	if cleanup == nil {
+		t.Errorf("NewCallCredentials(%s): returned nil cleanup function", config)
+	}
+
+	// Cleanup should not panic. Multiple cleanup calls should be safe
+	cleanup()
+	cleanup()
+}
+
+// testAuthInfo implements credentials.AuthInfo for testing.
+type testAuthInfo struct {
+	secLevel credentials.SecurityLevel
+}
+
+func (t *testAuthInfo) AuthType() string {
+	return "test"
+}
+
+func (t *testAuthInfo) GetCommonAuthInfo() credentials.CommonAuthInfo {
+	return credentials.CommonAuthInfo{SecurityLevel: t.secLevel}
+}
+
+// createTestJWT creates a test JWT token for testing.
+func createTestJWT(t *testing.T) string {
+	t.Helper()
+
+	// Header: {"typ":"JWT","alg":"HS256"}
+	header := "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9"
+	// Claims: {"aud":"https://example.com","exp":future_timestamp}
+	claims := "eyJhdWQiOiJodHRwczovL2V4YW1wbGUuY29tIiwiZXhwIjoyMDAwMDAwMDAwfQ"
+	signature := "fake_signature_for_testing"
+
+	return header + "." + claims + "." + signature
+}
+
+func writeTempFile(t *testing.T, content string) string {
+	t.Helper()
+	tempDir := t.TempDir()
+	filePath := filepath.Join(tempDir, "jwt_token")
+	if err := os.WriteFile(filePath, []byte(content), 0600); err != nil {
+		t.Fatalf("Failed to write temp file: %v", err)
+	}
+	return filePath
+}

internal/xds/bootstrap/logging.go

@@ -0,0 +1,28 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package bootstrap
+
+import (
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-bootstrap] "
+
+var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix)

internal/xds/bootstrap/template.go

@@ -0,0 +1,47 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bootstrap
+
+import (
+	"net/url"
+	"strings"
+)
+
+// PopulateResourceTemplate populates the given template using the target
+// string. "%s", if exists in the template, will be replaced with target.
+//
+// If the template starts with "xdstp:", the replaced string will be %-encoded.
+// But note that "/" is not percent encoded.
+func PopulateResourceTemplate(template, target string) string {
+	if !strings.Contains(template, "%s") {
+		return template
+	}
+	if strings.HasPrefix(template, "xdstp:") {
+		target = percentEncode(target)
+	}
+	return strings.ReplaceAll(template, "%s", target)
+}
+
+// percentEncode percent encode t, except for "/". See the tests for examples.
+func percentEncode(t string) string {
+	segs := strings.Split(t, "/")
+	for i := range segs {
+		segs[i] = url.PathEscape(segs[i])
+	}
+	return strings.Join(segs, "/")
+}

internal/xds/bootstrap/template_test.go

@@ -0,0 +1,97 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bootstrap
+
+import "testing"
+
+func Test_percentEncode(t *testing.T) {
+	tests := []struct {
+		name   string
+		target string
+		want   string
+	}{
+		{
+			name:   "normal name",
+			target: "server.example.com",
+			want:   "server.example.com",
+		},
+		{
+			name:   "ipv4",
+			target: "0.0.0.0:8080",
+			want:   "0.0.0.0:8080",
+		},
+		{
+			name:   "ipv6",
+			target: "[::1]:8080",
+			want:   "%5B::1%5D:8080", // [ and ] are percent encoded.
+		},
+		{
+			name:   "/ should not be percent encoded",
+			target: "my/service/region",
+			want:   "my/service/region", // "/"s are kept.
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := percentEncode(tt.target); got != tt.want {
+				t.Errorf("percentEncode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestPopulateResourceTemplate(t *testing.T) {
+	tests := []struct {
+		name     string
+		template string
+		target   string
+		want     string
+	}{
+		{
+			name:     "no %s",
+			template: "/name/template",
+			target:   "[::1]:8080",
+			want:     "/name/template",
+		},
+		{
+			name:     "with %s, no xdstp: prefix, ipv6",
+			template: "/name/template/%s",
+			target:   "[::1]:8080",
+			want:     "/name/template/[::1]:8080",
+		},
+		{
+			name:     "with %s, with xdstp: prefix",
+			template: "xdstp://authority.com/%s",
+			target:   "0.0.0.0:8080",
+			want:     "xdstp://authority.com/0.0.0.0:8080",
+		},
+		{
+			name:     "with %s, with xdstp: prefix, and ipv6",
+			template: "xdstp://authority.com/%s",
+			target:   "[::1]:8080",
+			want:     "xdstp://authority.com/%5B::1%5D:8080",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := PopulateResourceTemplate(tt.template, tt.target); got != tt.want {
+				t.Errorf("PopulateResourceTemplate() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/xds/bootstrap/tlscreds/bundle.go

@@ -0,0 +1,194 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package tlscreds implements mTLS Credentials in xDS Bootstrap File.
+// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md.
+package tlscreds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/spiffe/go-spiffe/v2/bundle/spiffebundle"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
+	"google.golang.org/grpc/internal/credentials/spiffe"
+	"google.golang.org/grpc/internal/envconfig"
+)
+
+// bundle is an implementation of credentials.Bundle which implements mTLS
+// Credentials in xDS Bootstrap File.
+type bundle struct {
+	transportCredentials credentials.TransportCredentials
+}
+
+// NewBundle returns a credentials.Bundle which implements mTLS Credentials in xDS
+// Bootstrap File. It delegates certificate loading to a file_watcher provider
+// if either client certificates or server root CA is specified. The second
+// return value is a close func that should be called when the caller no longer
+// needs this bundle.
+// See gRFC A65: github.com/grpc/proposal/blob/master/A65-xds-mtls-creds-in-bootstrap.md
+func NewBundle(jd json.RawMessage) (credentials.Bundle, func(), error) {
+	cfg := &struct {
+		CertificateFile          string `json:"certificate_file"`
+		CACertificateFile        string `json:"ca_certificate_file"`
+		PrivateKeyFile           string `json:"private_key_file"`
+		SPIFFETrustBundleMapFile string `json:"spiffe_trust_bundle_map_file"`
+	}{}
+
+	if jd != nil {
+		if err := json.Unmarshal(jd, cfg); err != nil {
+			return nil, nil, fmt.Errorf("failed to unmarshal config: %v", err)
+		}
+	} // Else the config field is absent. Treat it as an empty config.
+
+	if !envconfig.XDSSPIFFEEnabled {
+		cfg.SPIFFETrustBundleMapFile = ""
+	}
+	if cfg.CACertificateFile == "" && cfg.CertificateFile == "" && cfg.PrivateKeyFile == "" && cfg.SPIFFETrustBundleMapFile == "" {
+		// We cannot use (and do not need) a file_watcher provider in this case,
+		// and can simply directly use the TLS transport credentials.
+		// Quoting A65:
+		//
+		// > The only difference between the file-watcher certificate provider
+		// > config and this one is that in the file-watcher certificate
+		// > provider, at least one of the "certificate_file" or
+		// > "ca_certificate_file" fields must be specified, whereas in this
+		// > configuration, it is acceptable to specify neither one.
+		// Further, with the introduction of SPIFFE Trust Map support, we also
+		// check for this value.
+		return &bundle{transportCredentials: credentials.NewTLS(&tls.Config{})}, func() {}, nil
+	}
+	// Otherwise we need to use a file_watcher provider to watch the CA,
+	// private and public keys.
+
+	// The pemfile plugin (file_watcher) currently ignores BuildOptions.
+	provider, err := certprovider.GetProvider(pemfile.PluginName, jd, certprovider.BuildOptions{})
+	if err != nil {
+		return nil, nil, err
+	}
+	return &bundle{
+		transportCredentials: &reloadingCreds{provider: provider},
+	}, sync.OnceFunc(func() { provider.Close() }), nil
+}
+
+func (t *bundle) TransportCredentials() credentials.TransportCredentials {
+	return t.transportCredentials
+}
+
+func (t *bundle) PerRPCCredentials() credentials.PerRPCCredentials {
+	// mTLS provides transport credentials only. There are no per-RPC
+	// credentials.
+	return nil
+}
+
+func (t *bundle) NewWithMode(string) (credentials.Bundle, error) {
+	// This bundle has a single mode which only uses TLS transport credentials,
+	// so there is no legitimate case where callers would call NewWithMode.
+	return nil, fmt.Errorf("xDS TLS credentials only support one mode")
+}
+
+// reloadingCreds is a credentials.TransportCredentials for client
+// side mTLS that reloads the server root CA certificate and the client
+// certificates from the provider on every client handshake. This is necessary
+// because the standard TLS credentials do not support reloading CA
+// certificates.
+type reloadingCreds struct {
+	provider certprovider.Provider
+}
+
+func (c *reloadingCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	km, err := c.provider.KeyMaterial(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	var config *tls.Config
+	if km.SPIFFEBundleMap != nil {
+		config = &tls.Config{
+			InsecureSkipVerify:    true,
+			VerifyPeerCertificate: buildSPIFFEVerifyFunc(km.SPIFFEBundleMap),
+			Certificates:          km.Certs,
+		}
+	} else {
+		config = &tls.Config{
+			RootCAs:      km.Roots,
+			Certificates: km.Certs,
+		}
+	}
+	return credentials.NewTLS(config).ClientHandshake(ctx, authority, rawConn)
+}
+
+func (c *reloadingCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{SecurityProtocol: "tls"}
+}
+
+func (c *reloadingCreds) Clone() credentials.TransportCredentials {
+	return &reloadingCreds{provider: c.provider}
+}
+
+func (c *reloadingCreds) OverrideServerName(string) error {
+	return errors.New("overriding server name is not supported by xDS client TLS credentials")
+}
+
+func (c *reloadingCreds) ServerHandshake(net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, nil, errors.New("server handshake is not supported by xDS client TLS credentials")
+}
+
+func buildSPIFFEVerifyFunc(spiffeBundleMap map[string]*spiffebundle.Bundle) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+	return func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+		rawCertList := make([]*x509.Certificate, len(rawCerts))
+		for i, asn1Data := range rawCerts {
+			cert, err := x509.ParseCertificate(asn1Data)
+			if err != nil {
+				return fmt.Errorf("spiffe: verify function could not parse input certificate: %v", err)
+			}
+			rawCertList[i] = cert
+		}
+		if len(rawCertList) == 0 {
+			return fmt.Errorf("spiffe: verify function has no valid input certificates")
+		}
+		leafCert := rawCertList[0]
+		roots, err := spiffe.GetRootsFromSPIFFEBundleMap(spiffeBundleMap, leafCert)
+		if err != nil {
+			return err
+		}
+
+		opts := x509.VerifyOptions{
+			Roots:         roots,
+			CurrentTime:   time.Now(),
+			Intermediates: x509.NewCertPool(),
+		}
+
+		for _, cert := range rawCertList[1:] {
+			opts.Intermediates.AddCert(cert)
+		}
+		// The verified chain is (surprisingly) unused.
+		if _, err = rawCertList[0].Verify(opts); err != nil {
+			return fmt.Errorf("spiffe: x509 certificate Verify failed: %v", err)
+		}
+		return nil
+	}
+}

internal/xds/bootstrap/tlscreds/bundle_ext_test.go

@@ -0,0 +1,520 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package tlscreds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds/bootstrap/tlscreds"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/testdata"
+)
+
+const defaultTestTimeout = 5 * time.Second
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type Closable interface {
+	Close()
+}
+
+func (s) TestValidTlsBuilder(t *testing.T) {
+	caCert := testdata.Path("x509/server_ca_cert.pem")
+	clientCert := testdata.Path("x509/client1_cert.pem")
+	clientKey := testdata.Path("x509/client1_key.pem")
+	clientSpiffeBundle := testdata.Path("spiffe_end2end/client_spiffebundle.json")
+	tests := []struct {
+		name string
+		jd   string
+	}{
+		{
+			name: "Absent configuration",
+			jd:   `null`,
+		},
+		{
+			name: "Empty configuration",
+			jd:   `{}`,
+		},
+		{
+			name: "Only CA certificate chain",
+			jd:   fmt.Sprintf(`{"ca_certificate_file": "%s"}`, caCert),
+		},
+		{
+			name: "Only private key and certificate chain",
+			jd:   fmt.Sprintf(`{"certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
+		},
+		{
+			name: "CA chain, private key and certificate chain",
+			jd:   fmt.Sprintf(`{"ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
+		},
+		{
+			name: "Only refresh interval", jd: `{"refresh_interval": "1s"}`,
+		},
+		{
+			name: "Refresh interval and CA certificate chain",
+			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file": "%s"}`, caCert),
+		},
+		{
+			name: "Refresh interval, private key and certificate chain",
+			jd:   fmt.Sprintf(`{"refresh_interval": "1s","certificate_file":"%s","private_key_file":"%s"}`, clientCert, clientKey),
+		},
+		{
+			name: "Refresh interval, CA chain, private key and certificate chain",
+			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s"}`, caCert, clientCert, clientKey),
+		},
+		{
+			name: "Refresh interval, CA chain, private key, certificate chain, spiffe bundle",
+			jd:   fmt.Sprintf(`{"refresh_interval": "1s","ca_certificate_file":"%s","certificate_file":"%s","private_key_file":"%s","spiffe_trust_bundle_map_file":"%s"}`, caCert, clientCert, clientKey, clientSpiffeBundle),
+		},
+		{
+			name: "Unknown field",
+			jd:   `{"unknown_field": "foo"}`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			msg := json.RawMessage(test.jd)
+			_, stop, err := tlscreds.NewBundle(msg)
+			if err != nil {
+				t.Fatalf("NewBundle(%s) returned error %s when expected to succeed", test.jd, err)
+			}
+			stop()
+		})
+	}
+}
+
+func (s) TestInvalidTlsBuilder(t *testing.T) {
+	tests := []struct {
+		name, jd, wantErrPrefix string
+	}{
+		{
+			name:          "Wrong type in json",
+			jd:            `{"ca_certificate_file": 1}`,
+			wantErrPrefix: "failed to unmarshal config:"},
+		{
+			name:          "Missing private key",
+			jd:            fmt.Sprintf(`{"certificate_file":"%s"}`, testdata.Path("x509/server_cert.pem")),
+			wantErrPrefix: "pemfile: private key file and identity cert file should be both specified or not specified",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			msg := json.RawMessage(test.jd)
+			_, stop, err := tlscreds.NewBundle(msg)
+			if err == nil || !strings.HasPrefix(err.Error(), test.wantErrPrefix) {
+				if stop != nil {
+					stop()
+				}
+				t.Fatalf("NewBundle(%s): got error %s, want an error with prefix %s", msg, err, test.wantErrPrefix)
+			}
+		})
+	}
+}
+
+func (s) TestCaReloading(t *testing.T) {
+	serverCa, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("Failed to read test CA cert: %s", err)
+	}
+
+	// Write CA certs to a temporary file so that we can modify it later.
+	caPath := t.TempDir() + "/ca.pem"
+	if err = os.WriteFile(caPath, serverCa, 0644); err != nil {
+		t.Fatalf("Failed to write test CA cert: %v", err)
+	}
+	cfg := fmt.Sprintf(`{
+		"ca_certificate_file": "%s",
+		"refresh_interval": ".01s"
+	}`, caPath)
+	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+	if err != nil {
+		t.Fatalf("Failed to create TLS bundle: %v", err)
+	}
+	defer stop()
+
+	serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.NoClientCert))
+	server := stubserver.StartTestService(t, nil, serverCredentials)
+
+	conn, err := grpc.NewClient(
+		server.Address,
+		grpc.WithCredentialsBundle(tlsBundle),
+		grpc.WithAuthority("x.test.example.com"),
+	)
+	if err != nil {
+		t.Fatalf("Error dialing: %v", err)
+	}
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	client := testgrpc.NewTestServiceClient(conn)
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Errorf("Error calling EmptyCall: %v", err)
+	}
+	// close the server and create a new one to force client to do a new
+	// handshake.
+	server.Stop()
+
+	invalidCa, err := os.ReadFile(testdata.Path("ca.pem"))
+	if err != nil {
+		t.Fatalf("Failed to read test CA cert: %v", err)
+	}
+	// unload root cert
+	err = os.WriteFile(caPath, invalidCa, 0644)
+	if err != nil {
+		t.Fatalf("Failed to write test CA cert: %v", err)
+	}
+	for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+		ss := stubserver.StubServer{
+			Address:    server.Address,
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+		}
+		server = stubserver.StartTestService(t, &ss, serverCredentials)
+
+		// Client handshake should eventually fail because the client CA was
+		// reloaded, and thus the server cert is signed by an unknown CA.
+		t.Log(server)
+		_, err = client.EmptyCall(ctx, &testpb.Empty{})
+		const wantErr = "certificate signed by unknown authority"
+		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
+			// Certs have reloaded.
+			server.Stop()
+			break
+		}
+		t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
+		server.Stop()
+	}
+	if ctx.Err() != nil {
+		t.Errorf("Timed out waiting for CA certs reloading")
+	}
+}
+
+// Test_SPIFFE_Reloading sets up a client and server. The client is configured
+// to use a SPIFFE bundle map, and the server is configured to use TLS creds
+// compatible with this bundle. A handshake is performed and connection is
+// expected to be successful. Then we change the client's SPIFFE Bundle Map file
+// on disk to one that should fail with the server's credentials. This change
+// should be picked up by the client via our file reloading. Another handshake
+// is performed and checked for failure, ensuring that gRPC is correctly using
+// the changed-on-disk bundle map.
+func (s) Test_SPIFFE_Reloading(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
+	clientSPIFFEBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+	if err != nil {
+		t.Fatalf("Failed to read test SPIFFE bundle: %v", err)
+	}
+
+	// Write CA certs to a temporary file so that we can modify it later.
+	spiffePath := t.TempDir() + "/client_spiffe.json"
+	if err = os.WriteFile(spiffePath, clientSPIFFEBundle, 0644); err != nil {
+		t.Fatalf("Failed to write test SPIFFE Bundle %v: %v", clientSPIFFEBundle, err)
+	}
+	cfg := fmt.Sprintf(`{
+		"spiffe_trust_bundle_map_file": "%s",
+		"refresh_interval": ".01s"
+	}`, spiffePath)
+	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+	if err != nil {
+		t.Fatalf("Failed to create TLS bundle: %v", err)
+	}
+	defer stop()
+
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+	defer lis.Close()
+	ss := stubserver.StubServer{
+		Listener:   lis,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+
+	serverCredentials := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.NoClientCert))
+	server := stubserver.StartTestService(t, &ss, serverCredentials)
+
+	defer server.Stop()
+
+	conn, err := grpc.NewClient(
+		server.Address,
+		grpc.WithCredentialsBundle(tlsBundle),
+		grpc.WithAuthority("x.test.example.com"),
+	)
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) failed: %v", server.Address, err)
+	}
+	defer conn.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	client := testgrpc.NewTestServiceClient(conn)
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Errorf("Error calling EmptyCall: %v", err)
+	}
+
+	// Setup the wrong bundle to be reloaded
+	wrongBundle, err := os.ReadFile(testdata.Path("spiffe_end2end/server_spiffebundle.json"))
+	if err != nil {
+		t.Fatalf("Failed to read test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
+	}
+	// Write the bundle that will fail to the tmp file path to be reloaded
+	err = os.WriteFile(spiffePath, wrongBundle, 0644)
+	if err != nil {
+		t.Fatalf("Failed to write test spiffe bundle %v: %v", "spiffe_end2end/server_spiffebundle.json", err)
+	}
+
+	for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+		// Stop and restart the listener to force new handshakes
+		lis.Stop()
+		lis.Restart()
+		// Client handshake should eventually fail because the client CA was
+		// reloaded, and thus the server cert is signed by an unknown CA.
+		t.Log(server)
+		_, err = client.EmptyCall(ctx, &testpb.Empty{})
+		const wantErr = "no bundle found for peer certificates trust domain"
+		if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), wantErr) {
+			// Certs have reloaded.
+			server.Stop()
+			break
+		}
+		t.Logf("EmptyCall() got err: %s, want code: %s, want err: %s", err, codes.Unavailable, wantErr)
+	}
+	if ctx.Err() != nil {
+		t.Errorf("Timed out waiting for CA certs reloading")
+	}
+}
+
+func (s) TestMTLS(t *testing.T) {
+	s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)))
+	defer s.Stop()
+
+	cfg := fmt.Sprintf(`{
+		"ca_certificate_file": "%s",
+		"certificate_file": "%s",
+		"private_key_file": "%s"
+	}`,
+		testdata.Path("x509/server_ca_cert.pem"),
+		testdata.Path("x509/client1_cert.pem"),
+		testdata.Path("x509/client1_key.pem"))
+	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+	if err != nil {
+		t.Fatalf("Failed to create TLS bundle: %v", err)
+	}
+	defer stop()
+	conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
+	if err != nil {
+		t.Fatalf("Error dialing: %v", err)
+	}
+	defer conn.Close()
+	client := testgrpc.NewTestServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
+	}
+}
+
+// Test_MTLS_SPIFFE configures a client and server. The server has a certificate
+// chain that is compatible with the client's configured SPIFFE bundle map. An
+// MTLS connection is attempted between the two and checked for success.
+func (s) Test_MTLS_SPIFFE(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
+	tests := []struct {
+		name         string
+		serverOption grpc.ServerOption
+	}{
+		{
+			name:         "MTLS SPIFFE",
+			serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
+		},
+		{
+			name:         "MTLS SPIFFE Chain",
+			serverOption: grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)),
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			s := stubserver.StartTestService(t, nil, tc.serverOption)
+			defer s.Stop()
+
+			cfg := fmt.Sprintf(`{
+	"certificate_file": "%s",
+	"private_key_file": "%s",
+	"spiffe_trust_bundle_map_file": "%s"
+}`,
+				testdata.Path("spiffe_end2end/client_spiffe.pem"),
+				testdata.Path("spiffe_end2end/client.key"),
+				testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+			tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+			if err != nil {
+				t.Fatalf("Failed to create TLS bundle: %v", err)
+			}
+			defer stop()
+			conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
+			if err != nil {
+				t.Fatalf("Error dialing: %v", err)
+			}
+			defer conn.Close()
+			client := testgrpc.NewTestServiceClient(conn)
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Errorf("EmptyCall(): got error %v when expected to succeed", err)
+			}
+		})
+	}
+}
+
+// Test_MTLS_SPIFFE_FlagDisabled configures a client and server. The server has
+// a certificate chain that is compatible with the client's configured SPIFFE
+// bundle map. However, the XDS flag that enabled SPIFFE usage is disabled. An
+// MTLS connection is attempted between the two and checked for failure.
+func (s) Test_MTLS_SPIFFE_FlagDisabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, false)
+	serverOption := grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert))
+	s := stubserver.StartTestService(t, nil, serverOption)
+	defer s.Stop()
+
+	cfg := fmt.Sprintf(`{
+"certificate_file": "%s",
+"private_key_file": "%s",
+"spiffe_trust_bundle_map_file": "%s"
+}`,
+		testdata.Path("spiffe_end2end/client_spiffe.pem"),
+		testdata.Path("spiffe_end2end/client.key"),
+		testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+	tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+	if err != nil {
+		t.Fatalf("Failed to create TLS bundle: %v", err)
+	}
+	defer stop()
+	conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
+	if err != nil {
+		t.Fatalf("Error dialing: %v", err)
+	}
+	defer conn.Close()
+	client := testgrpc.NewTestServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Errorf("EmptyCall(): got success want failure")
+	}
+}
+
+func (s) Test_MTLS_SPIFFE_Failure(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSPIFFEEnabled, true)
+	tests := []struct {
+		name             string
+		certFile         string
+		keyFile          string
+		spiffeBundleFile string
+		serverOption     grpc.ServerOption
+		wantErrContains  string
+		wantErrCode      codes.Code
+	}{
+		{
+			name:             "No matching trust domain in bundle",
+			certFile:         "spiffe_end2end/client_spiffe.pem",
+			keyFile:          "spiffe_end2end/client.key",
+			spiffeBundleFile: "spiffe_end2end/server_spiffebundle.json",
+			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)),
+			wantErrContains:  "spiffe: no bundle found for peer certificates",
+			wantErrCode:      codes.Unavailable,
+		},
+		{
+			name:             "Server cert has no valid SPIFFE URIs",
+			certFile:         "spiffe_end2end/client_spiffe.pem",
+			keyFile:          "spiffe_end2end/client.key",
+			spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
+			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)),
+			wantErrContains:  "spiffe: could not get spiffe ID from peer leaf cert",
+			wantErrCode:      codes.Unavailable,
+		},
+		{
+			name:             "Server cert has valid spiffe ID but doesn't chain to the root CA",
+			certFile:         "spiffe_end2end/client_spiffe.pem",
+			keyFile:          "spiffe_end2end/client.key",
+			spiffeBundleFile: "spiffe_end2end/client_spiffebundle.json",
+			serverOption:     grpc.Creds(testutils.CreateServerTLSCredentialsValidSPIFFEButWrongCA(t, tls.RequireAndVerifyClientCert)),
+			wantErrContains:  "spiffe: x509 certificate Verify failed: x509: certificate signed by unknown authority",
+			wantErrCode:      codes.Unavailable,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			s := stubserver.StartTestService(t, nil, tc.serverOption)
+			defer s.Stop()
+			cfg := fmt.Sprintf(`{
+"certificate_file": "%s",
+"private_key_file": "%s",
+"spiffe_trust_bundle_map_file": "%s"
+}`,
+				testdata.Path(tc.certFile),
+				testdata.Path(tc.keyFile),
+				testdata.Path(tc.spiffeBundleFile))
+			tlsBundle, stop, err := tlscreds.NewBundle([]byte(cfg))
+			if err != nil {
+				t.Fatalf("Failed to create TLS bundle: %v", err)
+			}
+			defer stop()
+			conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
+			if err != nil {
+				t.Fatalf("grpc.NewClient(%q) failed: %v", s.Address, err)
+			}
+			defer conn.Close()
+			client := testgrpc.NewTestServiceClient(conn)
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+				t.Errorf("EmptyCall(): got success. want failure")
+			}
+			if status.Code(err) != tc.wantErrCode {
+				t.Errorf("EmptyCall(): failed with wrong error. got code %v. want code: %v", status.Code(err), tc.wantErrCode)
+			}
+			if !strings.Contains(err.Error(), tc.wantErrContains) {
+				t.Errorf("EmptyCall(): failed with wrong error. got %v. want contains: %v", err, tc.wantErrContains)
+			}
+		})
+	}
+}

internal/xds/bootstrap/tlscreds/bundle_test.go

@@ -0,0 +1,162 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package tlscreds
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/credentials/spiffe"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const defaultTestTimeout = 5 * time.Second
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type failingProvider struct{}
+
+func (f failingProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) {
+	return nil, errors.New("test error")
+}
+
+func (f failingProvider) Close() {}
+
+func (s) TestFailingProvider(t *testing.T) {
+	s := stubserver.StartTestService(t, nil, grpc.Creds(testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)))
+	defer s.Stop()
+
+	cfg := fmt.Sprintf(`{
+               "ca_certificate_file": "%s",
+               "certificate_file": "%s",
+               "private_key_file": "%s",
+			   "spiffe_trust_bundle_map_file": "%s"
+       }`,
+		testdata.Path("x509/server_ca_cert.pem"),
+		testdata.Path("x509/client1_cert.pem"),
+		testdata.Path("x509/client1_key.pem"),
+		testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+	tlsBundle, stop, err := NewBundle([]byte(cfg))
+	if err != nil {
+		t.Fatalf("Failed to create TLS bundle: %v", err)
+	}
+	stop()
+
+	// Force a provider that returns an error, and make sure the client fails
+	// the handshake.
+	creds, ok := tlsBundle.TransportCredentials().(*reloadingCreds)
+	if !ok {
+		t.Fatalf("Got %T, expected reloadingCreds", tlsBundle.TransportCredentials())
+	}
+	creds.provider = &failingProvider{}
+
+	conn, err := grpc.NewClient(s.Address, grpc.WithCredentialsBundle(tlsBundle), grpc.WithAuthority("x.test.example.com"))
+	if err != nil {
+		t.Fatalf("Error dialing: %v", err)
+	}
+	defer conn.Close()
+
+	client := testgrpc.NewTestServiceClient(conn)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if wantErr := "test error"; err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Errorf("EmptyCall() got err: %s, want err to contain: %s", err, wantErr)
+	}
+}
+
+func rawCertsFromFile(t *testing.T, filePath string) [][]byte {
+	t.Helper()
+	rawCert, err := os.ReadFile(testdata.Path(filePath))
+	if err != nil {
+		t.Fatalf("Reading certificate file failed: %v", err)
+	}
+	block, _ := pem.Decode(rawCert)
+	if block == nil || block.Type != "CERTIFICATE" {
+		t.Fatalf("pem.Decode() failed to decode certificate in file %q", "spiffe/server1_spiffe.pem")
+	}
+	return [][]byte{block.Bytes}
+}
+
+func (s) TestSPIFFEVerifyFuncMismatchedCert(t *testing.T) {
+	spiffeBundleBytes, err := os.ReadFile(testdata.Path("spiffe_end2end/client_spiffebundle.json"))
+	if err != nil {
+		t.Fatalf("Reading spiffebundle file failed: %v", err)
+	}
+	spiffeBundle, err := spiffe.BundleMapFromBytes(spiffeBundleBytes)
+	if err != nil {
+		t.Fatalf("spiffe.BundleMapFromBytes() failed: %v", err)
+	}
+	verifyFunc := buildSPIFFEVerifyFunc(spiffeBundle)
+	verifiedChains := [][]*x509.Certificate{}
+	tests := []struct {
+		name            string
+		rawCerts        [][]byte
+		wantErrContains string
+	}{
+		{
+			name:            "mismathed cert",
+			rawCerts:        rawCertsFromFile(t, "spiffe/server1_spiffe.pem"),
+			wantErrContains: "spiffe: x509 certificate Verify failed",
+		},
+		{
+			name:            "bad input cert",
+			rawCerts:        [][]byte{[]byte("NOT_GOOD_DATA")},
+			wantErrContains: "spiffe: verify function could not parse input certificate",
+		},
+		{
+			name:            "no input bytes",
+			rawCerts:        nil,
+			wantErrContains: "no valid input certificates",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			err = verifyFunc(tc.rawCerts, verifiedChains)
+			if err == nil {
+				t.Fatalf("buildSPIFFEVerifyFunc call succeeded. want failure")
+			}
+			if !strings.Contains(err.Error(), tc.wantErrContains) {
+				t.Fatalf("buildSPIFFEVerifyFunc got err %v want err to contain %v", err, tc.wantErrContains)
+			}
+		})
+	}
+}

internal/xds/clients/config.go

@@ -0,0 +1,140 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clients provides implementations of the clients to interact with
+// xDS and LRS servers.
+//
+// # xDS Client
+//
+// The xDS client allows applications to:
+//   - Create client instances with in-memory configurations.
+//   - Register watches for named resources.
+//   - Receive resources via the ADS (Aggregated Discovery Service) stream.
+//
+// This enables applications to dynamically discover and configure resources
+// such as listeners, routes, clusters, and endpoints from an xDS management
+// server.
+//
+// # LRS Client
+//
+// The LRS (Load Reporting Service) client allows applications to report load
+// data to an LRS server via the LRS stream. This data can be used for
+// monitoring, traffic management, and other purposes.
+//
+// # Experimental
+//
+// NOTICE: This package is EXPERIMENTAL and may be changed or removed
+// in a later release.
+package clients
+
+// ServerIdentifier holds identifying information for connecting to an xDS
+// management or LRS server.
+type ServerIdentifier struct {
+	// ServerURI is the target URI of the server.
+	ServerURI string
+
+	// Extensions can be populated with arbitrary data to be passed to the
+	// TransportBuilder and/or xDS Client's ResourceType implementations.
+	// This field can be used to provide additional configuration or context
+	// specific to the user's needs.
+	//
+	// The xDS and LRS clients do not interpret the contents of this field.
+	// It is the responsibility of the user's custom TransportBuilder and/or
+	// ResourceType implementations to handle and interpret these extensions.
+	//
+	// For example, a custom TransportBuilder might use this field to
+	// configure a specific security credentials.
+	//
+	// Extensions may be any type that is comparable, as they are used as map
+	// keys internally. If Extensions are not able to be used as a map key,
+	// the client may panic.
+	//
+	// See: https://go.dev/ref/spec#Comparison_operators
+	//
+	// Any equivalent extensions in all ServerIdentifiers present in a single
+	// client's configuration should have the same value. Not following this
+	// restriction may result in excess resource usage.
+	Extensions any
+}
+
+// Node represents the identity of the xDS client, allowing xDS and LRS servers
+// to identify the source of xDS requests.
+type Node struct {
+	// ID is a string identifier of the application.
+	ID string
+	// Cluster is the name of the cluster the application belongs to.
+	Cluster string
+	// Locality is the location of the application including region, zone,
+	// sub-zone.
+	Locality Locality
+	// Metadata provides additional context about the application by associating
+	// arbitrary key-value pairs with it.
+	Metadata any
+	// UserAgentName is the user agent name of application.
+	UserAgentName string
+	// UserAgentVersion is the user agent version of application.
+	UserAgentVersion string
+}
+
+// Locality represents the location of the xDS client application.
+type Locality struct {
+	// Region is the region of the xDS client application.
+	Region string
+	// Zone is the area within a region.
+	Zone string
+	// SubZone is the further subdivision within a zone.
+	SubZone string
+}
+
+// MetricsReporter is used by the XDSClient to report metrics.
+type MetricsReporter interface {
+	// ReportMetric reports a metric. The metric will be one of the predefined
+	// set of types depending on the client (XDSClient or LRSClient).
+	//
+	// Each client will produce different metrics. Please see the client's
+	// documentation for a list of possible metrics events.
+	ReportMetric(metric any)
+
+	// RegisterAsyncReporter registers a reporter to produce metric values for
+	// the set of metrics supported by the client. See the metrics sub-package
+	// for the specific client (e.g. internal/xds/clients/xdsclient/metrics/metrics.go)
+	// for the list of supported metrics. The returned function must be called
+	// when the metrics are no longer needed, which will remove the reporter.
+	// The function is expected to be idempotent.
+	//
+	// Once the returned cancel function is called, the Report method on the
+	// registered reporter is guaranteed not to be called again.
+	RegisterAsyncReporter(reporter AsyncReporter) func()
+}
+
+// AsyncReporter records metrics asynchronously.
+// Implementations must be concurrent-safe.
+// The metric will be recorded once per collection cycle, rather than every time
+// its value changes.
+type AsyncReporter interface {
+	// Report records metric values using the provided recorder.
+	Report(AsyncMetricsRecorder) error
+}
+
+// AsyncMetricsRecorder is a recorder for async metrics (i.e the metric will be
+// recorded once per collection cycle, rather than every time its value changes).
+type AsyncMetricsRecorder interface {
+	// ReportMetric reports a metric. The metric will be one of the predefined
+	// set of types in the internal/xds/clients/xdsclient/metrics/metrics.go file.
+	ReportMetric(metric any)
+}

internal/xds/clients/grpctransport/examples_test.go

@@ -0,0 +1,39 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpctransport_test
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+)
+
+// ExampleServerIdentifierExtension demonstrates how to create
+// clients.ServerIdentifier with grpctransport.ServerIdentifierExtension as
+// its extensions.
+//
+// This example is creating clients.ServerIdentifier to connect to server at
+// localhost:5678 using the config named "local". Note that "local" must
+// exist as an entry in the provided configs to grpctransport.Builder.
+func ExampleServerIdentifierExtension() {
+	// Note the Extensions field is set by value and not by pointer.
+	fmt.Printf("%+v", clients.ServerIdentifier{ServerURI: "localhost:5678", Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"}})
+	// Output: {ServerURI:localhost:5678 Extensions:{ConfigName:local}}
+}

internal/xds/clients/grpctransport/grpc_transport.go

@@ -0,0 +1,237 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package grpctransport provides an implementation of the
+// clients.TransportBuilder interface using gRPC.
+package grpctransport
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/keepalive"
+)
+
+var (
+	logger = grpclog.Component("grpctransport")
+)
+
+// ServerIdentifierExtension holds settings for connecting to a gRPC server,
+// such as an xDS management or an LRS server.
+//
+// It must be set by value (not pointer) in the
+// clients.ServerIdentifier.Extensions field (See Example).
+type ServerIdentifierExtension struct {
+	// ConfigName is the name of the configuration to use for this transport.
+	// It must be present as a key in the map of configs passed to NewBuilder.
+	ConfigName string
+}
+
+// Builder creates gRPC-based Transports. It must be paired with ServerIdentifiers
+// that contain an Extension field of type ServerIdentifierExtension.
+type Builder struct {
+	// configs is a map of configuration names to their respective Config.
+	configs map[string]Config
+
+	mu sync.Mutex
+	// connections is a map of clients.ServerIdentifiers in use by the Builder
+	// to connect to different servers.
+	connections map[clients.ServerIdentifier]*grpc.ClientConn
+	// refs tracks the number of active references to each connection.
+	refs map[clients.ServerIdentifier]int
+}
+
+// Config defines the configuration for connecting to a gRPC server, including
+// credentials and an optional custom new client function.
+type Config struct {
+	// Credentials is the credentials bundle to be used for the connection.
+	Credentials credentials.Bundle
+	// GRPCNewClient is an optional custom function to establish a gRPC connection.
+	// If nil, grpc.NewClient will be used as the default.
+	GRPCNewClient func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error)
+}
+
+// NewBuilder provides a builder for creating gRPC-based Transports using
+// the credentials from provided map of credentials names to
+// credentials.Bundle.
+func NewBuilder(configs map[string]Config) *Builder {
+	return &Builder{
+		configs:     configs,
+		connections: make(map[clients.ServerIdentifier]*grpc.ClientConn),
+		refs:        make(map[clients.ServerIdentifier]int),
+	}
+}
+
+// Build returns a gRPC-based clients.Transport.
+//
+// The Extension field of the ServerIdentifier must be a ServerIdentifierExtension.
+func (b *Builder) Build(si clients.ServerIdentifier) (clients.Transport, error) {
+	if si.ServerURI == "" {
+		return nil, fmt.Errorf("grpctransport: ServerURI is not set in ServerIdentifier")
+	}
+	if si.Extensions == nil {
+		return nil, fmt.Errorf("grpctransport: Extensions is not set in ServerIdentifier")
+	}
+	sce, ok := si.Extensions.(ServerIdentifierExtension)
+	if !ok {
+		return nil, fmt.Errorf("grpctransport: Extensions field is %T, but must be %T in ServerIdentifier", si.Extensions, ServerIdentifierExtension{})
+	}
+
+	config, ok := b.configs[sce.ConfigName]
+	if !ok {
+		return nil, fmt.Errorf("grpctransport: unknown config name %q specified in ServerIdentifierExtension", sce.ConfigName)
+	}
+	if config.Credentials == nil {
+		return nil, fmt.Errorf("grpctransport: config %q has nil credentials bundle", sce.ConfigName)
+	}
+
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if cc, ok := b.connections[si]; ok {
+		if logger.V(2) {
+			logger.Infof("Reusing existing connection to the server for ServerIdentifier: %v", si)
+		}
+		b.refs[si]++
+		tr := &grpcTransport{cc: cc}
+		tr.cleanup = b.cleanupFunc(si, tr)
+		return tr, nil
+	}
+
+	// Create a new gRPC client/channel for the server with the provided
+	// credentials, server URI, and a byte codec to send and receive messages.
+	// Also set a static keepalive configuration that is common across gRPC
+	// language implementations.
+	kpCfg := grpc.WithKeepaliveParams(keepalive.ClientParameters{
+		Time:    5 * time.Minute,
+		Timeout: 20 * time.Second,
+	})
+	dopts := []grpc.DialOption{kpCfg, grpc.WithCredentialsBundle(config.Credentials), grpc.WithDefaultCallOptions(grpc.ForceCodec(&byteCodec{}))}
+	newClientFunc := grpc.NewClient
+	if config.GRPCNewClient != nil {
+		newClientFunc = config.GRPCNewClient
+	}
+	cc, err := newClientFunc(si.ServerURI, dopts...)
+	if err != nil {
+		return nil, fmt.Errorf("grpctransport: failed to create connection to server %q: %v", si.ServerURI, err)
+	}
+	tr := &grpcTransport{cc: cc}
+	// Register a cleanup function that decrements the refs to the gRPC
+	// transport each time Close() is called to close it and remove from
+	// transports and connections map if last reference is being released.
+	tr.cleanup = b.cleanupFunc(si, tr)
+
+	// Add the newly created connection to the maps to re-use the transport
+	// channel and track references.
+	b.connections[si] = cc
+	b.refs[si] = 1
+
+	if logger.V(2) {
+		logger.Infof("Created a new transport to the server for ServerIdentifier: %v", si)
+	}
+	return tr, nil
+}
+
+func (b *Builder) cleanupFunc(si clients.ServerIdentifier, tr *grpcTransport) func() {
+	return sync.OnceFunc(func() {
+		b.mu.Lock()
+		defer b.mu.Unlock()
+
+		b.refs[si]--
+		if b.refs[si] != 0 {
+			return
+		}
+
+		tr.cc.Close()
+		tr.cc = nil
+		delete(b.connections, si)
+		delete(b.refs, si)
+	})
+}
+
+type grpcTransport struct {
+	cc *grpc.ClientConn
+
+	// cleanup is the function to be invoked for releasing the references to
+	// the gRPC transport each time Close() is called.
+	cleanup func()
+}
+
+// NewStream creates a new gRPC stream to the server for the specified method.
+func (g *grpcTransport) NewStream(ctx context.Context, method string) (clients.Stream, error) {
+	s, err := g.cc.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, method)
+	if err != nil {
+		return nil, err
+	}
+	return &stream{stream: s}, nil
+}
+
+// Close closes the gRPC channel to the server.
+func (g *grpcTransport) Close() {
+	g.cleanup()
+}
+
+type stream struct {
+	stream grpc.ClientStream
+}
+
+// Send sends a message to the server.
+func (s *stream) Send(msg []byte) error {
+	return s.stream.SendMsg(msg)
+}
+
+// Recv receives a message from the server.
+func (s *stream) Recv() ([]byte, error) {
+	var typedRes []byte
+
+	if err := s.stream.RecvMsg(&typedRes); err != nil {
+		return nil, err
+	}
+	return typedRes, nil
+}
+
+// byteCodec here is still sending proto messages. It's just they are
+// in []byte form.
+type byteCodec struct{}
+
+func (c *byteCodec) Marshal(v any) ([]byte, error) {
+	if b, ok := v.([]byte); ok {
+		return b, nil
+	}
+	return nil, fmt.Errorf("grpctransport: message is %T, but must be a []byte", v)
+}
+
+func (c *byteCodec) Unmarshal(data []byte, v any) error {
+	if b, ok := v.(*[]byte); ok {
+		*b = data
+		return nil
+	}
+	return fmt.Errorf("grpctransport: target is %T, but must be *[]byte", v)
+}
+
+func (c *byteCodec) Name() string {
+	// Return "" to ensure the Content-Type header is "application/grpc",
+	// which is expected by standard gRPC servers for protobuf messages.
+	return ""
+}

internal/xds/clients/grpctransport/grpc_transport_ext_test.go

@@ -0,0 +1,316 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpctransport_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+)
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type testCredentials struct {
+	credentials.Bundle
+	transportCredentials credentials.TransportCredentials
+}
+
+func (tc *testCredentials) TransportCredentials() credentials.TransportCredentials {
+	return tc.transportCredentials
+}
+func (tc *testCredentials) PerRPCCredentials() credentials.PerRPCCredentials {
+	return nil
+}
+
+// TestBuild_Single tests that multiple calls to Build() with the same
+// clients.ServerIdentifier returns the same transport. Also verifies that
+// only when all references to the newly created transport are released,
+// the underlying transport is closed.
+func (s) TestBuild_Single(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	lis := testutils.NewListenerWrapper(t, nil)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
+
+	serverID := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"},
+	}
+	configs := map[string]grpctransport.Config{
+		"local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+	}
+
+	// Calling Build() first time should create new gRPC transport.
+	builder := grpctransport.NewBuilder(configs)
+	tr, err := builder.Build(serverID)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	// Create a new stream to the server and verify that a new transport is
+	// created.
+	if _, err = tr.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("Failed to create stream: %v", err)
+	}
+	val, err := lis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+	conn := val.(*testutils.ConnWrapper)
+
+	// Calling Build() again should not create new gRPC transport.
+	const count = 9
+	transports := make([]clients.Transport, count)
+	for i := 0; i < count; i++ {
+		func() {
+			transports[i], err = builder.Build(serverID)
+			if err != nil {
+				t.Fatalf("Failed to build transport: %v", err)
+			}
+			// Create a new stream to the server and verify that no connection
+			// is established to the management server at this point. A new
+			// transport is created only when an existing connection for
+			// serverID does not exist.
+			if _, err = tr.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+				t.Fatalf("Failed to create stream: %v", err)
+			}
+			sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+			defer sCancel()
+			if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+				t.Fatal("Unexpected new transport created to management server")
+			}
+		}()
+	}
+
+	// Call Close() multiple times on each of the transport received in the
+	// above for loop. Close() calls are idempotent. The underlying gRPC
+	// transport is removed after the Close() call but calling close second
+	// time should not panic and underlying gRPC transport should not be
+	// closed.
+	for i := 0; i < count; i++ {
+		func() {
+			transports[i].Close()
+			transports[i].Close()
+			sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+			defer sCancel()
+			if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
+				t.Fatal("Unexpected transport closure to management server")
+			}
+		}()
+	}
+
+	// Call the last Close(). The underlying gRPC transport should be closed
+	// because calls in the above for loop have released all references.
+	tr.Close()
+	if _, err := conn.CloseCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for connection to management server to be closed")
+	}
+
+	// Calling Build() again, after the previous transport was actually closed,
+	// should create a new one.
+	tr2, err := builder.Build(serverID)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer tr2.Close()
+	// Create a new stream to the server and verify that a new transport is
+	// created.
+	if _, err = tr2.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("Failed to create stream: %v", err)
+	}
+	if _, err := lis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+}
+
+// TestBuild_Multiple tests the scenario where there are multiple calls to
+// Build() with different clients.ServerIdentifier. Verifies that reference
+// counts are tracked correctly for each transport and that only when all
+// references are released for a transport, it is closed.
+func (s) TestBuild_Multiple(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	lis := testutils.NewListenerWrapper(t, nil)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
+
+	serverID1 := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "local"},
+	}
+	serverID2 := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	configs := map[string]grpctransport.Config{
+		"local":    {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+		"insecure": {Credentials: insecure.NewBundle()},
+	}
+
+	// Create two gRPC transports.
+	builder := grpctransport.NewBuilder(configs)
+
+	tr1, err := builder.Build(serverID1)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	// Create a new stream to the server and verify that a new transport is
+	// created.
+	if _, err = tr1.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("Failed to create stream: %v", err)
+	}
+	val, err := lis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+	conn1 := val.(*testutils.ConnWrapper)
+
+	tr2, err := builder.Build(serverID2)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	// Create a new stream to the server and verify that a new transport is
+	// created because credentials are different.
+	if _, err = tr2.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("Failed to create stream: %v", err)
+	}
+	val, err = lis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+	conn2 := val.(*testutils.ConnWrapper)
+
+	// Create N more references to each of the two transports.
+	const count = 9
+	transports1 := make([]clients.Transport, count)
+	transports2 := make([]clients.Transport, count)
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < count; i++ {
+			var err error
+			transports1[i], err = builder.Build(serverID1)
+			if err != nil {
+				t.Errorf("Failed to build transport: %v", err)
+			}
+			// Create a new stream to the server and verify that no connection
+			// is established to the management server at this point. A new
+			// transport is created only when an existing connection for
+			// serverID does not exist.
+			if _, err = transports1[i].NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+				t.Errorf("Failed to create stream: %v", err)
+			}
+			sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+			defer sCancel()
+			if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+				t.Error("Unexpected new transport created to management server")
+			}
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for i := 0; i < count; i++ {
+			var err error
+			transports2[i], err = builder.Build(serverID2)
+			if err != nil {
+				t.Errorf("%d-th call to Build() failed with error: %v", i, err)
+			}
+			// Create a new stream to the server and verify that no connection
+			// is established to the management server at this point. A new
+			// transport is created only when an existing connection for
+			// serverID does not exist.
+			if _, err = transports2[i].NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+				t.Errorf("Failed to create stream: %v", err)
+			}
+			sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+			defer sCancel()
+			if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+				t.Error("Unexpected new transport created to management server")
+			}
+		}
+	}()
+	wg.Wait()
+	if t.Failed() {
+		t.FailNow()
+	}
+
+	// Call Close() multiple times on each of the transport received in the
+	// above for loop. Close() calls are idempotent. The underlying gRPC
+	// transport is removed after the Close() call but calling close second
+	// time should not panic and underlying gRPC transport should not be
+	// closed.
+	for i := 0; i < count; i++ {
+		transports1[i].Close()
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		if _, err := conn1.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
+			t.Fatal("Unexpected transport closure to management server")
+		}
+		transports1[i].Close()
+	}
+	// Call the last Close(). The underlying gRPC transport should be closed
+	// because calls in the above for loop have released all references.
+	tr1.Close()
+	if _, err := conn1.CloseCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for connection to management server to be closed")
+	}
+
+	// Call Close() multiple times on each of the transport received in the
+	// above for loop. Close() calls are idempotent. The underlying gRPC
+	// transport is removed after the Close() call but calling close second
+	// time should not panic and underlying gRPC transport should not be
+	// closed.
+	for i := 0; i < count; i++ {
+		transports2[i].Close()
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		if _, err := conn2.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
+			t.Fatal("Unexpected transport closure to management server")
+		}
+		transports2[i].Close()
+	}
+	// Call the last Close(). The underlying gRPC transport should be closed
+	// because calls in the above for loop have released all references.
+	tr2.Close()
+	if _, err := conn2.CloseCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for connection to management server to be closed")
+	}
+}

internal/xds/clients/grpctransport/grpc_transport_test.go

@@ -0,0 +1,425 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package grpctransport
+
+import (
+	"context"
+	"io"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// testServer implements the AggregatedDiscoveryServiceServer interface to test
+// the gRPC transport implementation.
+type testServer struct {
+	v3discoverygrpc.UnimplementedAggregatedDiscoveryServiceServer
+
+	address     string                               // address of the server
+	requestChan chan *v3discoverypb.DiscoveryRequest // channel to send the received requests on for verification
+	response    *v3discoverypb.DiscoveryResponse     // response to send back to the client from handler
+}
+
+// setupTestServer set up the gRPC server for AggregatedDiscoveryService. It
+// creates an instance of testServer that returns the provided response from
+// the StreamAggregatedResources() handler and registers it with a gRPC server.
+func setupTestServer(t *testing.T, response *v3discoverypb.DiscoveryResponse) *testServer {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Failed to listen on localhost:0: %v", err)
+	}
+	ts := &testServer{
+		requestChan: make(chan *v3discoverypb.DiscoveryRequest),
+		address:     lis.Addr().String(),
+		response:    response,
+	}
+
+	s := grpc.NewServer()
+
+	v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(s, ts)
+	go s.Serve(lis)
+	t.Cleanup(s.Stop)
+
+	return ts
+}
+
+// StreamAggregatedResources handles bidirectional streaming of
+// DiscoveryRequest and DiscoveryResponse. It waits for a message from the
+// client on the stream, and then sends a discovery response message back to
+// the client. It also put the received message in requestChan for client to
+// verify if the correct request was received. It continues until the client
+// closes the stream.
+func (s *testServer) StreamAggregatedResources(stream v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error {
+	ctx := stream.Context()
+
+	for {
+		// Receive a DiscoveryRequest from the client
+		req, err := stream.Recv()
+		if err == io.EOF {
+			return nil // Stream closed by client
+		}
+		if err != nil {
+			return err // Handle other errors
+		}
+
+		// Push received request for client to verify the correct request was
+		// received.
+		select {
+		case s.requestChan <- req:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+
+		// Send the response back to the client
+		if err := stream.Send(s.response); err != nil {
+			return err
+		}
+	}
+}
+
+type testCredentials struct {
+	credentials.Bundle
+	transportCredentials credentials.TransportCredentials
+}
+
+func (tc *testCredentials) TransportCredentials() credentials.TransportCredentials {
+	return tc.transportCredentials
+}
+func (tc *testCredentials) PerRPCCredentials() credentials.PerRPCCredentials {
+	return nil
+}
+
+// TestBuild_Success verifies that the Builder successfully creates a new
+// Transport in both cases when provided clients.ServerIdentifer is same
+// one of the existing transport or a new one.
+func (s) TestBuild_Success(t *testing.T) {
+	configs := map[string]Config{
+		"local":    {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+		"insecure": {Credentials: insecure.NewBundle()},
+	}
+	b := NewBuilder(configs)
+
+	serverID1 := clients.ServerIdentifier{
+		ServerURI:  "server-address",
+		Extensions: ServerIdentifierExtension{ConfigName: "local"},
+	}
+	tr1, err := b.Build(serverID1)
+	if err != nil {
+		t.Fatalf("Build(serverID1) call failed: %v", err)
+	}
+	defer tr1.Close()
+
+	serverID2 := clients.ServerIdentifier{
+		ServerURI:  "server-address",
+		Extensions: ServerIdentifierExtension{ConfigName: "local"},
+	}
+	tr2, err := b.Build(serverID2)
+	if err != nil {
+		t.Fatalf("Build(serverID2) call failed: %v", err)
+	}
+	defer tr2.Close()
+
+	serverID3 := clients.ServerIdentifier{
+		ServerURI:  "server-address",
+		Extensions: ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	tr3, err := b.Build(serverID3)
+	if err != nil {
+		t.Fatalf("Build(serverID3) call failed: %v", err)
+	}
+	defer tr3.Close()
+}
+
+// TestBuild_Failure verifies that the Builder returns error when incorrect
+// ServerIdentifier is provided.
+//
+// It covers the following scenarios:
+// - ServerURI is empty.
+// - Extensions is nil.
+// - Extensions is not ServerIdentifierExtension.
+// - Credentials are nil.
+func (s) TestBuild_Failure(t *testing.T) {
+	tests := []struct {
+		name     string
+		serverID clients.ServerIdentifier
+	}{
+		{
+			name: "ServerURI is empty",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "",
+				Extensions: ServerIdentifierExtension{ConfigName: "local"},
+			},
+		},
+		{
+			name:     "Extensions is nil",
+			serverID: clients.ServerIdentifier{ServerURI: "server-address"},
+		},
+		{
+			name: "Extensions is not a ServerIdentifierExtension",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "server-address",
+				Extensions: 1,
+			},
+		},
+		{
+			name: "ServerIdentifierExtension without ConfigName",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "server-address",
+				Extensions: ServerIdentifierExtension{},
+			},
+		},
+		{
+			name: "ServerIdentifierExtension ConfigName is not present",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "server-address",
+				Extensions: ServerIdentifierExtension{ConfigName: "unknown"},
+			},
+		},
+		{
+			name: "ServerIdentifierExtension ConfigName maps to nil credentials",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "server-address",
+				Extensions: ServerIdentifierExtension{ConfigName: "nil-credentials"},
+			},
+		},
+		{
+			name: "ServerIdentifierExtension is added as pointer",
+			serverID: clients.ServerIdentifier{
+				ServerURI:  "server-address",
+				Extensions: &ServerIdentifierExtension{ConfigName: "local"},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			configs := map[string]Config{
+				"local":           {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+				"nil-credentials": {Credentials: nil},
+			}
+			b := NewBuilder(configs)
+			tr, err := b.Build(test.serverID)
+			if err == nil {
+				t.Fatalf("Build() succeeded, want error")
+			}
+			if tr != nil {
+				t.Fatalf("Got non-nil transport from Build(), want nil")
+			}
+		})
+	}
+}
+
+// TestNewStream_Success verifies that NewStream() successfully creates a new
+// client stream for the server when provided a valid server URI and a config
+// with valid credentials.
+func (s) TestNewStream_Success(t *testing.T) {
+	ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"})
+
+	serverCfg := clients.ServerIdentifier{
+		ServerURI:  ts.address,
+		Extensions: ServerIdentifierExtension{ConfigName: "local"},
+	}
+	configs := map[string]Config{
+		"local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+	}
+	builder := NewBuilder(configs)
+	transport, err := builder.Build(serverCfg)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	defer transport.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("transport.NewStream() failed: %v", err)
+	}
+}
+
+// TestNewStream_Success_WithCustomGRPCNewClient verifies that NewStream()
+// successfully creates a new client stream for the server when provided a
+// valid server URI and a config with valid credentials and a custom gRPC
+// NewClient function.
+func (s) TestNewStream_Success_WithCustomGRPCNewClient(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"})
+
+	// Create a custom dialer function that will be used by the gRPC client.
+	customDialerCalled := make(chan struct{}, 1)
+	customGRPCNewClient := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+		customDialerCalled <- struct{}{}
+		return grpc.NewClient(target, opts...)
+	}
+
+	configs := map[string]Config{
+		"custom-dialer-config": {
+			Credentials:   &testCredentials{transportCredentials: local.NewCredentials()},
+			GRPCNewClient: customGRPCNewClient,
+		},
+	}
+	builder := NewBuilder(configs)
+
+	serverID := clients.ServerIdentifier{
+		ServerURI:  ts.address,
+		Extensions: ServerIdentifierExtension{ConfigName: "custom-dialer-config"},
+	}
+
+	transport, err := builder.Build(serverID)
+	if err != nil {
+		t.Fatalf("builder.Build(%+v) failed: %v", serverID, err)
+	}
+	defer transport.Close()
+
+	select {
+	case <-customDialerCalled:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for custom dialer to be called: %v", ctx.Err())
+	}
+
+	// Verify that the transport works by creating a stream.
+	if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err != nil {
+		t.Fatalf("transport.NewStream() failed with custom dialer: %v", err)
+	}
+}
+
+// TestNewStream_Error verifies that NewStream() returns an error
+// when attempting to create a stream with an invalid server URI.
+func (s) TestNewStream_Error(t *testing.T) {
+	serverCfg := clients.ServerIdentifier{
+		ServerURI:  "invalid-server-uri",
+		Extensions: ServerIdentifierExtension{ConfigName: "local"},
+	}
+	configs := map[string]Config{
+		"local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+	}
+	builder := NewBuilder(configs)
+	transport, err := builder.Build(serverCfg)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	defer transport.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources"); err == nil {
+		t.Fatal("transport.NewStream() succeeded, want failure")
+	}
+}
+
+// TestStream_SendAndRecv verifies that Send() and Recv() successfully send
+// and receive messages on the stream to and from the gRPC server.
+//
+// It starts a gRPC test server using setupTestServer(). The test then sends a
+// testDiscoverRequest on the stream and verifies that the received discovery
+// request on the server is same as sent. It then wait to receive a
+// testDiscoverResponse from the server and verifies that the received
+// discovery response is same as sent from the server.
+func (s) TestStream_SendAndRecv(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	ts := setupTestServer(t, &v3discoverypb.DiscoveryResponse{VersionInfo: "1"})
+
+	// Build a grpc-based transport to the above server.
+	serverCfg := clients.ServerIdentifier{
+		ServerURI:  ts.address,
+		Extensions: ServerIdentifierExtension{ConfigName: "local"},
+	}
+	configs := map[string]Config{
+		"local": {Credentials: &testCredentials{transportCredentials: local.NewCredentials()}},
+	}
+	builder := NewBuilder(configs)
+	transport, err := builder.Build(serverCfg)
+	if err != nil {
+		t.Fatalf("Failed to build transport: %v", err)
+	}
+	defer transport.Close()
+
+	// Create a new stream to the server.
+	stream, err := transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources")
+	if err != nil {
+		t.Fatalf("Failed to create stream: %v", err)
+	}
+
+	// Send a discovery request message on the stream.
+	testDiscoverRequest := &v3discoverypb.DiscoveryRequest{VersionInfo: "1"}
+	msg, err := proto.Marshal(testDiscoverRequest)
+	if err != nil {
+		t.Fatalf("Failed to marshal DiscoveryRequest: %v", err)
+	}
+	if err := stream.Send(msg); err != nil {
+		t.Fatalf("Failed to send message: %v", err)
+	}
+
+	// Verify that the DiscoveryRequest received on the server was same as
+	// sent.
+	select {
+	case gotReq := <-ts.requestChan:
+		if diff := cmp.Diff(testDiscoverRequest, gotReq, protocmp.Transform()); diff != "" {
+			t.Fatalf("Unexpected diff in request received on server (-want +got):\n%s", diff)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for request to reach server")
+	}
+
+	// Wait until response message is received from the server.
+	res, err := stream.Recv()
+	if err != nil {
+		t.Fatalf("Failed to receive message: %v", err)
+	}
+
+	// Verify that the DiscoveryResponse received was same as sent from the
+	// server.
+	var gotRes v3discoverypb.DiscoveryResponse
+	if err := proto.Unmarshal(res, &gotRes); err != nil {
+		t.Fatalf("Failed to unmarshal response from server to DiscoveryResponse: %v", err)
+	}
+	if diff := cmp.Diff(ts.response, &gotRes, protocmp.Transform()); diff != "" {
+		t.Fatalf("proto.Unmarshal(res, &gotRes) returned unexpected diff (-want +got):\n%s", diff)
+	}
+}

internal/xds/clients/internal/backoff/backoff.go

@@ -0,0 +1,124 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package backoff implements the backoff strategy for clients.
+//
+// This is kept in internal until the clients project decides whether or not to
+// allow alternative backoff strategies.
+package backoff
+
+import (
+	"context"
+	"errors"
+	rand "math/rand/v2"
+	"time"
+)
+
+// config defines the configuration options for backoff.
+type config struct {
+	// baseDelay is the amount of time to backoff after the first failure.
+	baseDelay time.Duration
+	// multiplier is the factor with which to multiply backoffs after a
+	// failed retry. Should ideally be greater than 1.
+	multiplier float64
+	// jitter is the factor with which backoffs are randomized.
+	jitter float64
+	// maxDelay is the upper bound of backoff delay.
+	maxDelay time.Duration
+}
+
+// defaultConfig is a backoff configuration with the default values specified
+// at https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+//
+// This should be useful for callers who want to configure backoff with
+// non-default values only for a subset of the options.
+var defaultConfig = config{
+	baseDelay:  1.0 * time.Second,
+	multiplier: 1.6,
+	jitter:     0.2,
+	maxDelay:   120 * time.Second,
+}
+
+// DefaultExponential is an exponential backoff implementation using the
+// default values for all the configurable knobs defined in
+// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+var DefaultExponential = exponential{config: defaultConfig}
+
+// exponential implements exponential backoff algorithm as defined in
+// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+type exponential struct {
+	// Config contains all options to configure the backoff algorithm.
+	config config
+}
+
+// Backoff returns the amount of time to wait before the next retry given the
+// number of retries.
+func (bc exponential) Backoff(retries int) time.Duration {
+	if retries == 0 {
+		return bc.config.baseDelay
+	}
+	backoff, max := float64(bc.config.baseDelay), float64(bc.config.maxDelay)
+	for backoff < max && retries > 0 {
+		backoff *= bc.config.multiplier
+		retries--
+	}
+	if backoff > max {
+		backoff = max
+	}
+	// Randomize backoff delays so that if a cluster of requests start at
+	// the same time, they won't operate in lockstep.
+	backoff *= 1 + bc.config.jitter*(rand.Float64()*2-1)
+	if backoff < 0 {
+		return 0
+	}
+	return time.Duration(backoff)
+}
+
+// ErrResetBackoff is the error to be returned by the function executed by RunF,
+// to instruct the latter to reset its backoff state.
+var ErrResetBackoff = errors.New("reset backoff state")
+
+// RunF provides a convenient way to run a function f repeatedly until the
+// context expires or f returns a non-nil error that is not ErrResetBackoff.
+// When f returns ErrResetBackoff, RunF continues to run f, but resets its
+// backoff state before doing so. backoff accepts an integer representing the
+// number of retries, and returns the amount of time to backoff.
+func RunF(ctx context.Context, f func() error, backoff func(int) time.Duration) {
+	attempt := 0
+	timer := time.NewTimer(0)
+	for ctx.Err() == nil {
+		select {
+		case <-timer.C:
+		case <-ctx.Done():
+			timer.Stop()
+			return
+		}
+
+		err := f()
+		if errors.Is(err, ErrResetBackoff) {
+			timer.Reset(0)
+			attempt = 0
+			continue
+		}
+		if err != nil {
+			return
+		}
+		timer.Reset(backoff(attempt))
+		attempt++
+	}
+}

internal/xds/clients/internal/buffer/unbounded.go

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package buffer provides an implementation of an unbounded buffer.
+package buffer
+
+import (
+	"errors"
+	"sync"
+)
+
+// Unbounded is an implementation of an unbounded buffer which does not use
+// extra goroutines. This is typically used for passing updates from one entity
+// to another within gRPC.
+//
+// To avoid extra memory allocations and type assertions, using any on
+// performance-critical code paths is discouraged. Use concrete types wherever
+// possible when instantiating Unbounded.
+//
+// All methods on this type are thread-safe and don't block on anything except
+// the underlying mutex used for synchronization.
+type Unbounded[T any] struct {
+	c       chan T
+	closed  bool
+	closing bool
+	mu      sync.Mutex
+	backlog []T
+}
+
+// NewUnbounded returns a new instance of Unbounded.
+func NewUnbounded[T any]() *Unbounded[T] {
+	return &Unbounded[T]{c: make(chan T, 1)}
+}
+
+var errBufferClosed = errors.New("Put() called on closed buffer.Unbounded")
+
+// Put adds t to the unbounded buffer.
+func (b *Unbounded[T]) Put(t T) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.closing {
+		return errBufferClosed
+	}
+	if len(b.backlog) == 0 {
+		select {
+		case b.c <- t:
+			return nil
+		default:
+		}
+	}
+	b.backlog = append(b.backlog, t)
+	return nil
+}
+
+// Load sends the earliest buffered data, if any, onto the read channel returned
+// by Get(). Users are expected to call this every time they successfully read a
+// value from the read channel.
+func (b *Unbounded[T]) Load() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if len(b.backlog) > 0 {
+		select {
+		case b.c <- b.backlog[0]:
+			var zero T
+			b.backlog[0] = zero
+			b.backlog = b.backlog[1:]
+		default:
+		}
+	} else if b.closing && !b.closed {
+		b.closed = true
+		close(b.c)
+	}
+}
+
+// Get returns a read channel on which values added to the buffer, via Put(),
+// are sent on.
+//
+// Upon reading a value from this channel, users are expected to call Load() to
+// send the next buffered value onto the channel if there is any.
+//
+// If the unbounded buffer is closed, the read channel returned by this method
+// is closed after all data is drained.
+func (b *Unbounded[T]) Get() <-chan T {
+	return b.c
+}
+
+// Close closes the unbounded buffer. No subsequent data may be Put(), and the
+// channel returned from Get() will be closed after all the data is read and
+// Load() is called for the final time.
+func (b *Unbounded[T]) Close() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if b.closing {
+		return
+	}
+	b.closing = true
+	if len(b.backlog) == 0 {
+		b.closed = true
+		close(b.c)
+	}
+}

internal/xds/clients/internal/buffer/unbounded_test.go

@@ -0,0 +1,148 @@
+/*
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package buffer
+
+import (
+	"sort"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+const (
+	numWriters = 10
+	numWrites  = 10
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// wantReads contains the set of values expected to be read by the reader
+// goroutine in the tests.
+var wantReads []int
+
+func init() {
+	for i := 0; i < numWriters; i++ {
+		for j := 0; j < numWrites; j++ {
+			wantReads = append(wantReads, i)
+		}
+	}
+}
+
+// TestSingleWriter starts one reader and one writer goroutine and makes sure
+// that the reader gets all the values added to the buffer by the writer.
+func (s) TestSingleWriter(t *testing.T) {
+	ub := NewUnbounded[int]()
+	reads := []int{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ch := ub.Get()
+		for i := 0; i < numWriters*numWrites; i++ {
+			r := <-ch
+			reads = append(reads, r)
+			ub.Load()
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < numWriters; i++ {
+			for j := 0; j < numWrites; j++ {
+				ub.Put(i)
+			}
+		}
+	}()
+
+	wg.Wait()
+	if !cmp.Equal(reads, wantReads) {
+		t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads)
+	}
+}
+
+// TestMultipleWriters starts multiple writers and one reader goroutine and
+// makes sure that the reader gets all the data written by all writers.
+func (s) TestMultipleWriters(t *testing.T) {
+	ub := NewUnbounded[int]()
+	reads := []int{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		ch := ub.Get()
+		for i := 0; i < numWriters*numWrites; i++ {
+			r := <-ch
+			reads = append(reads, r)
+			ub.Load()
+		}
+	}()
+
+	wg.Add(numWriters)
+	for i := 0; i < numWriters; i++ {
+		go func(index int) {
+			defer wg.Done()
+			for j := 0; j < numWrites; j++ {
+				ub.Put(index)
+			}
+		}(i)
+	}
+
+	wg.Wait()
+	sort.Ints(reads)
+	if !cmp.Equal(reads, wantReads) {
+		t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads)
+	}
+}
+
+// TestClose closes the buffer and makes sure that nothing is sent after the
+// buffer is closed.
+func (s) TestClose(t *testing.T) {
+	ub := NewUnbounded[int]()
+	if err := ub.Put(1); err != nil {
+		t.Fatalf("Unbounded.Put() = %v; want nil", err)
+	}
+	ub.Close()
+	if err := ub.Put(1); err == nil {
+		t.Fatalf("Unbounded.Put() = <nil>; want non-nil error")
+	}
+	if v, ok := <-ub.Get(); !ok {
+		t.Errorf("Unbounded.Get() = %v, %v, want %v, %v", v, ok, 1, true)
+	}
+	if err := ub.Put(1); err == nil {
+		t.Fatalf("Unbounded.Put() = <nil>; want non-nil error")
+	}
+	ub.Load()
+	if v, ok := <-ub.Get(); ok {
+		t.Errorf("Unbounded.Get() = %v, want closed channel", v)
+	}
+	if err := ub.Put(1); err == nil {
+		t.Fatalf("Unbounded.Put() = <nil>; want non-nil error")
+	}
+	ub.Close() // ignored
+}

internal/xds/clients/internal/internal.go

@@ -0,0 +1,71 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package internal contains helpers for xDS and LRS clients.
+package internal
+
+import (
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+)
+
+// NodeProto returns a protobuf representation of clients.Node n.
+//
+// This function is intended to be used by the client implementation to convert
+// the user-provided Node configuration to its protobuf representation.
+func NodeProto(n clients.Node) *v3corepb.Node {
+	return &v3corepb.Node{
+		Id:      n.ID,
+		Cluster: n.Cluster,
+		Locality: func() *v3corepb.Locality {
+			if isLocalityEmpty(n.Locality) {
+				return nil
+			}
+			return &v3corepb.Locality{
+				Region:  n.Locality.Region,
+				Zone:    n.Locality.Zone,
+				SubZone: n.Locality.SubZone,
+			}
+		}(),
+		Metadata: func() *structpb.Struct {
+			if n.Metadata == nil {
+				return nil
+			}
+			if md, ok := n.Metadata.(*structpb.Struct); ok {
+				return proto.Clone(md).(*structpb.Struct)
+			}
+			return nil
+		}(),
+		UserAgentName:        n.UserAgentName,
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: n.UserAgentVersion},
+	}
+}
+
+// isLocalityEqual reports whether clients.Locality l is considered empty.
+func isLocalityEmpty(l clients.Locality) bool {
+	return isLocalityEqual(l, clients.Locality{})
+}
+
+// isLocalityEqual returns true if clients.Locality l1 and l2 are considered
+// equal.
+func isLocalityEqual(l1, l2 clients.Locality) bool {
+	return l1.Region == l2.Region && l1.Zone == l2.Zone && l1.SubZone == l2.SubZone
+}

internal/xds/clients/internal/internal_test.go

@@ -0,0 +1,213 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func newStructProtoFromMap(t *testing.T, input map[string]any) *structpb.Struct {
+	t.Helper()
+
+	ret, err := structpb.NewStruct(input)
+	if err != nil {
+		t.Fatalf("Failed to create new struct proto from map %v: %v", input, err)
+	}
+	return ret
+}
+
+func (s) TestIsLocalityEmpty(t *testing.T) {
+	tests := []struct {
+		name     string
+		locality clients.Locality
+		want     bool
+	}{
+		{
+			name:     "empty_locality",
+			locality: clients.Locality{},
+			want:     true,
+		},
+		{
+			name:     "non_empty_region",
+			locality: clients.Locality{Region: "region"},
+			want:     false,
+		},
+		{
+			name:     "non_empty_zone",
+			locality: clients.Locality{Zone: "zone"},
+			want:     false,
+		},
+		{
+			name:     "non_empty_subzone",
+			locality: clients.Locality{SubZone: "subzone"},
+			want:     false,
+		},
+		{
+			name:     "non_empty_all_fields",
+			locality: clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"},
+			want:     false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := isLocalityEmpty(test.locality); got != test.want {
+				t.Errorf("IsEmpty() = %v, want %v", got, test.want)
+			}
+		})
+	}
+}
+
+func (s) TestIsLocalityEqual(t *testing.T) {
+	tests := []struct {
+		name   string
+		l1     clients.Locality
+		l2     clients.Locality
+		wantEq bool
+	}{
+		{
+			name:   "both_equal",
+			l1:     clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"},
+			l2:     clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"},
+			wantEq: true,
+		},
+		{
+			name:   "different_regions",
+			l1:     clients.Locality{Region: "region1", Zone: "zone", SubZone: "subzone"},
+			l2:     clients.Locality{Region: "region2", Zone: "zone", SubZone: "subzone"},
+			wantEq: false,
+		},
+
+		{
+			name:   "different_zones",
+			l1:     clients.Locality{Region: "region", Zone: "zone1", SubZone: "subzone"},
+			l2:     clients.Locality{Region: "region", Zone: "zone2", SubZone: "subzone"},
+			wantEq: false,
+		},
+		{
+			name:   "different_subzones",
+			l1:     clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone1"},
+			l2:     clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone2"},
+			wantEq: false,
+		},
+		{
+			name:   "one_empty",
+			l1:     clients.Locality{},
+			l2:     clients.Locality{Region: "region", Zone: "zone", SubZone: "subzone"},
+			wantEq: false,
+		},
+		{
+			name:   "both_empty",
+			l1:     clients.Locality{},
+			l2:     clients.Locality{},
+			wantEq: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if gotEq := isLocalityEqual(test.l1, test.l2); gotEq != test.wantEq {
+				t.Errorf("Equal() = %v, want %v", gotEq, test.wantEq)
+			}
+		})
+	}
+}
+
+func (s) TestNodeProto(t *testing.T) {
+	tests := []struct {
+		desc      string
+		inputNode clients.Node
+		wantProto *v3corepb.Node
+	}{
+		{
+			desc: "all_fields_set",
+			inputNode: clients.Node{
+				ID:      "id",
+				Cluster: "cluster",
+				Locality: clients.Locality{
+					Region:  "region",
+					Zone:    "zone",
+					SubZone: "sub_zone",
+				},
+				Metadata:         newStructProtoFromMap(t, map[string]any{"k1": "v1", "k2": 101, "k3": 280.0}),
+				UserAgentName:    "user agent",
+				UserAgentVersion: "version",
+			},
+			wantProto: &v3corepb.Node{
+				Id:      "id",
+				Cluster: "cluster",
+				Locality: &v3corepb.Locality{
+					Region:  "region",
+					Zone:    "zone",
+					SubZone: "sub_zone",
+				},
+				Metadata:             newStructProtoFromMap(t, map[string]any{"k1": "v1", "k2": 101, "k3": 280.0}),
+				UserAgentName:        "user agent",
+				UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "version"},
+			},
+		},
+		{
+			desc: "some_fields_unset",
+			inputNode: clients.Node{
+				ID: "id",
+			},
+			wantProto: &v3corepb.Node{
+				Id:                   "id",
+				UserAgentName:        "",
+				UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: ""},
+			},
+		},
+		{
+			desc: "empty_locality",
+			inputNode: clients.Node{
+				ID:       "id",
+				Locality: clients.Locality{},
+			},
+			wantProto: &v3corepb.Node{
+				Id:                   "id",
+				UserAgentName:        "",
+				UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: ""},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			gotProto := NodeProto(test.inputNode)
+			if diff := cmp.Diff(test.wantProto, gotProto, protocmp.Transform()); diff != "" {
+				t.Fatalf("Unexpected diff in node proto: (-want, +got):\n%s", diff)
+			}
+		})
+	}
+}

internal/xds/clients/internal/pretty/pretty.go

@@ -0,0 +1,73 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package pretty defines helper functions to pretty-print structs for logging.
+package pretty
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/protoadapt"
+)
+
+const jsonIndent = "  "
+
+// ToJSON marshals the input into a json string.
+//
+// If marshal fails, it falls back to fmt.Sprintf("%+v").
+func ToJSON(e any) string {
+	if ee, ok := e.(protoadapt.MessageV1); ok {
+		e = protoadapt.MessageV2Of(ee)
+	}
+
+	if ee, ok := e.(protoadapt.MessageV2); ok {
+		mm := protojson.MarshalOptions{
+			Indent:    jsonIndent,
+			Multiline: true,
+		}
+		ret, err := mm.Marshal(ee)
+		if err != nil {
+			// This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2
+			// messages are not imported, and this will fail because the message
+			// is not found.
+			return fmt.Sprintf("%+v", ee)
+		}
+		return string(ret)
+	}
+
+	ret, err := json.MarshalIndent(e, "", jsonIndent)
+	if err != nil {
+		return fmt.Sprintf("%+v", e)
+	}
+	return string(ret)
+}
+
+// FormatJSON formats the input json bytes with indentation.
+//
+// If Indent fails, it returns the unchanged input as string.
+func FormatJSON(b []byte) string {
+	var out bytes.Buffer
+	err := json.Indent(&out, b, "", jsonIndent)
+	if err != nil {
+		return string(b)
+	}
+	return out.String()
+}

internal/xds/clients/internal/syncutil/callback_serializer.go

@@ -0,0 +1,98 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package syncutil
+
+import (
+	"context"
+
+	"google.golang.org/grpc/internal/xds/clients/internal/buffer"
+)
+
+// CallbackSerializer provides a mechanism to schedule callbacks in a
+// synchronized manner. It provides a FIFO guarantee on the order of execution
+// of scheduled callbacks. New callbacks can be scheduled by invoking the
+// Schedule() method.
+//
+// This type is safe for concurrent access.
+type CallbackSerializer struct {
+	// done is closed once the serializer is shut down completely, i.e all
+	// scheduled callbacks are executed and the serializer has deallocated all
+	// its resources.
+	done chan struct{}
+
+	callbacks *buffer.Unbounded[func(context.Context)]
+}
+
+// NewCallbackSerializer returns a new CallbackSerializer instance. The provided
+// context will be passed to the scheduled callbacks. Users should cancel the
+// provided context to shutdown the CallbackSerializer. It is guaranteed that no
+// callbacks will be added once this context is canceled, and any pending un-run
+// callbacks will be executed before the serializer is shut down.
+func NewCallbackSerializer(ctx context.Context) *CallbackSerializer {
+	cs := &CallbackSerializer{
+		done:      make(chan struct{}),
+		callbacks: buffer.NewUnbounded[func(context.Context)](),
+	}
+	go cs.run(ctx)
+	return cs
+}
+
+// TrySchedule tries to schedule the provided callback function f to be
+// executed in the order it was added. This is a best-effort operation. If the
+// context passed to NewCallbackSerializer was canceled before this method is
+// called, the callback will not be scheduled.
+//
+// Callbacks are expected to honor the context when performing any blocking
+// operations, and should return early when the context is canceled.
+func (cs *CallbackSerializer) TrySchedule(f func(ctx context.Context)) {
+	cs.callbacks.Put(f)
+}
+
+// ScheduleOr schedules the provided callback function f to be executed in the
+// order it was added. If the context passed to NewCallbackSerializer has been
+// canceled before this method is called, the onFailure callback will be
+// executed inline instead.
+//
+// Callbacks are expected to honor the context when performing any blocking
+// operations, and should return early when the context is canceled.
+func (cs *CallbackSerializer) ScheduleOr(f func(ctx context.Context), onFailure func()) {
+	if cs.callbacks.Put(f) != nil {
+		onFailure()
+	}
+}
+
+func (cs *CallbackSerializer) run(ctx context.Context) {
+	defer close(cs.done)
+
+	// Close the buffer when the context is canceled
+	// to prevent new callbacks from being added.
+	context.AfterFunc(ctx, cs.callbacks.Close)
+
+	// Run all callbacks.
+	for cb := range cs.callbacks.Get() {
+		cs.callbacks.Load()
+		cb(ctx)
+	}
+}
+
+// Done returns a channel that is closed after the context passed to
+// NewCallbackSerializer is canceled and all callbacks have been executed.
+func (cs *CallbackSerializer) Done() <-chan struct{} {
+	return cs.done
+}

internal/xds/clients/internal/syncutil/callback_serializer_test.go

@@ -0,0 +1,206 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package syncutil
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+// TestCallbackSerializer_Schedule_FIFO verifies that callbacks are executed in
+// the same order in which they were scheduled.
+func (s) TestCallbackSerializer_Schedule_FIFO(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	cs := NewCallbackSerializer(ctx)
+	defer cancel()
+
+	// We have two channels, one to record the order of scheduling, and the
+	// other to record the order of execution. We spawn a bunch of goroutines
+	// which record the order of scheduling and call the actual Schedule()
+	// method as well.  The callbacks record the order of execution.
+	//
+	// We need to grab a lock to record order of scheduling to guarantee that
+	// the act of recording and the act of calling Schedule() happen atomically.
+	const numCallbacks = 100
+	var mu sync.Mutex
+	scheduleOrderCh := make(chan int, numCallbacks)
+	executionOrderCh := make(chan int, numCallbacks)
+	for i := 0; i < numCallbacks; i++ {
+		go func(id int) {
+			mu.Lock()
+			defer mu.Unlock()
+			scheduleOrderCh <- id
+			cs.TrySchedule(func(ctx context.Context) {
+				select {
+				case <-ctx.Done():
+					return
+				case executionOrderCh <- id:
+				}
+			})
+		}(i)
+	}
+
+	// Spawn a couple of goroutines to capture the order or scheduling and the
+	// order of execution.
+	scheduleOrder := make([]int, numCallbacks)
+	executionOrder := make([]int, numCallbacks)
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < numCallbacks; i++ {
+			select {
+			case <-ctx.Done():
+				return
+			case id := <-scheduleOrderCh:
+				scheduleOrder[i] = id
+			}
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for i := 0; i < numCallbacks; i++ {
+			select {
+			case <-ctx.Done():
+				return
+			case id := <-executionOrderCh:
+				executionOrder[i] = id
+			}
+		}
+	}()
+	wg.Wait()
+
+	if diff := cmp.Diff(executionOrder, scheduleOrder); diff != "" {
+		t.Fatalf("Callbacks are not executed in scheduled order. diff(-want, +got):\n%s", diff)
+	}
+}
+
+// TestCallbackSerializer_Schedule_Concurrent verifies that all concurrently
+// scheduled callbacks get executed.
+func (s) TestCallbackSerializer_Schedule_Concurrent(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	cs := NewCallbackSerializer(ctx)
+	defer cancel()
+
+	// Schedule callbacks concurrently by calling Schedule() from goroutines.
+	// The execution of the callbacks call Done() on the waitgroup, which
+	// eventually unblocks the test and allows it to complete.
+	const numCallbacks = 100
+	var wg sync.WaitGroup
+	wg.Add(numCallbacks)
+	for i := 0; i < numCallbacks; i++ {
+		go func() {
+			cs.TrySchedule(func(context.Context) {
+				wg.Done()
+			})
+		}()
+	}
+
+	// We call Wait() on the waitgroup from a goroutine so that we can select on
+	// the Wait() being unblocked and the overall test deadline expiring.
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for all scheduled callbacks to be executed")
+	case <-done:
+	}
+}
+
+// TestCallbackSerializer_Schedule_Close verifies that callbacks in the queue
+// are not executed once Close() returns.
+func (s) TestCallbackSerializer_Schedule_Close(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	serializerCtx, serializerCancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	cs := NewCallbackSerializer(serializerCtx)
+
+	// Schedule a callback which blocks until the context passed to it is
+	// canceled. It also closes a channel to signal that it has started.
+	firstCallbackStartedCh := make(chan struct{})
+	cs.TrySchedule(func(ctx context.Context) {
+		close(firstCallbackStartedCh)
+		<-ctx.Done()
+	})
+
+	// Schedule a bunch of callbacks. These should be executed since they are
+	// scheduled before the serializer is closed.
+	const numCallbacks = 10
+	callbackCh := make(chan int, numCallbacks)
+	for i := 0; i < numCallbacks; i++ {
+		num := i
+		callback := func(context.Context) { callbackCh <- num }
+		onFailure := func() { t.Fatal("Schedule failed to accept a callback when the serializer is yet to be closed") }
+		cs.ScheduleOr(callback, onFailure)
+	}
+
+	// Ensure that none of the newer callbacks are executed at this point.
+	select {
+	case <-time.After(defaultTestShortTimeout):
+	case <-callbackCh:
+		t.Fatal("Newer callback executed when older one is still executing")
+	}
+
+	// Wait for the first callback to start before closing the scheduler.
+	<-firstCallbackStartedCh
+
+	// Cancel the context which will unblock the first callback. All of the
+	// other callbacks (which have not started executing at this point) should
+	// be executed after this.
+	serializerCancel()
+
+	// Ensure that the newer callbacks are executed.
+	for i := 0; i < numCallbacks; i++ {
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for callback scheduled before close to be executed")
+		case num := <-callbackCh:
+			if num != i {
+				t.Fatalf("Executing callback %d, want %d", num, i)
+			}
+		}
+	}
+	<-cs.Done()
+
+	// Ensure that a callback cannot be scheduled after the serializer is
+	// closed.
+	done := make(chan struct{})
+	callback := func(context.Context) { t.Fatal("Scheduled a callback after closing the serializer") }
+	onFailure := func() { close(done) }
+	cs.ScheduleOr(callback, onFailure)
+	select {
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Successfully scheduled callback after serializer is closed")
+	case <-done:
+	}
+}

internal/xds/clients/internal/syncutil/event.go

@@ -0,0 +1,61 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package syncutil implements additional synchronization primitives built upon
+// the sync package.
+package syncutil
+
+import (
+	"sync"
+	"sync/atomic"
+)
+
+// Event represents a one-time event that may occur in the future.
+type Event struct {
+	fired int32
+	c     chan struct{}
+	o     sync.Once
+}
+
+// Fire causes e to complete.  It is safe to call multiple times, and
+// concurrently.  It returns true iff this call to Fire caused the signaling
+// channel returned by Done to close.
+func (e *Event) Fire() bool {
+	ret := false
+	e.o.Do(func() {
+		atomic.StoreInt32(&e.fired, 1)
+		close(e.c)
+		ret = true
+	})
+	return ret
+}
+
+// Done returns a channel that will be closed when Fire is called.
+func (e *Event) Done() <-chan struct{} {
+	return e.c
+}
+
+// HasFired returns true if Fire has been called.
+func (e *Event) HasFired() bool {
+	return atomic.LoadInt32(&e.fired) == 1
+}
+
+// NewEvent returns a new, ready-to-use Event.
+func NewEvent() *Event {
+	return &Event{c: make(chan struct{})}
+}

internal/xds/clients/internal/syncutil/event_test.go

@@ -0,0 +1,81 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package syncutil
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestEventHasFired(t *testing.T) {
+	e := NewEvent()
+	if e.HasFired() {
+		t.Fatal("e.HasFired() = true; want false")
+	}
+	if !e.Fire() {
+		t.Fatal("e.Fire() = false; want true")
+	}
+	if !e.HasFired() {
+		t.Fatal("e.HasFired() = false; want true")
+	}
+}
+
+func (s) TestEventDoneChannel(t *testing.T) {
+	e := NewEvent()
+	select {
+	case <-e.Done():
+		t.Fatal("e.HasFired() = true; want false")
+	default:
+	}
+	if !e.Fire() {
+		t.Fatal("e.Fire() = false; want true")
+	}
+	select {
+	case <-e.Done():
+	default:
+		t.Fatal("e.HasFired() = false; want true")
+	}
+}
+
+func (s) TestEventMultipleFires(t *testing.T) {
+	e := NewEvent()
+	if e.HasFired() {
+		t.Fatal("e.HasFired() = true; want false")
+	}
+	if !e.Fire() {
+		t.Fatal("e.Fire() = false; want true")
+	}
+	for i := 0; i < 3; i++ {
+		if !e.HasFired() {
+			t.Fatal("e.HasFired() = false; want true")
+		}
+		if e.Fire() {
+			t.Fatal("e.Fire() = true; want false")
+		}
+	}
+}

internal/xds/clients/internal/testutils/channel.go

@@ -0,0 +1,87 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package testutils contains testing helpers for xDS and LRS clients.
+package testutils
+
+import "context"
+
+// Channel wraps a generic channel and provides a timed receive operation.
+type Channel struct {
+	// C is the underlying channel on which values sent using the SendXxx()
+	// methods are delivered. Tests which cannot use ReceiveXxx() for whatever
+	// reasons can use C to read the values.
+	C chan any
+}
+
+// Send sends value on the underlying channel.
+func (c *Channel) Send(value any) {
+	c.C <- value
+}
+
+// Receive returns the value received on the underlying channel, or the error
+// returned by ctx if it is closed or cancelled.
+func (c *Channel) Receive(ctx context.Context) (any, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case got := <-c.C:
+		return got, nil
+	}
+}
+
+// Replace clears the value on the underlying channel, and sends the new value.
+//
+// It's expected to be used with a size-1 channel, to only keep the most
+// up-to-date item. This method is inherently racy when invoked concurrently
+// from multiple goroutines.
+func (c *Channel) Replace(value any) {
+	for {
+		select {
+		case c.C <- value:
+			return
+		case <-c.C:
+		}
+	}
+}
+
+// SendContext sends value on the underlying channel, or returns an error if
+// the context expires.
+func (c *Channel) SendContext(ctx context.Context, value any) error {
+	select {
+	case c.C <- value:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+// Drain drains the channel by repeatedly reading from it until it is empty.
+func (c *Channel) Drain() {
+	for {
+		select {
+		case <-c.C:
+		default:
+			return
+		}
+	}
+}
+
+// NewChannelWithSize returns a new Channel with a buffer of bufSize.
+func NewChannelWithSize(bufSize int) *Channel {
+	return &Channel{C: make(chan any, bufSize)}
+}

internal/xds/clients/internal/testutils/e2e/clientresources.go

@@ -0,0 +1,75 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+import (
+	"fmt"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+)
+
+const (
+	// ClientSideCertProviderInstance is the certificate provider instance name
+	// used in the Cluster resource on the client side.
+	ClientSideCertProviderInstance = "client-side-certificate-provider-instance"
+)
+
+// RouterHTTPFilter is the HTTP Filter configuration for the Router filter.
+var RouterHTTPFilter = HTTPFilter("router", &v3routerpb.Router{})
+
+// DefaultClientListener returns a basic xds Listener resource to be used on
+// the client side.
+func DefaultClientListener(target, routeName string) *v3listenerpb.Listener {
+	hcm := marshalAny(&v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{
+			ConfigSource: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+			},
+			RouteConfigName: routeName,
+		}},
+	})
+	return &v3listenerpb.Listener{
+		Name:        target,
+		ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
+	}
+}
+
+func marshalAny(m proto.Message) *anypb.Any {
+	a, err := anypb.New(m)
+	if err != nil {
+		panic(fmt.Sprintf("anypb.New(%+v) failed: %v", m, err))
+	}
+	return a
+}
+
+// HTTPFilter constructs an xds HttpFilter with the provided name and config.
+func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter {
+	return &v3httppb.HttpFilter{
+		Name: name,
+		ConfigType: &v3httppb.HttpFilter_TypedConfig{
+			TypedConfig: marshalAny(config),
+		},
+	}
+}

internal/xds/clients/internal/testutils/e2e/logging.go

@@ -0,0 +1,40 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package e2e
+
+// serverLogger implements the Logger interface defined at
+// envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache.
+type serverLogger struct {
+	logger interface {
+		Logf(format string, args ...any)
+	}
+}
+
+func (l serverLogger) Debugf(format string, args ...any) {
+	l.logger.Logf(format, args...)
+}
+func (l serverLogger) Infof(format string, args ...any) {
+	l.logger.Logf(format, args...)
+}
+func (l serverLogger) Warnf(format string, args ...any) {
+	l.logger.Logf(format, args...)
+}
+func (l serverLogger) Errorf(format string, args ...any) {
+	l.logger.Logf(format, args...)
+}

internal/xds/clients/internal/testutils/e2e/server.go

@@ -0,0 +1,257 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package e2e provides utilities for end2end testing of xDS and LRS clients
+// functionalities.
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/envoyproxy/go-control-plane/pkg/cache/types"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+	v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3"
+	v3resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3"
+	v3server "github.com/envoyproxy/go-control-plane/pkg/server/v3"
+)
+
+// ManagementServer is a thin wrapper around the xDS control plane
+// implementation provided by envoyproxy/go-control-plane.
+type ManagementServer struct {
+	// Address is the host:port on which the management server is listening for
+	// new connections.
+	Address string
+
+	// LRSServer points to the fake LRS server implementation. Set only if the
+	// SupportLoadReportingService option was set to true when creating this
+	// management server.
+	LRSServer *fakeserver.Server
+
+	cancel  context.CancelFunc    // To stop the v3 ADS service.
+	xs      v3server.Server       // v3 implementation of ADS.
+	gs      *grpc.Server          // gRPC server which exports the ADS service.
+	cache   v3cache.SnapshotCache // Resource snapshot.
+	version int                   // Version of resource snapshot.
+
+	// A logging interface, usually supplied from *testing.T.
+	logger interface {
+		Logf(format string, args ...any)
+	}
+}
+
+// ManagementServerOptions contains options to be passed to the management
+// server during creation.
+type ManagementServerOptions struct {
+	// Listener to accept connections on. If nil, a TPC listener on a local port
+	// will be created and used.
+	Listener net.Listener
+
+	// SupportLoadReportingService, if set, results in the load reporting
+	// service being registered on the same port as that of ADS.
+	SupportLoadReportingService bool
+
+	// AllowResourceSubSet allows the management server to respond to requests
+	// before all configured resources are explicitly named in the request. The
+	// default behavior that we want is for the management server to wait for
+	// all configured resources to be requested before responding to any of
+	// them, since this is how we have run our tests historically, and should be
+	// set to true only for tests which explicitly require the other behavior.
+	AllowResourceSubset bool
+
+	// ServerFeaturesIgnoreResourceDeletion, if set, results in a bootstrap config
+	// where the server features list contains `ignore_resource_deletion`. This
+	// results in gRPC ignoring resource deletions from the management server, as
+	// per A53.
+	ServerFeaturesIgnoreResourceDeletion bool
+
+	// The callbacks defined below correspond to the state of the world (sotw)
+	// version of the xDS API on the management server.
+
+	// OnStreamOpen is called when an xDS stream is opened. The callback is
+	// invoked with the assigned stream ID and the type URL from the incoming
+	// request (or "" for ADS).
+	//
+	// Returning an error from this callback will end processing and close the
+	// stream. OnStreamClosed will still be called.
+	OnStreamOpen func(context.Context, int64, string) error
+
+	// OnStreamClosed is called immediately prior to closing an xDS stream. The
+	// callback is invoked with the stream ID of the stream being closed.
+	OnStreamClosed func(int64, *v3corepb.Node)
+
+	// OnStreamRequest is called when a request is received on the stream. The
+	// callback is invoked with the stream ID of the stream on which the request
+	// was received and the received request.
+	//
+	// Returning an error from this callback will end processing and close the
+	// stream. OnStreamClosed will still be called.
+	OnStreamRequest func(int64, *v3discoverypb.DiscoveryRequest) error
+
+	// OnStreamResponse is called immediately prior to sending a response on the
+	// stream. The callback is invoked with the stream ID of the stream on which
+	// the response is being sent along with the incoming request and the outgoing
+	// response.
+	OnStreamResponse func(context.Context, int64, *v3discoverypb.DiscoveryRequest, *v3discoverypb.DiscoveryResponse)
+}
+
+// StartManagementServer initializes a management server which implements the
+// AggregatedDiscoveryService endpoint. The management server is initialized
+// with no resources. Tests should call the Update() method to change the
+// resource snapshot held by the management server, as per by the test logic.
+//
+// Registers a cleanup function on t to stop the management server.
+func StartManagementServer(t *testing.T, opts ManagementServerOptions) *ManagementServer {
+	t.Helper()
+
+	// Create a snapshot cache. The first parameter to NewSnapshotCache()
+	// controls whether the server should wait for all resources to be
+	// explicitly named in the request before responding to any of them.
+	wait := !opts.AllowResourceSubset
+	cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{t})
+	t.Logf("Created new snapshot cache...")
+
+	lis := opts.Listener
+	if lis == nil {
+		var err error
+		lis, err = net.Listen("tcp", "localhost:0")
+		if err != nil {
+			t.Fatalf("Failed to listen on localhost:0: %v", err)
+		}
+	}
+
+	// Cancelling the context passed to the server is the only way of stopping it
+	// at the end of the test.
+	ctx, cancel := context.WithCancel(context.Background())
+	callbacks := v3server.CallbackFuncs{
+		StreamOpenFunc:     opts.OnStreamOpen,
+		StreamClosedFunc:   opts.OnStreamClosed,
+		StreamRequestFunc:  opts.OnStreamRequest,
+		StreamResponseFunc: opts.OnStreamResponse,
+	}
+
+	// Create an xDS management server and register the ADS implementation
+	// provided by it on a gRPC server.
+	xs := v3server.NewServer(ctx, cache, callbacks)
+	gs := grpc.NewServer()
+	v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs)
+	t.Logf("Registered Aggregated Discovery Service (ADS)...")
+
+	mgmtServer := &ManagementServer{
+		Address: lis.Addr().String(),
+		cancel:  cancel,
+		version: 0,
+		gs:      gs,
+		xs:      xs,
+		cache:   cache,
+		logger:  t,
+	}
+	if opts.SupportLoadReportingService {
+		lrs := fakeserver.NewServer(lis.Addr().String())
+		v3lrsgrpc.RegisterLoadReportingServiceServer(gs, lrs)
+		mgmtServer.LRSServer = lrs
+		t.Logf("Registered Load Reporting Service (LRS)...")
+	}
+
+	// Start serving.
+	go gs.Serve(lis)
+	t.Logf("xDS management server serving at: %v...", lis.Addr().String())
+	t.Cleanup(mgmtServer.Stop)
+	return mgmtServer
+}
+
+// UpdateOptions wraps parameters to be passed to the Update() method.
+type UpdateOptions struct {
+	// NodeID is the id of the client to which this update is to be pushed.
+	NodeID string
+	// Endpoints, Clusters, Routes, and Listeners are the updated list of xds
+	// resources for the server.  All must be provided with each Update.
+	Endpoints []*v3endpointpb.ClusterLoadAssignment
+	Clusters  []*v3clusterpb.Cluster
+	Routes    []*v3routepb.RouteConfiguration
+	Listeners []*v3listenerpb.Listener
+	// SkipValidation indicates whether we want to skip validation (by not
+	// calling snapshot.Consistent()). It can be useful for negative tests,
+	// where we send updates that the client will NACK.
+	SkipValidation bool
+}
+
+// Update changes the resource snapshot held by the management server, which
+// updates connected clients as required.
+func (s *ManagementServer) Update(ctx context.Context, opts UpdateOptions) error {
+	s.version++
+
+	// Create a snapshot with the passed in resources.
+	resources := map[v3resource.Type][]types.Resource{
+		v3resource.ListenerType: resourceSlice(opts.Listeners),
+		v3resource.RouteType:    resourceSlice(opts.Routes),
+		v3resource.ClusterType:  resourceSlice(opts.Clusters),
+		v3resource.EndpointType: resourceSlice(opts.Endpoints),
+	}
+	snapshot, err := v3cache.NewSnapshot(strconv.Itoa(s.version), resources)
+	if err != nil {
+		return fmt.Errorf("failed to create new snapshot cache: %v", err)
+
+	}
+	if !opts.SkipValidation {
+		if err := snapshot.Consistent(); err != nil {
+			return fmt.Errorf("failed to create new resource snapshot: %v", err)
+		}
+	}
+	s.logger.Logf("Created new resource snapshot...")
+
+	// Update the cache with the new resource snapshot.
+	if err := s.cache.SetSnapshot(ctx, opts.NodeID, snapshot); err != nil {
+		return fmt.Errorf("failed to update resource snapshot in management server: %v", err)
+	}
+	s.logger.Logf("Updated snapshot cache with resource snapshot...")
+	return nil
+}
+
+// Stop stops the management server.
+func (s *ManagementServer) Stop() {
+	if s.cancel != nil {
+		s.cancel()
+	}
+	s.gs.Stop()
+}
+
+// resourceSlice accepts a slice of any type of proto messages and returns a
+// slice of types.Resource.  Will panic if there is an input type mismatch.
+func resourceSlice(i any) []types.Resource {
+	v := reflect.ValueOf(i)
+	rs := make([]types.Resource, v.Len())
+	for i := 0; i < v.Len(); i++ {
+		rs[i] = v.Index(i).Interface().(types.Resource)
+	}
+	return rs
+}

internal/xds/clients/internal/testutils/fakeserver/server.go

@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package fakeserver provides a fake implementation of the management server.
+//
+// This package is recommended only for scenarios which cannot be tested using
+// the xDS management server (which uses envoy-go-control-plane) provided by the
+// `internal/testutils/e2e` package.
+package fakeserver
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+
+	v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+)
+
+const (
+	// TODO: Make this a var or a field in the server if there is a need to use a
+	// value other than this default.
+	defaultChannelBufferSize = 50
+	defaultDialTimeout       = 5 * time.Second
+)
+
+// Request wraps the request protobuf (xds/LRS) and error received by the
+// Server in a call to stream.Recv().
+type Request struct {
+	Req proto.Message
+	Err error
+}
+
+// Response wraps the response protobuf (xds/LRS) and error that the Server
+// should send out to the client through a call to stream.Send()
+type Response struct {
+	Resp proto.Message
+	Err  error
+}
+
+// Server is a fake implementation of xDS and LRS protocols. It listens on the
+// same port for both services and exposes a bunch of channels to send/receive
+// messages.
+//
+// This server is recommended only for scenarios which cannot be tested using
+// the xDS management server (which uses envoy-go-control-plane) provided by the
+// `internal/testutils/xds/e2e` package.
+type Server struct {
+	// XDSRequestChan is a channel on which received xDS requests are made
+	// available to the users of this Server.
+	XDSRequestChan *testutils.Channel
+	// XDSResponseChan is a channel on which the Server accepts xDS responses
+	// to be sent to the client.
+	XDSResponseChan chan *Response
+	// LRSRequestChan is a channel on which received LRS requests are made
+	// available to the users of this Server.
+	LRSRequestChan *testutils.Channel
+	// LRSResponseChan is a channel on which the Server accepts the LRS
+	// response to be sent to the client.
+	LRSResponseChan chan *Response
+	// LRSStreamOpenChan is a channel on which the Server sends notifications
+	// when a new LRS stream is created.
+	LRSStreamOpenChan *testutils.Channel
+	// LRSStreamCloseChan is a channel on which the Server sends notifications
+	// when an existing LRS stream is closed.
+	LRSStreamCloseChan *testutils.Channel
+	// NewConnChan is a channel on which the fake server notifies receipt of new
+	// connection attempts. Tests can gate on this event before proceeding to
+	// other actions which depend on a connection to the fake server being up.
+	NewConnChan *testutils.Channel
+	// Address is the host:port on which the Server is listening for requests.
+	Address string
+
+	// The underlying fake implementation of xDS and LRS.
+	*xdsServer
+	*lrsServer
+}
+
+type wrappedListener struct {
+	net.Listener
+	server *Server
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	c, err := wl.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	wl.server.NewConnChan.Send(struct{}{})
+	return c, err
+}
+
+// StartServer makes a new Server and gets it to start listening on the given
+// net.Listener. If the given net.Listener is nil, a new one is created on a
+// local port for gRPC requests. The returned cancel function should be invoked
+// by the caller upon completion of the test.
+func StartServer(lis net.Listener) (*Server, func(), error) {
+	if lis == nil {
+		var err error
+		lis, err = net.Listen("tcp", "localhost:0")
+		if err != nil {
+			return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err)
+		}
+	}
+
+	s := NewServer(lis.Addr().String())
+	wp := &wrappedListener{
+		Listener: lis,
+		server:   s,
+	}
+
+	server := grpc.NewServer()
+	v3lrsgrpc.RegisterLoadReportingServiceServer(server, s)
+	v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(server, s)
+	go server.Serve(wp)
+
+	return s, func() { server.Stop() }, nil
+}
+
+// NewServer returns a new instance of Server, set to accept requests on addr.
+// It is the responsibility of the caller to register the exported ADS and LRS
+// services on an appropriate gRPC server. Most usages should prefer
+// StartServer() instead of this.
+func NewServer(addr string) *Server {
+	s := &Server{
+		XDSRequestChan:     testutils.NewChannelWithSize(defaultChannelBufferSize),
+		LRSRequestChan:     testutils.NewChannelWithSize(defaultChannelBufferSize),
+		NewConnChan:        testutils.NewChannelWithSize(defaultChannelBufferSize),
+		XDSResponseChan:    make(chan *Response, defaultChannelBufferSize),
+		LRSResponseChan:    make(chan *Response, 1), // The server only ever sends one response.
+		LRSStreamOpenChan:  testutils.NewChannelWithSize(defaultChannelBufferSize),
+		LRSStreamCloseChan: testutils.NewChannelWithSize(defaultChannelBufferSize),
+		Address:            addr,
+	}
+	s.xdsServer = &xdsServer{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan}
+	s.lrsServer = &lrsServer{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan, streamOpenChan: s.LRSStreamOpenChan, streamCloseChan: s.LRSStreamCloseChan}
+	return s
+}
+
+type xdsServer struct {
+	reqChan  *testutils.Channel
+	respChan chan *Response
+}
+
+func (xdsS *xdsServer) StreamAggregatedResources(s v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error {
+	errCh := make(chan error, 2)
+	go func() {
+		for {
+			req, err := s.Recv()
+			if err != nil {
+				errCh <- err
+				return
+			}
+			xdsS.reqChan.Send(&Request{req, err})
+		}
+	}()
+	go func() {
+		var retErr error
+		defer func() {
+			errCh <- retErr
+		}()
+
+		for {
+			select {
+			case r := <-xdsS.respChan:
+				if r.Err != nil {
+					retErr = r.Err
+					return
+				}
+				if err := s.Send(r.Resp.(*v3discoverypb.DiscoveryResponse)); err != nil {
+					retErr = err
+					return
+				}
+			case <-s.Context().Done():
+				retErr = s.Context().Err()
+				return
+			}
+		}
+	}()
+
+	if err := <-errCh; err != nil {
+		return err
+	}
+	return nil
+}
+
+func (xdsS *xdsServer) DeltaAggregatedResources(v3discoverygrpc.AggregatedDiscoveryService_DeltaAggregatedResourcesServer) error {
+	return status.Error(codes.Unimplemented, "")
+}
+
+type lrsServer struct {
+	reqChan         *testutils.Channel
+	respChan        chan *Response
+	streamOpenChan  *testutils.Channel
+	streamCloseChan *testutils.Channel
+}
+
+func (lrsS *lrsServer) StreamLoadStats(s v3lrsgrpc.LoadReportingService_StreamLoadStatsServer) error {
+	lrsS.streamOpenChan.Send(nil)
+	defer lrsS.streamCloseChan.Send(nil)
+
+	req, err := s.Recv()
+	lrsS.reqChan.Send(&Request{req, err})
+	if err != nil {
+		return err
+	}
+
+	select {
+	case r := <-lrsS.respChan:
+		if r.Err != nil {
+			return r.Err
+		}
+		if err := s.Send(r.Resp.(*v3lrspb.LoadStatsResponse)); err != nil {
+			return err
+		}
+	case <-s.Context().Done():
+		return s.Context().Err()
+	}
+
+	for {
+		req, err := s.Recv()
+		lrsS.reqChan.Send(&Request{req, err})
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}

internal/xds/clients/internal/testutils/faketransport/xds_fake_transport.go

@@ -0,0 +1,252 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package faketransport provides a fake implementation of the xDS client's
+// transport layer. It implements the clients.TransportBuilder,
+// clients.Transport and clients.Stream interfaces for testing purposes.
+package faketransport
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/proto"
+
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// This compile time checks ensures that the Builder, transport and stream
+// implementations satisfy the required interfaces.
+var _ clients.TransportBuilder = &Builder{}
+var _ clients.Transport = &transport{}
+var _ clients.Stream = &stream{}
+
+// Builder implements clients.TransportBuilder.
+type Builder struct {
+	mu                   sync.Mutex
+	activeTransports     map[string]*transport    // Tracks created transports for the fuzzer to interact with.
+	activeTransportsChan map[string]chan struct{} // Notifies when transport and stream are ready
+}
+
+// NewBuilder creates a new Builder.
+func NewBuilder() *Builder {
+	return &Builder{
+		activeTransports:     make(map[string]*transport),
+		activeTransportsChan: make(map[string]chan struct{}),
+	}
+}
+
+// Build creates a new Transport for the given server identifier.
+func (b *Builder) Build(serverIdentifier clients.ServerIdentifier) (clients.Transport, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if at, ok := b.activeTransports[serverIdentifier.ServerURI]; ok {
+		return at, nil
+	}
+
+	streamReadyCh, ok := b.activeTransportsChan[serverIdentifier.ServerURI]
+	if !ok {
+		streamReadyCh = make(chan struct{})
+		b.activeTransportsChan[serverIdentifier.ServerURI] = streamReadyCh
+	}
+
+	ft := newTransport(streamReadyCh)
+	b.activeTransports[serverIdentifier.ServerURI] = ft
+	return ft, nil
+}
+
+// Close closes the transport for the given server identifier.
+func (b *Builder) Close(serverURI string) {
+	b.mu.Lock()
+	t, ok := b.activeTransports[serverURI]
+	b.mu.Unlock()
+	if ok {
+		t.Close()
+	}
+}
+
+// Transport returns the active transport for a given server URI.
+func (b *Builder) Transport(ctx context.Context, serverURI string) (*ServerHandle, error) {
+	b.mu.Lock()
+	if t, ok := b.activeTransports[serverURI]; ok && t.serverHandle() != nil {
+		b.mu.Unlock()
+		return t.serverHandle(), nil
+	}
+
+	ch, ok := b.activeTransportsChan[serverURI]
+	if !ok {
+		ch = make(chan struct{})
+		b.activeTransportsChan[serverURI] = ch
+	}
+	b.mu.Unlock()
+
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-ch:
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		return b.activeTransports[serverURI].serverHandle(), nil
+	}
+}
+
+// transport implements clients.Transport.
+type transport struct {
+	mu              sync.Mutex
+	activeADSStream *stream
+	closed          bool
+	streamReady     func()
+}
+
+func newTransport(streamReady chan struct{}) *transport {
+	return &transport{
+		streamReady: sync.OnceFunc(func() {
+			close(streamReady)
+		}),
+	}
+}
+
+// serverHandle returns a serverhandle for testing.
+func (t *transport) serverHandle() *ServerHandle {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.activeADSStream == nil {
+		return nil
+	}
+	return &ServerHandle{fs: t.activeADSStream}
+}
+
+// NewStream creates a new stream to the server.
+func (t *transport) NewStream(ctx context.Context, _ string) (clients.Stream, error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.closed {
+		return nil, fmt.Errorf("transport is closed")
+	}
+
+	fs := newStream(ctx)
+	t.activeADSStream = fs
+	t.streamReady()
+	return fs, nil
+}
+
+// Close closes the stream.
+func (t *transport) Close() {
+	t.mu.Lock()
+	t.closed = true
+	stream := t.activeADSStream
+	t.mu.Unlock()
+	if stream != nil {
+		stream.close()
+	}
+}
+
+// stream implements clients.Stream.
+type stream struct {
+	ctx    context.Context
+	cancel context.CancelFunc
+
+	reqChan  chan []byte
+	respChan chan []byte
+}
+
+func newStream(ctx context.Context) *stream {
+	c, cancel := context.WithCancel(ctx)
+	return &stream{
+		ctx:      c,
+		cancel:   cancel,
+		reqChan:  make(chan []byte),
+		respChan: make(chan []byte),
+	}
+}
+
+// Send sends the provided message on the stream. It puts the request into the
+// reqChan for consumption.
+func (s *stream) Send(data []byte) error {
+	// A short timeout prevents the client from deadlocking if there is no
+	// receiver on the server-side to consume the request (e.g., if the xDS
+	// client sends an ACK after creating a stream but the server handle is
+	// not actively receiving).
+	ctx, cancel := context.WithTimeout(s.ctx, 10*time.Millisecond)
+	defer cancel()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case s.reqChan <- data:
+		return nil
+	}
+}
+
+// Recv blocks until the next message is received on the stream. It blocks until
+// a response is available in the respChan or the context is canceled.
+func (s *stream) Recv() ([]byte, error) {
+	select {
+	case data := <-s.respChan:
+		return data, nil
+	case <-s.ctx.Done():
+		return nil, s.ctx.Err()
+	}
+}
+
+// Close closes the stream.
+func (s *stream) close() {
+	s.cancel()
+}
+
+// ServerHandle provides the server-side send/recv methods to interact with
+// the stream.
+type ServerHandle struct {
+	fs *stream
+}
+
+// Recv reads the next request from the reqChan. It blocks until a
+// request is available or the context expires. It returns an error if the
+// context expires or if the request cannot be unmarshaled.
+func (h *ServerHandle) Recv(ctx context.Context) (*v3discoverypb.DiscoveryRequest, error) {
+	select {
+	case data := <-h.fs.reqChan:
+		req := &v3discoverypb.DiscoveryRequest{}
+		if err := proto.Unmarshal(data, req); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal request: %v", err)
+		}
+		return req, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// Send simulates a server response. It marshals the provided
+// DiscoveryResponse, puts it in the respChan to notify that a response
+// is available for the client to Recv.
+func (h *ServerHandle) Send(ctx context.Context, resp *v3discoverypb.DiscoveryResponse) error {
+	data, err := proto.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case h.fs.respChan <- data:
+		return nil
+	}
+}

internal/xds/clients/internal/testutils/marshal_any.go

@@ -0,0 +1,37 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package testutils
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+)
+
+// MarshalAny is a convenience function to marshal protobuf messages into any
+// protos. function will fail the test with a fatal error if the marshaling fails.
+func MarshalAny(t *testing.T, m proto.Message) *anypb.Any {
+	t.Helper()
+
+	a, err := anypb.New(m)
+	if err != nil {
+		t.Fatalf("Failed to marshal proto %+v into an Any: %v", m, err)
+	}
+	return a
+}

internal/xds/clients/internal/testutils/restartable_listener.go

@@ -0,0 +1,106 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import (
+	"net"
+	"sync"
+
+	"google.golang.org/grpc/grpclog"
+)
+
+var logger = grpclog.Component("testutils")
+
+type tempError struct{}
+
+func (*tempError) Error() string {
+	return "restartable listener temporary error"
+}
+func (*tempError) Temporary() bool {
+	return true
+}
+
+// RestartableListener wraps a net.Listener and supports stopping and restarting
+// the latter.
+type RestartableListener struct {
+	lis net.Listener
+
+	mu      sync.Mutex
+	stopped bool
+	conns   []net.Conn
+}
+
+// NewRestartableListener returns a new RestartableListener wrapping l.
+func NewRestartableListener(l net.Listener) *RestartableListener {
+	return &RestartableListener{lis: l}
+}
+
+// Accept waits for and returns the next connection to the listener.
+//
+// If the listener is currently not accepting new connections, because `Stop`
+// was called on it, the connection is immediately closed after accepting
+// without any bytes being sent on it.
+func (l *RestartableListener) Accept() (net.Conn, error) {
+	conn, err := l.lis.Accept()
+	if err != nil {
+		return nil, err
+	}
+
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.stopped {
+		conn.Close()
+		return nil, &tempError{}
+	}
+	l.conns = append(l.conns, conn)
+	return conn, nil
+}
+
+// Close closes the listener.
+func (l *RestartableListener) Close() error {
+	return l.lis.Close()
+}
+
+// Addr returns the listener's network address.
+func (l *RestartableListener) Addr() net.Addr {
+	return l.lis.Addr()
+}
+
+// Stop closes existing connections on the listener and prevents new connections
+// from being accepted.
+func (l *RestartableListener) Stop() {
+	logger.Infof("Stopping restartable listener %q", l.Addr())
+
+	l.mu.Lock()
+	l.stopped = true
+	for _, conn := range l.conns {
+		conn.Close()
+	}
+	l.conns = nil
+	l.mu.Unlock()
+}
+
+// Restart gets a previously stopped listener to start accepting connections.
+func (l *RestartableListener) Restart() {
+	logger.Infof("Restarting listener %q", l.Addr())
+
+	l.mu.Lock()
+	l.stopped = false
+	l.mu.Unlock()
+}

internal/xds/clients/internal/testutils/wrappers.go

@@ -0,0 +1,74 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package testutils
+
+import (
+	"net"
+	"testing"
+)
+
+// ConnWrapper wraps a net.Conn and pushes on a channel when closed.
+type ConnWrapper struct {
+	net.Conn
+	CloseCh *Channel
+}
+
+// Close closes the connection and sends a value on the close channel.
+func (cw *ConnWrapper) Close() error {
+	err := cw.Conn.Close()
+	cw.CloseCh.Replace(nil)
+	return err
+}
+
+// ListenerWrapper wraps a net.Listener and the returned net.Conn.
+//
+// It pushes on a channel whenever it accepts a new connection.
+type ListenerWrapper struct {
+	net.Listener
+	NewConnCh *Channel
+}
+
+// Accept wraps the Listener Accept and sends the accepted connection on a
+// channel.
+func (l *ListenerWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	closeCh := NewChannelWithSize(1)
+	conn := &ConnWrapper{Conn: c, CloseCh: closeCh}
+	l.NewConnCh.Replace(conn)
+	return conn, nil
+}
+
+// NewListenerWrapper returns a ListenerWrapper.
+func NewListenerWrapper(t *testing.T, lis net.Listener) *ListenerWrapper {
+	if lis == nil {
+		var err error
+		lis, err = net.Listen("tcp", "localhost:0")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	return &ListenerWrapper{
+		Listener:  lis,
+		NewConnCh: NewChannelWithSize(1),
+	}
+}

internal/xds/clients/lrsclient/internal/internal.go

@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package internal contains functionality internal to the lrsclient package.
+package internal
+
+import "time"
+
+var (
+	// TimeNow is used to get the current time. It can be overridden in tests.
+	TimeNow func() time.Time
+)

internal/xds/clients/lrsclient/load_store.go

@@ -0,0 +1,438 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+)
+
+// A LoadStore aggregates loads for multiple clusters and services that are
+// intended to be reported via LRS.
+//
+// LoadStore stores loads reported to a single LRS server. Use multiple stores
+// for multiple servers.
+//
+// It is safe for concurrent use.
+type LoadStore struct {
+	// stop is the function to call to Stop the LoadStore reporting.
+	stop func(ctx context.Context)
+
+	// mu only protects the map (2 layers). The read/write to
+	// *PerClusterReporter doesn't need to hold the mu.
+	mu sync.Mutex
+	// clusters is a map with cluster name as the key. The second layer is a
+	// map with service name as the key. Each value (PerClusterReporter)
+	// contains data for a (cluster, service) pair.
+	//
+	// Note that new entries are added to this map, but never removed. This is
+	// potentially a memory leak. But the memory is allocated for each new
+	// (cluster,service) pair, and the memory allocated is just pointers and
+	// maps. So this shouldn't get too bad.
+	clusters map[string]map[string]*PerClusterReporter
+}
+
+func init() {
+	lrsclientinternal.TimeNow = time.Now
+}
+
+// newLoadStore creates a LoadStore.
+func newLoadStore() *LoadStore {
+	return &LoadStore{
+		clusters: make(map[string]map[string]*PerClusterReporter),
+	}
+}
+
+// Stop signals the LoadStore to stop reporting.
+//
+// Before closing the underlying LRS stream, this method may block until a
+// final load report send attempt completes or the provided context `ctx`
+// expires.
+//
+// The provided context must have a deadline or timeout set to prevent Stop
+// from blocking indefinitely if the final send attempt fails to complete.
+//
+// Calling Stop on an already stopped LoadStore is a no-op.
+func (ls *LoadStore) Stop(ctx context.Context) {
+	ls.stop(ctx)
+}
+
+// ReporterForCluster returns the PerClusterReporter for the given cluster and
+// service.
+func (ls *LoadStore) ReporterForCluster(clusterName, serviceName string) *PerClusterReporter {
+	ls.mu.Lock()
+	defer ls.mu.Unlock()
+	c, ok := ls.clusters[clusterName]
+	if !ok {
+		c = make(map[string]*PerClusterReporter)
+		ls.clusters[clusterName] = c
+	}
+
+	if p, ok := c[serviceName]; ok {
+		return p
+	}
+	p := &PerClusterReporter{
+		cluster:          clusterName,
+		service:          serviceName,
+		lastLoadReportAt: lrsclientinternal.TimeNow(),
+	}
+	c[serviceName] = p
+	return p
+}
+
+// stats returns the load data for the given cluster names. Data is returned in
+// a slice with no specific order.
+//
+// If no clusterName is given (an empty slice), all data for all known clusters
+// is returned.
+//
+// If a cluster's loadData is empty (no load to report), it's not appended to
+// the returned slice.
+func (ls *LoadStore) stats(clusterNames []string) []*loadData {
+	ls.mu.Lock()
+	defer ls.mu.Unlock()
+
+	var ret []*loadData
+	if len(clusterNames) == 0 {
+		for _, c := range ls.clusters {
+			ret = appendClusterStats(ret, c)
+		}
+		return ret
+	}
+	for _, n := range clusterNames {
+		if c, ok := ls.clusters[n]; ok {
+			ret = appendClusterStats(ret, c)
+		}
+	}
+
+	return ret
+}
+
+// PerClusterReporter records load data pertaining to a single cluster. It
+// provides methods to record call starts, finishes, server-reported loads,
+// and dropped calls.
+//
+// It is safe for concurrent use.
+//
+// TODO(purnesh42h): Use regular maps with mutexes instead of sync.Map here.
+// The latter is optimized for two common use cases: (1) when the entry for a
+// given key is only ever written once but read many times, as in caches that
+// only grow, or (2) when multiple goroutines read, write, and overwrite
+// entries for disjoint sets of keys. In these two cases, use of a Map may
+// significantly reduce lock contention compared to a Go map paired with a
+// separate Mutex or RWMutex.
+// Neither of these conditions are met here, and we should transition to a
+// regular map with a mutex for better type safety.
+type PerClusterReporter struct {
+	cluster, service string
+	drops            sync.Map // map[string]*uint64
+	localityRPCCount sync.Map // map[clients.Locality]*rpcCountData
+
+	mu               sync.Mutex
+	lastLoadReportAt time.Time
+}
+
+// CallStarted records a call started in the LoadStore.
+func (p *PerClusterReporter) CallStarted(locality clients.Locality) {
+	s, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		tp := newRPCCountData()
+		s, _ = p.localityRPCCount.LoadOrStore(locality, tp)
+	}
+	s.(*rpcCountData).incrInProgress()
+	s.(*rpcCountData).incrIssued()
+}
+
+// CallFinished records a call finished in the LoadStore.
+func (p *PerClusterReporter) CallFinished(locality clients.Locality, err error) {
+	f, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for call-finish is not found should never happen.
+		return
+	}
+	f.(*rpcCountData).decrInProgress()
+	if err == nil {
+		f.(*rpcCountData).incrSucceeded()
+	} else {
+		f.(*rpcCountData).incrErrored()
+	}
+}
+
+// CallServerLoad records the server load in the LoadStore.
+func (p *PerClusterReporter) CallServerLoad(locality clients.Locality, name string, val float64) {
+	s, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for callServerLoad is not found should never happen.
+		return
+	}
+	s.(*rpcCountData).addServerLoad(name, val)
+}
+
+// CallDropped records a call dropped in the LoadStore.
+func (p *PerClusterReporter) CallDropped(category string) {
+	d, ok := p.drops.Load(category)
+	if !ok {
+		tp := new(uint64)
+		d, _ = p.drops.LoadOrStore(category, tp)
+	}
+	atomic.AddUint64(d.(*uint64), 1)
+}
+
+// stats returns and resets all loads reported to the store, except inProgress
+// rpc counts.
+//
+// It returns nil if the store doesn't contain any (new) data.
+func (p *PerClusterReporter) stats() *loadData {
+	sd := newLoadData(p.cluster, p.service)
+	p.drops.Range(func(key, val any) bool {
+		d := atomic.SwapUint64(val.(*uint64), 0)
+		if d == 0 {
+			return true
+		}
+		sd.totalDrops += d
+		keyStr := key.(string)
+		if keyStr != "" {
+			// Skip drops without category. They are counted in total_drops, but
+			// not in per category. One example is drops by circuit breaking.
+			sd.drops[keyStr] = d
+		}
+		return true
+	})
+	p.localityRPCCount.Range(func(key, val any) bool {
+		countData := val.(*rpcCountData)
+		succeeded := countData.loadAndClearSucceeded()
+		inProgress := countData.loadInProgress()
+		errored := countData.loadAndClearErrored()
+		issued := countData.loadAndClearIssued()
+		if succeeded == 0 && inProgress == 0 && errored == 0 && issued == 0 {
+			return true
+		}
+
+		ld := localityData{
+			requestStats: requestData{
+				succeeded:  succeeded,
+				errored:    errored,
+				inProgress: inProgress,
+				issued:     issued,
+			},
+			loadStats: make(map[string]serverLoadData),
+		}
+		countData.serverLoads.Range(func(key, val any) bool {
+			sum, count := val.(*rpcLoadData).loadAndClear()
+			if count == 0 {
+				return true
+			}
+			ld.loadStats[key.(string)] = serverLoadData{
+				count: count,
+				sum:   sum,
+			}
+			return true
+		})
+		sd.localityStats[key.(clients.Locality)] = ld
+		return true
+	})
+
+	p.mu.Lock()
+	sd.reportInterval = lrsclientinternal.TimeNow().Sub(p.lastLoadReportAt)
+	p.lastLoadReportAt = lrsclientinternal.TimeNow()
+	p.mu.Unlock()
+
+	if sd.totalDrops == 0 && len(sd.drops) == 0 && len(sd.localityStats) == 0 {
+		return nil
+	}
+	return sd
+}
+
+// loadData contains all load data reported to the LoadStore since the most recent
+// call to stats().
+type loadData struct {
+	// cluster is the name of the cluster this data is for.
+	cluster string
+	// service is the name of the EDS service this data is for.
+	service string
+	// totalDrops is the total number of dropped requests.
+	totalDrops uint64
+	// drops is the number of dropped requests per category.
+	drops map[string]uint64
+	// localityStats contains load reports per locality.
+	localityStats map[clients.Locality]localityData
+	// reportInternal is the duration since last time load was reported (stats()
+	// was called).
+	reportInterval time.Duration
+}
+
+// localityData contains load data for a single locality.
+type localityData struct {
+	// requestStats contains counts of requests made to the locality.
+	requestStats requestData
+	// loadStats contains server load data for requests made to the locality,
+	// indexed by the load type.
+	loadStats map[string]serverLoadData
+}
+
+// requestData contains request counts.
+type requestData struct {
+	// succeeded is the number of succeeded requests.
+	succeeded uint64
+	// errored is the number of requests which ran into errors.
+	errored uint64
+	// inProgress is the number of requests in flight.
+	inProgress uint64
+	// issued is the total number requests that were sent.
+	issued uint64
+}
+
+// serverLoadData contains server load data.
+type serverLoadData struct {
+	// count is the number of load reports.
+	count uint64
+	// sum is the total value of all load reports.
+	sum float64
+}
+
+// appendClusterStats gets the Data for all the given clusters, append to ret,
+// and return the new slice.
+//
+// Data is only appended to ret if it's not empty.
+func appendClusterStats(ret []*loadData, clusters map[string]*PerClusterReporter) []*loadData {
+	for _, d := range clusters {
+		data := d.stats()
+		if data == nil {
+			// Skip this data if it doesn't contain any information.
+			continue
+		}
+		ret = append(ret, data)
+	}
+	return ret
+}
+
+func newLoadData(cluster, service string) *loadData {
+	return &loadData{
+		cluster:       cluster,
+		service:       service,
+		drops:         make(map[string]uint64),
+		localityStats: make(map[clients.Locality]localityData),
+	}
+}
+
+type rpcCountData struct {
+	// Only atomic accesses are allowed for the fields.
+	succeeded  *uint64
+	errored    *uint64
+	inProgress *uint64
+	issued     *uint64
+
+	// Map from load desc to load data (sum+count). Loading data from map is
+	// atomic, but updating data takes a lock, which could cause contention when
+	// multiple RPCs try to report loads for the same desc.
+	//
+	// To fix the contention, shard this map.
+	serverLoads sync.Map // map[string]*rpcLoadData
+}
+
+func newRPCCountData() *rpcCountData {
+	return &rpcCountData{
+		succeeded:  new(uint64),
+		errored:    new(uint64),
+		inProgress: new(uint64),
+		issued:     new(uint64),
+	}
+}
+
+func (rcd *rpcCountData) incrSucceeded() {
+	atomic.AddUint64(rcd.succeeded, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearSucceeded() uint64 {
+	return atomic.SwapUint64(rcd.succeeded, 0)
+}
+
+func (rcd *rpcCountData) incrErrored() {
+	atomic.AddUint64(rcd.errored, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearErrored() uint64 {
+	return atomic.SwapUint64(rcd.errored, 0)
+}
+
+func (rcd *rpcCountData) incrInProgress() {
+	atomic.AddUint64(rcd.inProgress, 1)
+}
+
+func (rcd *rpcCountData) decrInProgress() {
+	atomic.AddUint64(rcd.inProgress, ^uint64(0)) // atomic.Add(x, -1)
+}
+
+func (rcd *rpcCountData) loadInProgress() uint64 {
+	return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading.
+}
+
+func (rcd *rpcCountData) incrIssued() {
+	atomic.AddUint64(rcd.issued, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearIssued() uint64 {
+	return atomic.SwapUint64(rcd.issued, 0)
+}
+
+func (rcd *rpcCountData) addServerLoad(name string, d float64) {
+	loads, ok := rcd.serverLoads.Load(name)
+	if !ok {
+		tl := newRPCLoadData()
+		loads, _ = rcd.serverLoads.LoadOrStore(name, tl)
+	}
+	loads.(*rpcLoadData).add(d)
+}
+
+// rpcLoadData is data for server loads (from trailers or oob). Fields in this
+// struct must be updated consistently.
+//
+// The current solution is to hold a lock, which could cause contention. To fix,
+// shard serverLoads map in rpcCountData.
+type rpcLoadData struct {
+	mu    sync.Mutex
+	sum   float64
+	count uint64
+}
+
+func newRPCLoadData() *rpcLoadData {
+	return &rpcLoadData{}
+}
+
+func (rld *rpcLoadData) add(v float64) {
+	rld.mu.Lock()
+	rld.sum += v
+	rld.count++
+	rld.mu.Unlock()
+}
+
+func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) {
+	rld.mu.Lock()
+	s, rld.sum = rld.sum, 0
+	c, rld.count = rld.count, 0
+	rld.mu.Unlock()
+	return s, c
+}

internal/xds/clients/lrsclient/load_store_test.go

@@ -0,0 +1,525 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lrsclient
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/xds/clients"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+)
+
+var (
+	dropCategories = []string{"drop_for_real", "drop_for_fun"}
+	localities     = []clients.Locality{{Region: "locality-A"}, {Region: "locality-B"}}
+	errTest        = fmt.Errorf("test error")
+)
+
+// rpcData wraps the rpc counts and load data to be pushed to the store.
+type rpcData struct {
+	start, success, failure int
+	serverData              map[string]float64 // Will be reported with successful RPCs.
+}
+
+func verifyLoadStoreData(wantStoreData, gotStoreData []*loadData) error {
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmp.AllowUnexported(loadData{}, localityData{}, requestData{}, serverLoadData{}), cmpopts.IgnoreFields(loadData{}, "reportInterval"), sortDataSlice); diff != "" {
+		return fmt.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+	return nil
+}
+
+// TestDrops spawns a bunch of goroutines which report drop data. After the
+// goroutines have exited, the test dumps the stats from the Store and makes
+// sure they are as expected.
+func TestDrops(t *testing.T) {
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+			"":                10,
+		}
+		wantStoreData = &loadData{
+			totalDrops: 80,
+			drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+		}
+	)
+
+	ls := PerClusterReporter{}
+	var wg sync.WaitGroup
+	for category, count := range drops {
+		for i := 0; i < count; i++ {
+			wg.Add(1)
+			go func(c string) {
+				ls.CallDropped(c)
+				wg.Done()
+			}(category)
+		}
+	}
+	wg.Wait()
+
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+// TestLocalityStats spawns a bunch of goroutines which report rpc and load
+// data. After the goroutines have exited, the test dumps the stats from the
+// Store and makes sure they are as expected.
+func TestLocalityStats(t *testing.T) {
+	var (
+		ld = map[clients.Locality]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &loadData{
+			localityStats: map[clients.Locality]localityData{
+				localities[0]: {
+					requestStats: requestData{
+						succeeded:  20,
+						errored:    10,
+						inProgress: 10,
+						issued:     40,
+					},
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 20, sum: 20},
+						"disk": {count: 20, sum: 40},
+						"cpu":  {count: 20, sum: 60},
+						"mem":  {count: 20, sum: 80},
+					},
+				},
+				localities[1]: {
+					requestStats: requestData{
+						succeeded:  40,
+						errored:    20,
+						inProgress: 20,
+						issued:     80,
+					},
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 40, sum: 40},
+						"disk": {count: 40, sum: 80},
+						"cpu":  {count: 40, sum: 120},
+						"mem":  {count: 40, sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	ls := PerClusterReporter{}
+	var wg sync.WaitGroup
+	for locality, data := range ld {
+		wg.Add(data.start)
+		for i := 0; i < data.start; i++ {
+			go func(l clients.Locality) {
+				ls.CallStarted(l)
+				wg.Done()
+			}(locality)
+		}
+		// The calls to callStarted() need to happen before the other calls are
+		// made. Hence the wait here.
+		wg.Wait()
+
+		wg.Add(data.success)
+		for i := 0; i < data.success; i++ {
+			go func(l clients.Locality, serverData map[string]float64) {
+				ls.CallFinished(l, nil)
+				for n, d := range serverData {
+					ls.CallServerLoad(l, n, d)
+				}
+				wg.Done()
+			}(locality, data.serverData)
+		}
+		wg.Add(data.failure)
+		for i := 0; i < data.failure; i++ {
+			go func(l clients.Locality) {
+				ls.CallFinished(l, errTest)
+				wg.Done()
+			}(locality)
+		}
+		wg.Wait()
+	}
+
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestResetAfterStats(t *testing.T) {
+	// Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero.
+	// Dump the stats. Verify expected
+	// Push the same set of loads as before
+	// Now dump and verify the newly expected ones.
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+		}
+		ld = map[clients.Locality]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &loadData{
+			totalDrops: 70,
+			drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+			localityStats: map[clients.Locality]localityData{
+				localities[0]: {
+					requestStats: requestData{
+						succeeded:  20,
+						errored:    10,
+						inProgress: 10,
+						issued:     40,
+					},
+
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 20, sum: 20},
+						"disk": {count: 20, sum: 40},
+						"cpu":  {count: 20, sum: 60},
+						"mem":  {count: 20, sum: 80},
+					},
+				},
+				localities[1]: {
+					requestStats: requestData{
+						succeeded:  40,
+						errored:    20,
+						inProgress: 20,
+						issued:     80,
+					},
+
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 40, sum: 40},
+						"disk": {count: 40, sum: 80},
+						"cpu":  {count: 40, sum: 120},
+						"mem":  {count: 40, sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	reportLoad := func(ls *PerClusterReporter) {
+		for category, count := range drops {
+			for i := 0; i < count; i++ {
+				ls.CallDropped(category)
+			}
+		}
+		for locality, data := range ld {
+			for i := 0; i < data.start; i++ {
+				ls.CallStarted(locality)
+			}
+			for i := 0; i < data.success; i++ {
+				ls.CallFinished(locality, nil)
+				for n, d := range data.serverData {
+					ls.CallServerLoad(locality, n, d)
+				}
+			}
+			for i := 0; i < data.failure; i++ {
+				ls.CallFinished(locality, errTest)
+			}
+		}
+	}
+
+	ls := PerClusterReporter{}
+	reportLoad(&ls)
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+
+	// The above call to stats() should have reset all load reports except the
+	// inProgress rpc count. We are now going to push the same load data into
+	// the store. So, we should expect to see twice the count for inProgress.
+	for _, l := range localities {
+		ls := wantStoreData.localityStats[l]
+		ls.requestStats.inProgress *= 2
+		wantStoreData.localityStats[l] = ls
+	}
+	reportLoad(&ls)
+	gotStoreData = ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*loadData) []*loadData {
+	out := append([]*loadData(nil), in...) // Copy input to avoid mutating it
+	sort.Slice(out,
+		func(i, j int) bool {
+			if out[i].cluster < out[j].cluster {
+				return true
+			}
+			if out[i].cluster == out[j].cluster {
+				return out[i].service < out[j].service
+			}
+			return false
+		},
+	)
+	return out
+})
+
+// Test all load are returned for the given clusters, and all clusters are
+// reported if no cluster is specified.
+func TestStoreStats(t *testing.T) {
+	var (
+		testClusters = []string{"c0", "c1", "c2"}
+		testServices = []string{"s0", "s1"}
+		testLocality = clients.Locality{Region: "test-locality"}
+	)
+
+	store := newLoadStore()
+	for _, c := range testClusters {
+		for _, s := range testServices {
+			store.ReporterForCluster(c, s).CallStarted(testLocality)
+			store.ReporterForCluster(c, s).CallServerLoad(testLocality, "abc", 123)
+			store.ReporterForCluster(c, s).CallDropped("dropped")
+			store.ReporterForCluster(c, s).CallFinished(testLocality, nil)
+		}
+	}
+
+	wantC0 := []*loadData{
+		{
+			cluster: "c0", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c0", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with just "c0", this should return data for "c0", and not
+	// touch data for other clusters.
+	gotC0 := store.stats([]string{"c0"})
+	verifyLoadStoreData(wantC0, gotC0)
+
+	wantOther := []*loadData{
+		{
+			cluster: "c1", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c2", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c2", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// remaining clusters, and not include c0 (because c0 data was cleared).
+	gotOther := store.stats(nil)
+	if err := verifyLoadStoreData(wantOther, gotOther); err != nil {
+		t.Error(err)
+	}
+}
+
+// Test the cases that if a cluster doesn't have load to report, its data is not
+// appended to the slice returned by Stats().
+func TestStoreStatsEmptyDataNotReported(t *testing.T) {
+	var (
+		testServices = []string{"s0", "s1"}
+		testLocality = clients.Locality{Region: "test-locality"}
+	)
+
+	store := newLoadStore()
+	// "c0"'s RPCs all finish with success.
+	for _, s := range testServices {
+		store.ReporterForCluster("c0", s).CallStarted(testLocality)
+		store.ReporterForCluster("c0", s).CallFinished(testLocality, nil)
+	}
+	// "c1"'s RPCs never finish (always inprocess).
+	for _, s := range testServices {
+		store.ReporterForCluster("c1", s).CallStarted(testLocality)
+	}
+
+	want0 := []*loadData{
+		{
+			cluster: "c0", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{succeeded: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c0", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{succeeded: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1, issued: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// clusters.
+	got0 := store.stats(nil)
+	if err := verifyLoadStoreData(want0, got0); err != nil {
+		t.Error(err)
+	}
+
+	want1 := []*loadData{
+		{
+			cluster: "c1", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice again, this should return data only for "c1",
+	// because "c0" data was cleared, but "c1" has in-progress RPCs.
+	got1 := store.stats(nil)
+	if err := verifyLoadStoreData(want1, got1); err != nil {
+		t.Error(err)
+	}
+}
+
+// TestStoreReportInterval verify that the load report interval gets
+// calculated at every stats() call and is the duration between start of last
+// load reporting to next stats() call.
+func TestStoreReportInterval(t *testing.T) {
+	originalTimeNow := lrsclientinternal.TimeNow
+	t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow })
+
+	// Initial time for reporter creation
+	currentTime := time.Now()
+	lrsclientinternal.TimeNow = func() time.Time {
+		return currentTime
+	}
+
+	store := newLoadStore()
+	reporter := store.ReporterForCluster("test-cluster", "test-service")
+	// Report dummy drop to ensure stats1 is not nil.
+	reporter.CallDropped("dummy-category")
+
+	// Update currentTime to simulate the passage of time between the reporter
+	// creation and first stats() call.
+	currentTime = currentTime.Add(5 * time.Second)
+	stats1 := reporter.stats()
+
+	if stats1 == nil {
+		t.Fatalf("stats1 is nil after reporting a drop, want non-nil")
+	}
+	// Verify stats() call calculate the report interval from the time of
+	// reporter creation.
+	if got, want := stats1.reportInterval, 5*time.Second; got != want {
+		t.Errorf("stats1.reportInterval = %v, want %v", stats1.reportInterval, want)
+	}
+
+	// Update currentTime to simulate the passage of time between the first
+	// and second stats() call.
+	currentTime = currentTime.Add(10 * time.Second)
+	// Report another dummy drop to ensure stats2 is not nil.
+	reporter.CallDropped("dummy-category-2")
+	stats2 := reporter.stats()
+
+	if stats2 == nil {
+		t.Fatalf("stats2 is nil after reporting a drop, want non-nil")
+	}
+	// Verify stats() call calculate the report interval from the time of first
+	// stats() call.
+	if got, want := stats2.reportInterval, 10*time.Second; got != want {
+		t.Errorf("stats2.reportInterval = %v, want %v", stats2.reportInterval, want)
+	}
+}

internal/xds/clients/lrsclient/loadreport_test.go

@@ -0,0 +1,730 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient_test
+
+import (
+	"context"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpctest"
+	grpctestutils "google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	testKey1                      = "test-key1"
+	testKey2                      = "test-key2"
+	defaultTestWatchExpiryTimeout = 100 * time.Millisecond
+	defaultTestTimeout            = 5 * time.Second
+	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+var (
+	testLocality1     = clients.Locality{Region: "test-region1"}
+	testLocality2     = clients.Locality{Region: "test-region2"}
+	toleranceCmpOpt   = cmpopts.EquateApprox(0, 1e-5)
+	ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats",
+		cmpopts.SortSlices(func(a, b protocmp.Message) bool {
+			return a.String() < b.String()
+		}),
+	)
+)
+
+type wrappedListener struct {
+	net.Listener
+	newConnChan *testutils.Channel // Connection attempts are pushed here.
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	c, err := wl.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	wl.newConnChan.Send(struct{}{})
+	return c, err
+}
+
+// Tests a load reporting scenario where the LRS client is reporting loads to
+// multiple servers. Verifies the following:
+//   - calling the load reporting API with different server configuration
+//     results in connections being created to those corresponding servers
+//   - the same load.Store is not returned when the load reporting API called
+//     with different server configurations
+//   - canceling the load reporting from the client results in the LRS stream
+//     being canceled on the server
+func (s) TestReportLoad_ConnectionCreation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create two management servers that also serve LRS.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	newConnChan1 := testutils.NewChannelWithSize(1)
+	lis1 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan1,
+	}
+	mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis1,
+		SupportLoadReportingService: true,
+	})
+	l, err = net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	newConnChan2 := testutils.NewChannelWithSize(1)
+	lis2 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan2,
+	}
+	mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis2,
+		SupportLoadReportingService: true,
+	})
+
+	// Create an LRS client with a configuration that contains both of
+	// the above two servers. The authority name is immaterial here since load
+	// reporting is per-server and not per-authority.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	serverIdentifier1 := clients.ServerIdentifier{ServerURI: mgmtServer1.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier1)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	defer loadStore1.Stop(ssCtx)
+
+	// Call the load reporting API to report load to the first management
+	// server, and ensure that a connection to the server is created.
+	if _, err := newConnChan1.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting")
+	}
+	if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	// Call the load reporting API to report load to the first management
+	// server, and ensure that a connection to the server is created.
+	serverIdentifier2 := clients.ServerIdentifier{ServerURI: mgmtServer2.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore2, err := client.ReportLoad(serverIdentifier2)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	if _, err := newConnChan2.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting")
+	}
+	if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	if loadStore1 == loadStore2 {
+		t.Fatalf("Got same store for different servers, want different")
+	}
+
+	// Push some loads on the received store.
+	loadStore2.ReporterForCluster("cluster", "eds").CallDropped("test")
+
+	// Ensure the initial load reporting request is received at the server.
+	lrsServer := mgmtServer2.LRSServer
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for LRS request with loads: %v", err)
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster",
+		ClusterServiceName:   "eds",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Stop this load reporting stream, server should see error canceled.
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore2.Stop(ssCtx)
+
+	// Server should receive a stream canceled error. There may be additional
+	// load reports from the client in the channel.
+	for {
+		if ctx.Err() != nil {
+			t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server")
+		}
+		u, err := lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		// Ignore load reports sent before the stream was cancelled.
+		if u.(*fakeserver.Request).Err == nil {
+			continue
+		}
+		if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled {
+			t.Fatalf("Unexpected LRS request: %v, want error canceled", u)
+		}
+		break
+	}
+}
+
+// Tests a load reporting scenario where the load reporting API is called
+// multiple times for the same server. The test verifies the following:
+//   - calling the load reporting API the second time for the same server
+//     configuration does not create a new LRS stream
+//   - the LRS stream is closed *only* after all the API calls invoke their
+//     cancel functions
+//   - creating new streams after the previous one was closed works
+func (s) TestReportLoad_StreamCreation(t *testing.T) {
+	grpctestutils.SetEnvConfig(t, &envconfig.XDSORCAToLRSPropEnabled, true)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	// Push some loads on the received store.
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "cpu_utilization", 1.5)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "mem_utilization", 2.0)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "application_utilization", 3.0)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil)
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster1",
+		ClusterServiceName:   "eds1",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{
+			{
+				Locality: &v3corepb.Locality{Region: "test-region1"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					// TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858
+					{MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}},
+				CpuUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.5,
+				},
+				MemUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 2.0,
+				},
+				ApplicationUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 3.0,
+				},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+			{
+				Locality: &v3corepb.Locality{Region: "test-region2"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					{MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+		},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Make another call to the load reporting API, and ensure that a new LRS
+	// stream is not created.
+	loadStore2, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("New LRS stream created when expected to use an existing one")
+	}
+
+	// Push more loads.
+	loadStore2.ReporterForCluster("cluster2", "eds2").CallDropped("test")
+
+	// Ensure that loads are seen on the server. We need a loop here because
+	// there could have been some requests from the client in the time between
+	// us reading the first request and now. Those would have been queued in the
+	// request channel that we read out of.
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for new loads to be seen on the server")
+		}
+
+		req, err = lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+		if l := len(gotLoad); l != 1 {
+			continue
+		}
+		gotLoad[0].LoadReportInterval = nil
+		wantLoad := &v3endpointpb.ClusterStats{
+			ClusterName:          "cluster2",
+			ClusterServiceName:   "eds2",
+			TotalDroppedRequests: 1,
+			DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		}
+		if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
+			t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+			continue
+		}
+		break
+	}
+
+	// Cancel the first load reporting call, and ensure that the stream does not
+	// close (because we have another call open).
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore1.Stop(ssCtx)
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("LRS stream closed when expected to stay open")
+	}
+
+	// Stop the second load reporting call, and ensure the stream is closed.
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore2.Stop(ssCtx)
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for LRS stream to close")
+	}
+
+	// Calling the load reporting API again should result in the creation of a
+	// new LRS stream. This ensures that creating and closing multiple streams
+	// works smoothly.
+	loadStore3, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore3.Stop(ssCtx)
+}
+
+// TestReportLoad_StopWithContext tests the behavior of LoadStore.Stop() when
+// called with a context. It verifies that:
+//   - Stop() blocks until the context expires or final load send attempt is
+//     made.
+//   - Final load report is seen on the server after stop is called.
+//   - The stream is closed after Stop() returns.
+func (s) TestReportLoad_StopWithContext(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	// Push some loads on the received store.
+	loadStore.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster1",
+		ClusterServiceName:   "eds1",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Create a context for Stop() that remains until the end of test to ensure
+	// that only possibility of Stop()s to finish is if final load send attempt
+	// is made. If final load attempt is not made, test will timeout.
+	stopCtx, stopCancel := context.WithCancel(ctx)
+	defer stopCancel()
+
+	// Push more loads.
+	loadStore.ReporterForCluster("cluster2", "eds2").CallDropped("test")
+
+	stopCalled := make(chan struct{})
+	// Call Stop in a separate goroutine. It will block until
+	// final load send attempt is made.
+	go func() {
+		loadStore.Stop(stopCtx)
+		close(stopCalled)
+	}()
+	<-stopCalled
+
+	// Ensure that loads are seen on the server. We need a loop here because
+	// there could have been some requests from the client in the time between
+	// us reading the first request and now. Those would have been queued in the
+	// request channel that we read out of.
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for new loads to be seen on the server")
+		}
+
+		req, err = lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil || req.(*fakeserver.Request).Err != nil {
+			continue
+		}
+		if req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) == nil {
+			// This can happen due to a race:
+			// 1. Load for "cluster2" is reported just before Stop().
+			// 2. The periodic ticker might send this load before Stop()'s
+			//    final send mechanism processes it, clearing the data.
+			// 3. Stop()'s final send might then send an empty report.
+			//    This is acceptable for this test because we only need to verify
+			//    if the final load report send attempt was made.
+			t.Logf("Empty final load report sent on server")
+			break
+		}
+		gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+		if l := len(gotLoad); l != 1 {
+			continue
+		}
+		gotLoad[0].LoadReportInterval = nil
+		wantLoad := &v3endpointpb.ClusterStats{
+			ClusterName:          "cluster2",
+			ClusterServiceName:   "eds2",
+			TotalDroppedRequests: 1,
+			DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		}
+		if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
+			t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+			continue
+		}
+		break
+	}
+
+	// Verify the stream is eventually closed on the server side.
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for LRS stream to close")
+	}
+}
+
+// TestReportLoad_LoadReportInterval tests verify that the load report interval
+// received by the LRS server is the duration between start of last load
+// reporting by the client and the time when the load is reported to the LRS
+// server.
+func (s) TestReportLoad_LoadReportInterval(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	originalTimeNow := lrsclientinternal.TimeNow
+	t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow })
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	stateMu := sync.Mutex{}
+	// Initial time for reporter creation
+	currentTime := time.Now()
+	lrsclientinternal.TimeNow = func() time.Time {
+		stateMu.Lock()
+		defer stateMu.Unlock()
+		return currentTime
+	}
+
+	// Report dummy drop to ensure stats is not nil.
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+
+	// Update currentTime to simulate the passage of time between the reporter
+	// creation and first stats() call.
+	stateMu.Lock()
+	currentTime = currentTime.Add(5 * time.Second)
+	stateMu.Unlock()
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+	// Verify load received at LRS server has load report interval calculated
+	// from the time of reporter creation.
+	if got, want := gotLoad[0].GetLoadReportInterval().AsDuration(), 5*time.Second; got != want {
+		t.Errorf("Got load report interval %v, want %v", got, want)
+	}
+
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), time.Millisecond)
+	defer ssCancel()
+	loadStore1.Stop(ssCtx)
+}

internal/xds/clients/lrsclient/logging.go

@@ -0,0 +1,36 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(c *LRSClient) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, clientPrefix(c))
+}
+
+func clientPrefix(c *LRSClient) string {
+	return fmt.Sprintf("[lrs-client %p] ", c)
+}

internal/xds/clients/lrsclient/lrs_stream.go

@@ -0,0 +1,334 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lrsclient
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/internal/envconfig"
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+)
+
+// Any per-RPC level logs which print complete request or response messages
+// should be gated at this verbosity level. Other per-RPC level logs which print
+// terse output should be at `INFO` and verbosity 2.
+const perRPCVerbosityLevel = 9
+
+// streamImpl provides all the functionality associated with an LRS (Load
+// Reporting Service) stream on the client-side. It manages the lifecycle of
+// the LRS stream, including starting, stopping, and retrying the stream. It
+// also provides a LoadStore that can be used to report load, with a Stop
+// function that should be called when the load reporting is no longer
+// needed.
+type streamImpl struct {
+	// The following fields are initialized when a stream instance is created
+	// and are read-only afterwards, and hence can be accessed without a mutex.
+	transport clients.Transport       // Transport to use for LRS stream.
+	backoff   func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto *v3corepb.Node          // Identifies the gRPC application.
+	doneCh    chan struct{}           // To notify exit of LRS goroutine.
+	logger    *igrpclog.PrefixLogger
+
+	cancelStream context.CancelFunc // Cancel the stream. If nil, the stream is not active.
+	loadStore    *LoadStore         // LoadStore returned to user for pushing loads.
+
+	finalSendRequest chan struct{} // To request for the final attempt to send loads.
+	finalSendDone    chan error    // To signal completion of the final attempt of sending loads.
+}
+
+// streamOpts holds the options for creating an lrsStream.
+type streamOpts struct {
+	transport clients.Transport       // xDS transport to create the stream on.
+	backoff   func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto *v3corepb.Node          // Node proto to identify the gRPC application.
+	logPrefix string                  // Prefix to be used for log messages.
+}
+
+// newStreamImpl creates a new StreamImpl with the provided options.
+//
+// The actual streaming RPC call is initiated when the first call to ReportLoad
+// is made, and is terminated when the last call to ReportLoad is canceled.
+func newStreamImpl(opts streamOpts) *streamImpl {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	lrs := &streamImpl{
+		transport:        opts.transport,
+		backoff:          opts.backoff,
+		nodeProto:        opts.nodeProto,
+		cancelStream:     cancel,
+		doneCh:           make(chan struct{}),
+		finalSendRequest: make(chan struct{}, 1),
+		finalSendDone:    make(chan error, 1),
+	}
+
+	l := grpclog.Component("xds")
+	lrs.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[lrs-stream %p] ", lrs))
+	lrs.loadStore = newLoadStore()
+	go lrs.runner(ctx)
+	return lrs
+}
+
+// runner is responsible for managing the lifetime of an LRS streaming call. It
+// creates the stream, sends the initial LoadStatsRequest, receives the first
+// LoadStatsResponse, and then starts a goroutine to periodically send
+// LoadStatsRequests. The runner will restart the stream if it encounters any
+// errors.
+func (lrs *streamImpl) runner(ctx context.Context) {
+	defer close(lrs.doneCh)
+
+	// This feature indicates that the client supports the
+	// LoadStatsResponse.send_all_clusters field in the LRS response.
+	node := proto.Clone(lrs.nodeProto).(*v3corepb.Node)
+	node.ClientFeatures = append(node.ClientFeatures, "envoy.lrs.supports_send_all_clusters")
+
+	runLoadReportStream := func() error {
+		// streamCtx is created and canceled in case we terminate the stream
+		// early for any reason, to avoid gRPC-Go leaking the RPC's monitoring
+		// goroutine.
+		streamCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		stream, err := lrs.transport.NewStream(streamCtx, "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats")
+		if err != nil {
+			lrs.logger.Warningf("Failed to create new LRS streaming RPC: %v", err)
+			return nil
+		}
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("LRS stream created")
+		}
+
+		if err := lrs.sendFirstLoadStatsRequest(stream, node); err != nil {
+			lrs.logger.Warningf("Sending first LRS request failed: %v", err)
+			return nil
+		}
+
+		clusters, interval, err := lrs.recvFirstLoadStatsResponse(stream)
+		if err != nil {
+			lrs.logger.Warningf("Reading from LRS streaming RPC failed: %v", err)
+			return nil
+		}
+
+		// We reset backoff state when we successfully receive at least one
+		// message from the server.
+		lrs.sendLoads(streamCtx, stream, clusters, interval)
+		return backoff.ErrResetBackoff
+	}
+	backoff.RunF(ctx, runLoadReportStream, lrs.backoff)
+}
+
+// sendLoads is responsible for periodically sending load reports to the LRS
+// server at the specified interval for the specified clusters, until the passed
+// in context is canceled.
+func (lrs *streamImpl) sendLoads(ctx context.Context, stream clients.Stream, clusterNames []string, interval time.Duration) {
+	tick := time.NewTicker(interval)
+	defer tick.Stop()
+	for {
+		select {
+		case <-tick.C:
+		case <-ctx.Done():
+			return
+		case <-lrs.finalSendRequest:
+			var finalSendErr error
+			if lrs.logger.V(2) {
+				lrs.logger.Infof("Final send request received. Attempting final LRS report.")
+			}
+			if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil {
+				lrs.logger.Warningf("Failed to send final load report. Writing to LRS stream failed: %v", err)
+				finalSendErr = err
+			}
+			if lrs.logger.V(2) {
+				lrs.logger.Infof("Successfully sent final load report.")
+			}
+			lrs.finalSendDone <- finalSendErr
+			return
+		}
+
+		if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil {
+			lrs.logger.Warningf("Failed to send periodic load report. Writing to LRS stream failed: %v", err)
+			return
+		}
+	}
+}
+
+func (lrs *streamImpl) sendFirstLoadStatsRequest(stream clients.Stream, node *v3corepb.Node) error {
+	req := &v3lrspb.LoadStatsRequest{Node: node}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Sending initial LoadStatsRequest: %s", pretty.ToJSON(req))
+	}
+	msg, err := proto.Marshal(req)
+	if err != nil {
+		lrs.logger.Warningf("Failed to marshal LoadStatsRequest: %v", err)
+		return err
+	}
+	err = stream.Send(msg)
+	if err == io.EOF {
+		return getStreamError(stream)
+	}
+	return err
+}
+
+// recvFirstLoadStatsResponse receives the first LoadStatsResponse from the LRS
+// server.  Returns the following:
+//   - a list of cluster names requested by the server or an empty slice if the
+//     server requested for load from all clusters
+//   - the load reporting interval, and
+//   - any error encountered
+func (lrs *streamImpl) recvFirstLoadStatsResponse(stream clients.Stream) ([]string, time.Duration, error) {
+	r, err := stream.Recv()
+	if err != nil {
+		return nil, 0, fmt.Errorf("lrs: failed to receive first LoadStatsResponse: %v", err)
+	}
+	var resp v3lrspb.LoadStatsResponse
+	if err := proto.Unmarshal(r, &resp); err != nil {
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("Failed to unmarshal response to LoadStatsResponse: %v", err)
+		}
+		return nil, time.Duration(0), fmt.Errorf("lrs: unexpected message type %T", r)
+	}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Received first LoadStatsResponse: %s", pretty.ToJSON(&resp))
+	}
+
+	internal := resp.GetLoadReportingInterval()
+	if internal.CheckValid() != nil {
+		return nil, 0, fmt.Errorf("lrs: invalid load_reporting_interval: %v", err)
+	}
+	loadReportingInterval := internal.AsDuration()
+
+	clusters := resp.Clusters
+	if resp.SendAllClusters {
+		// Return an empty slice to send stats for all clusters.
+		clusters = []string{}
+	}
+
+	return clusters, loadReportingInterval, nil
+}
+
+func (lrs *streamImpl) sendLoadStatsRequest(stream clients.Stream, loads []*loadData) error {
+	clusterStats := make([]*v3endpointpb.ClusterStats, 0, len(loads))
+	for _, sd := range loads {
+		droppedReqs := make([]*v3endpointpb.ClusterStats_DroppedRequests, 0, len(sd.drops))
+		for category, count := range sd.drops {
+			droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
+				Category:     category,
+				DroppedCount: count,
+			})
+		}
+		localityStats := make([]*v3endpointpb.UpstreamLocalityStats, 0, len(sd.localityStats))
+		for lid, localityData := range sd.localityStats {
+			loadMetricStats := make([]*v3endpointpb.EndpointLoadMetricStats, 0, len(localityData.loadStats))
+			var cpuUtilization, memUtilization, appUtilization *v3endpointpb.UnnamedEndpointLoadMetricStats
+			for name, loadData := range localityData.loadStats {
+				if envconfig.XDSORCAToLRSPropEnabled {
+					switch name {
+					case "cpu_utilization":
+						cpuUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					case "mem_utilization":
+						memUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					case "application_utilization":
+						appUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					default:
+						loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+							MetricName:                    name,
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						})
+					}
+				} else {
+					loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+						MetricName:                    name,
+						NumRequestsFinishedWithMetric: loadData.count,
+						TotalMetricValue:              loadData.sum,
+					})
+				}
+			}
+			localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{
+				Locality: &v3corepb.Locality{
+					Region:  lid.Region,
+					Zone:    lid.Zone,
+					SubZone: lid.SubZone,
+				},
+				TotalSuccessfulRequests: localityData.requestStats.succeeded,
+				TotalRequestsInProgress: localityData.requestStats.inProgress,
+				TotalErrorRequests:      localityData.requestStats.errored,
+				TotalIssuedRequests:     localityData.requestStats.issued,
+				CpuUtilization:          cpuUtilization,
+				MemUtilization:          memUtilization,
+				ApplicationUtilization:  appUtilization,
+				LoadMetricStats:         loadMetricStats,
+				UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
+			})
+		}
+
+		clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{
+			ClusterName:           sd.cluster,
+			ClusterServiceName:    sd.service,
+			UpstreamLocalityStats: localityStats,
+			TotalDroppedRequests:  sd.totalDrops,
+			DroppedRequests:       droppedReqs,
+			LoadReportInterval:    durationpb.New(sd.reportInterval),
+		})
+	}
+
+	req := &v3lrspb.LoadStatsRequest{ClusterStats: clusterStats}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Sending LRS loads: %s", pretty.ToJSON(req))
+	}
+	msg, err := proto.Marshal(req)
+	if err != nil {
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("Failed to marshal LoadStatsRequest: %v", err)
+		}
+		return err
+	}
+	err = stream.Send(msg)
+	if err == io.EOF {
+		return getStreamError(stream)
+	}
+	return err
+}
+
+func getStreamError(stream clients.Stream) error {
+	for {
+		if _, err := stream.Recv(); err != nil {
+			return err
+		}
+	}
+}

internal/xds/clients/lrsclient/lrsclient.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package lrsclient provides an LRS (Load Reporting Service) client.
+//
+// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/load_stats/v3/lrs.proto
+package lrsclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	clientsinternal "google.golang.org/grpc/internal/xds/clients/internal"
+	"google.golang.org/grpc/internal/xds/clients/internal/backoff"
+)
+
+const (
+	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
+	clientFeatureResourceWrapper    = "xds.config.resource-in-sotw"
+)
+
+var (
+	defaultExponentialBackoff = backoff.DefaultExponential.Backoff
+)
+
+// LRSClient is an LRS (Load Reporting Service) client.
+type LRSClient struct {
+	transportBuilder clients.TransportBuilder
+	node             clients.Node
+	backoff          func(int) time.Duration // Backoff for LRS stream failures.
+	logger           *igrpclog.PrefixLogger
+
+	// The LRSClient owns a bunch of streams to individual LRS servers.
+	//
+	// Once all references to a stream are dropped, the stream is closed.
+	mu         sync.Mutex
+	lrsStreams map[clients.ServerIdentifier]*streamImpl // Map from server config to in-use streamImpls.
+	lrsRefs    map[clients.ServerIdentifier]int         // Map from server config to number of references.
+}
+
+// New returns a new LRS Client configured with the provided config.
+func New(config Config) (*LRSClient, error) {
+	if config.TransportBuilder == nil {
+		return nil, errors.New("lrsclient: transport builder is nil")
+	}
+
+	c := &LRSClient{
+		transportBuilder: config.TransportBuilder,
+		node:             config.Node,
+		backoff:          defaultExponentialBackoff,
+		lrsStreams:       make(map[clients.ServerIdentifier]*streamImpl),
+		lrsRefs:          make(map[clients.ServerIdentifier]int),
+	}
+	c.logger = prefixLogger(c)
+	return c, nil
+}
+
+// ReportLoad creates and returns a LoadStore for the caller to report loads
+// using a LoadReportingStream.
+//
+// Caller must call Stop on the returned LoadStore when they are done reporting
+// load to this server.
+func (c *LRSClient) ReportLoad(si clients.ServerIdentifier) (*LoadStore, error) {
+	lrs, err := c.getOrCreateLRSStream(si)
+	if err != nil {
+		return nil, err
+	}
+	return lrs.loadStore, nil
+}
+
+// getOrCreateLRSStream returns an lrs stream for the given server identifier.
+//
+// If an active lrs stream exists for the given server identifier, it is
+// returned. Otherwise, a new lrs stream is created and returned.
+func (c *LRSClient) getOrCreateLRSStream(serverIdentifier clients.ServerIdentifier) (*streamImpl, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.logger.V(2) {
+		c.logger.Infof("Received request for a reference to an lrs stream for server identifier %q", serverIdentifier)
+	}
+
+	// Use an existing stream, if one exists for this server identifier.
+	if s, ok := c.lrsStreams[serverIdentifier]; ok {
+		if c.logger.V(2) {
+			c.logger.Infof("Reusing an existing lrs stream for server identifier %q", serverIdentifier)
+		}
+		c.lrsRefs[serverIdentifier]++
+		return s, nil
+	}
+
+	if c.logger.V(2) {
+		c.logger.Infof("Creating a new lrs stream for server identifier %q", serverIdentifier)
+	}
+
+	// Create a new transport and create a new lrs stream, and add it to the
+	// map of lrs streams.
+	tr, err := c.transportBuilder.Build(serverIdentifier)
+	if err != nil {
+		return nil, fmt.Errorf("lrsclient: failed to create transport for server identifier %s: %v", serverIdentifier, err)
+	}
+
+	nodeProto := clientsinternal.NodeProto(c.node)
+	nodeProto.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}
+	lrs := newStreamImpl(streamOpts{
+		transport: tr,
+		backoff:   c.backoff,
+		nodeProto: nodeProto,
+		logPrefix: clientPrefix(c),
+	})
+
+	// Register a stop function that decrements the reference count, stops
+	// the LRS stream when the last reference is removed and closes the
+	// transport and removes the lrs stream and its references from the
+	// respective maps. Before closing the stream, it waits for the provided
+	// context to be done (timeout or cancellation).
+	stop := func(ctx context.Context) {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+
+		if r, ok := c.lrsRefs[serverIdentifier]; !ok || r == 0 {
+			c.logger.Errorf("Attempting to stop already stopped StreamImpl")
+			return
+		}
+		c.lrsRefs[serverIdentifier]--
+		if c.lrsRefs[serverIdentifier] != 0 {
+			return
+		}
+
+		lrs.finalSendRequest <- struct{}{}
+
+		select {
+		case err := <-lrs.finalSendDone:
+			if err != nil {
+				c.logger.Warningf("Final send attempt failed: %v", err)
+			}
+		case <-ctx.Done():
+			c.logger.Warningf("Context canceled before finishing the final send attempt: %v", err)
+		}
+
+		lrs.cancelStream()
+		lrs.cancelStream = nil
+		lrs.logger.Infof("Stopping LRS stream")
+		<-lrs.doneCh
+
+		delete(c.lrsStreams, serverIdentifier)
+		tr.Close()
+	}
+	lrs.loadStore.stop = stop
+
+	c.lrsStreams[serverIdentifier] = lrs
+	c.lrsRefs[serverIdentifier] = 1
+
+	return lrs, nil
+}

internal/xds/clients/lrsclient/lrsconfig.go

@@ -0,0 +1,35 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"google.golang.org/grpc/internal/xds/clients"
+)
+
+// Config is used to configure an LRS client. After one has been passed to the
+// LRS client's New function, no part of it may modified. A Config may be
+// reused; the lrsclient package will also not modify it.
+type Config struct {
+	// Node is the identity of the client application reporting load to the
+	// LRS server.
+	Node clients.Node
+
+	// TransportBuilder is used to connect to the LRS server.
+	TransportBuilder clients.TransportBuilder
+}

internal/xds/clients/transport_builder.go

@@ -0,0 +1,53 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clients
+
+import (
+	"context"
+)
+
+// TransportBuilder provides the functionality to create a communication
+// channel to an xDS or LRS server.
+type TransportBuilder interface {
+	// Build creates a new Transport instance to the server based on the
+	// provided ServerIdentifier.
+	Build(serverIdentifier ServerIdentifier) (Transport, error)
+}
+
+// Transport provides the functionality to communicate with an xDS or LRS
+// server using streaming calls.
+type Transport interface {
+	// NewStream creates a new streaming call to the server for the specific
+	// RPC method name. The returned Stream interface can be used to send and
+	// receive messages on the stream.
+	NewStream(context.Context, string) (Stream, error)
+
+	// Close closes the Transport.
+	Close()
+}
+
+// Stream provides methods to send and receive messages on a stream. Messages
+// are represented as a byte slice.
+type Stream interface {
+	// Send sends the provided message on the stream.
+	Send([]byte) error
+
+	// Recv blocks until the next message is received on the stream.
+	Recv() ([]byte, error)
+}

internal/xds/clients/xdsclient/ads_stream.go

@@ -0,0 +1,733 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/internal/backoff"
+	"google.golang.org/grpc/internal/xds/clients/internal/pretty"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	cpb "google.golang.org/genproto/googleapis/rpc/code"
+	statuspb "google.golang.org/genproto/googleapis/rpc/status"
+)
+
+const (
+	// Any per-RPC level logs which print complete request or response messages
+	// should be gated at this verbosity level. Other per-RPC level logs which print
+	// terse output should be at `INFO` and verbosity 2.
+	perRPCVerbosityLevel = 9
+)
+
+// request represents a queued request message to be sent on the ADS stream. It
+// contains the type of the resource and the list of resource names to be sent.
+type request struct {
+	typ           ResourceType
+	resourceNames []string
+}
+
+// response represents a response received on the ADS stream. It contains the
+// type URL, version, and resources for the response.
+type response struct {
+	typeURL   string
+	version   string
+	resources []*anypb.Any
+}
+
+// dataAndErrTuple is a struct that holds a resource and an error. It is used to
+// return a resource and any associated error from a function.
+type dataAndErrTuple struct {
+	Resource ResourceData
+	Err      error
+}
+
+// adsStreamEventHandler is an interface that defines the callbacks for events that
+// occur on the ADS stream. Methods on this interface may be invoked
+// concurrently and implementations need to handle them in a thread-safe manner.
+type adsStreamEventHandler interface {
+	onStreamError(error)                           // Called when the ADS stream breaks.
+	onWatchExpiry(ResourceType, string)            // Called when the watch timer expires for a resource.
+	onResponse(response, func()) ([]string, error) // Called when a response is received on the ADS stream.
+}
+
+// state corresponding to a resource type.
+type resourceTypeState struct {
+	version             string                                     // Last acked version. Should not be reset when the stream breaks.
+	nonce               string                                     // Last received nonce. Should be reset when the stream breaks.
+	subscribedResources map[string]*xdsresource.ResourceWatchState // Map of subscribed resource names to their state.
+}
+
+// adsStreamImpl provides the functionality associated with an ADS (Aggregated
+// Discovery Service) stream on the client side. It manages the lifecycle of the
+// ADS stream, including creating the stream, sending requests, and handling
+// responses. It also handles flow control and retries for the stream.
+type adsStreamImpl struct {
+	// The following fields are initialized from arguments passed to the
+	// constructor and are read-only afterwards, and hence can be accessed
+	// without a mutex.
+	transport          clients.Transport       // Transport to use for ADS stream.
+	eventHandler       adsStreamEventHandler   // Callbacks into the xdsChannel.
+	backoff            func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto          *v3corepb.Node          // Identifies the gRPC application.
+	watchExpiryTimeout time.Duration           // Resource watch expiry timeout
+	logger             *igrpclog.PrefixLogger
+
+	// The following fields are initialized in the constructor and are not
+	// written to afterwards, and hence can be accessed without a mutex.
+	streamCh     chan clients.Stream // New ADS streams are pushed here.
+	runnerDoneCh chan struct{}       // Notify completion of runner goroutine.
+	cancel       context.CancelFunc  // To cancel the context passed to the runner goroutine.
+	fc           *adsFlowControl     // Flow control for ADS stream.
+	notifySender chan struct{}       // To notify the sending goroutine of a pending request.
+
+	// Guards access to the below fields (and to the contents of the map).
+	mu                sync.Mutex
+	resourceTypeState map[ResourceType]*resourceTypeState // Map of resource types to their state.
+	pendingRequests   []request                           // Subscriptions and unsubscriptions are pushed here.
+
+	// The following fields are accessed atomically.
+	firstRequest       atomic.Bool // False after the first request is sent out.
+	firstStreamCreated atomic.Bool // Set to true after the very first ADS stream is created.
+	streamEstablished  atomic.Bool // Set to true when an ADS stream is established and a response is received, except for the very first stream which is set to true immediately.
+}
+
+// adsStreamOpts contains the options for creating a new ADS Stream.
+type adsStreamOpts struct {
+	transport          clients.Transport       // xDS transport to create the stream on.
+	eventHandler       adsStreamEventHandler   // Callbacks for stream events.
+	backoff            func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto          *v3corepb.Node          // Node proto to identify the gRPC application.
+	watchExpiryTimeout time.Duration           // Resource watch expiry timeout.
+	logPrefix          string                  // Prefix to be used for log messages.
+}
+
+// newADSStreamImpl initializes a new adsStreamImpl instance using the given
+// parameters.  It also launches goroutines responsible for managing reads and
+// writes for messages of the underlying stream.
+func newADSStreamImpl(opts adsStreamOpts) *adsStreamImpl {
+	s := &adsStreamImpl{
+		transport:          opts.transport,
+		eventHandler:       opts.eventHandler,
+		backoff:            opts.backoff,
+		nodeProto:          opts.nodeProto,
+		watchExpiryTimeout: opts.watchExpiryTimeout,
+
+		streamCh:          make(chan clients.Stream, 1),
+		runnerDoneCh:      make(chan struct{}),
+		fc:                newADSFlowControl(),
+		notifySender:      make(chan struct{}, 1),
+		resourceTypeState: make(map[ResourceType]*resourceTypeState),
+	}
+
+	l := grpclog.Component("xds")
+	s.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[ads-stream %p] ", s))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	s.cancel = cancel
+	go s.runner(ctx)
+	return s
+}
+
+// Stop blocks until the stream is closed and all spawned goroutines exit.
+func (s *adsStreamImpl) Stop() {
+	s.cancel()
+	s.fc.stop()
+	<-s.runnerDoneCh
+	s.logger.Infof("Shutdown ADS stream")
+}
+
+// subscribe subscribes to the given resource. It is assumed that multiple
+// subscriptions for the same resource is deduped at the caller. A discovery
+// request is sent out on the underlying stream, for the resource type with the
+// newly subscribed resource.
+func (s *adsStreamImpl) subscribe(typ ResourceType, name string) {
+	if s.logger.V(2) {
+		s.logger.Infof("Subscribing to resource %q of type %q", name, typ.TypeName)
+	}
+
+	s.mu.Lock()
+	state, ok := s.resourceTypeState[typ]
+	if !ok {
+		// An entry in the type state map is created as part of the first
+		// subscription request for this type.
+		state = &resourceTypeState{subscribedResources: make(map[string]*xdsresource.ResourceWatchState)}
+		s.resourceTypeState[typ] = state
+	}
+
+	// Create state for the newly subscribed resource. The watch timer will
+	// be started when a request for this resource is actually sent out.
+	state.subscribedResources[name] = &xdsresource.ResourceWatchState{State: xdsresource.ResourceWatchStateStarted}
+
+	// Send a request for the resource type with updated subscriptions.
+	s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)})
+	s.mu.Unlock()
+
+	select {
+	case s.notifySender <- struct{}{}:
+	default:
+	}
+}
+
+// unsubscribe cancels the subscription to the given resource. It is a no-op if
+// the given resource does not exist. The watch expiry timer associated with the
+// resource is stopped if one is active. A discovery request is sent out on the
+// stream for the resource type with the updated set of resource names.
+func (s *adsStreamImpl) unsubscribe(typ ResourceType, name string) {
+	if s.logger.V(2) {
+		s.logger.Infof("Unsubscribing to resource %q of type %q", name, typ.TypeName)
+	}
+
+	s.mu.Lock()
+	state, ok := s.resourceTypeState[typ]
+	if !ok {
+		s.mu.Unlock()
+		return
+	}
+	rs, ok := state.subscribedResources[name]
+	if !ok {
+		s.mu.Unlock()
+		return
+	}
+	if rs.ExpiryTimer != nil {
+		rs.ExpiryTimer.Stop()
+	}
+	delete(state.subscribedResources, name)
+
+	// Send a request for the resource type with updated subscriptions.
+	s.pendingRequests = append(s.pendingRequests, request{typ: typ, resourceNames: resourceNames(state.subscribedResources)})
+	s.mu.Unlock()
+
+	select {
+	case s.notifySender <- struct{}{}:
+	default:
+	}
+}
+
+// runner is a long-running goroutine that handles the lifecycle of the ADS
+// stream. It spawns another goroutine to handle writes of discovery request
+// messages on the stream. Whenever an existing stream fails, it performs
+// exponential backoff (if no messages were received on that stream) before
+// creating a new stream.
+func (s *adsStreamImpl) runner(ctx context.Context) {
+	defer close(s.runnerDoneCh)
+
+	go s.send(ctx)
+
+	runStreamWithBackoff := func() error {
+		stream, err := s.transport.NewStream(ctx, "/envoy.service.discovery.v3.AggregatedDiscoveryService/StreamAggregatedResources")
+		if err != nil {
+			s.logger.Warningf("Failed to create a new ADS streaming RPC: %v", err)
+			s.streamEstablished.Store(false)
+			s.onError(err, false)
+			return nil
+		}
+		if s.logger.V(2) {
+			s.logger.Infof("ADS stream created")
+		}
+
+		s.firstRequest.Store(true)
+		if !s.firstStreamCreated.Load() {
+			s.streamEstablished.Store(true)
+			s.firstStreamCreated.Store(true)
+		}
+
+		// Ensure that the most recently created stream is pushed on the
+		// channel for the `send` goroutine to consume.
+		select {
+		case <-s.streamCh:
+		default:
+		}
+		s.streamCh <- stream
+
+		// Backoff state is reset upon successful receipt of at least one
+		// message from the server.
+		if s.recv(stream) {
+			return backoff.ErrResetBackoff
+		}
+		return nil
+	}
+	backoff.RunF(ctx, runStreamWithBackoff, s.backoff)
+}
+
+// send is a long running goroutine that handles sending discovery requests for
+// two scenarios:
+// - a new subscription or unsubscription request is received
+// - a new stream is created after the previous one failed
+func (s *adsStreamImpl) send(ctx context.Context) {
+	// Stores the most recent stream instance received on streamCh.
+	var stream clients.Stream
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case stream = <-s.streamCh:
+			if err := s.sendExisting(stream); err != nil {
+				// Send failed, clear the current stream. Attempt to resend will
+				// only be made after a new stream is created.
+				stream = nil
+				continue
+			}
+		case <-s.notifySender:
+			// If there's no stream yet, skip the request. This request will be resent
+			// when a new stream is created. If no stream is created, the watcher will
+			// timeout (same as server not sending response back).
+			if stream == nil {
+				continue
+			}
+
+			// Resetting the pendingRequests slice to nil works for both cases:
+			// - When we successfully sends the requests out on the wire.
+			// - When sending fails. This can happen only when the stream fails,
+			//   and in this case, we rely on the `sendExisting` to send out
+			//   requests for all subscriptions when the stream is recreated.
+			s.mu.Lock()
+			if err := s.sendNewLocked(stream, s.pendingRequests); err != nil {
+				stream = nil
+			}
+			s.pendingRequests = nil
+			s.mu.Unlock()
+		}
+	}
+}
+
+// sendNewLocked attempts to send a discovery request based on a new subscription or
+// unsubscription. This method also starts the watch expiry timer for resources
+// that were sent in the request for the first time, i.e. their watch state is
+// `watchStateStarted`.
+//
+// Caller needs to hold c.mu.
+func (s *adsStreamImpl) sendNewLocked(stream clients.Stream, requests []request) error {
+	for _, req := range requests {
+		state := s.resourceTypeState[req.typ]
+		if err := s.sendMessageLocked(stream, req.resourceNames, req.typ.TypeURL, state.version, state.nonce, nil); err != nil {
+			return err
+		}
+		s.startWatchTimersLocked(req.typ, req.resourceNames)
+	}
+	return nil
+}
+
+// sendExisting sends out discovery requests for existing resources when
+// recovering from a broken stream.
+//
+// The stream argument is guaranteed to be non-nil.
+func (s *adsStreamImpl) sendExisting(stream clients.Stream) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Clear any queued requests. Previously subscribed to resources will be
+	// resent below.
+	s.pendingRequests = nil
+
+	for typ, state := range s.resourceTypeState {
+		// Reset only the nonces map when the stream restarts.
+		//
+		// xDS spec says the following. See section:
+		// https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-resource-type-instance-version
+		//
+		// Note that the version for a resource type is not a property of an
+		// individual xDS stream but rather a property of the resources
+		// themselves. If the stream becomes broken and the client creates a new
+		// stream, the client’s initial request on the new stream should
+		// indicate the most recent version seen by the client on the previous
+		// stream
+		state.nonce = ""
+
+		if len(state.subscribedResources) == 0 {
+			continue
+		}
+
+		names := resourceNames(state.subscribedResources)
+		if err := s.sendMessageLocked(stream, names, typ.TypeURL, state.version, state.nonce, nil); err != nil {
+			return err
+		}
+		s.startWatchTimersLocked(typ, names)
+	}
+	return nil
+}
+
+// sendMessageLocked sends a discovery request to the server, populating the
+// different fields of the message with the given parameters. Returns a non-nil
+// error if the request could not be sent.
+//
+// Caller needs to hold c.mu.
+func (s *adsStreamImpl) sendMessageLocked(stream clients.Stream, names []string, url, version, nonce string, nackErr error) error {
+	req := &v3discoverypb.DiscoveryRequest{
+		ResourceNames: names,
+		TypeUrl:       url,
+		VersionInfo:   version,
+		ResponseNonce: nonce,
+	}
+
+	// The xDS protocol only requires that we send the node proto in the first
+	// discovery request on every stream. Sending the node proto in every
+	// request wastes CPU resources on the client and the server.
+	if s.firstRequest.Load() {
+		req.Node = s.nodeProto
+	}
+
+	if nackErr != nil {
+		req.ErrorDetail = &statuspb.Status{
+			Code: int32(cpb.Code_INVALID_ARGUMENT), Message: nackErr.Error(),
+		}
+	}
+
+	msg, err := proto.Marshal(req)
+	if err != nil {
+		s.logger.Warningf("Failed to marshal DiscoveryRequest: %v", err)
+		return err
+	}
+	if err := stream.Send(msg); err != nil {
+		s.logger.Warningf("Sending ADS request for type %q, resources: %v, version: %q, nonce: %q failed: %v", url, names, version, nonce, err)
+		return err
+	}
+	s.firstRequest.Store(false)
+
+	if s.logger.V(perRPCVerbosityLevel) {
+		s.logger.Infof("ADS request sent: %v", pretty.ToJSON(req))
+	} else if s.logger.V(2) {
+		s.logger.Infof("ADS request sent for type %q, resources: %v, version: %q, nonce: %q", url, names, version, nonce)
+	}
+
+	return nil
+}
+
+// recv is responsible for receiving messages from the ADS stream.
+//
+// It performs the following actions:
+//   - Waits for local flow control to be available before it receives a message
+//     from the ADS stream. If an error is encountered here, it is handled by
+//     the onError method which propagates the error to all watchers.
+//   - Invokes the event handler's OnADSResponse method to process the message.
+//   - Sends an ACK or NACK to the server based on the response.
+//
+// It returns a boolean indicating whether at least one message was received
+// from the server.
+func (s *adsStreamImpl) recv(stream clients.Stream) bool {
+	msgReceived := false
+	for {
+		// Wait for ADS stream level flow control to be available.
+		if s.fc.wait() {
+			if s.logger.V(2) {
+				s.logger.Infof("ADS stream stopped while waiting for flow control")
+			}
+			return msgReceived
+		}
+
+		resources, url, version, nonce, err := s.recvMessage(stream)
+		if err != nil {
+			if !msgReceived {
+				s.streamEstablished.Store(false)
+			}
+			s.onError(err, msgReceived)
+			s.logger.Warningf("ADS stream closed: %v", err)
+			return msgReceived
+		}
+		s.streamEstablished.Store(true)
+		msgReceived = true
+
+		// Invoke the onResponse event handler to parse the incoming message and
+		// decide whether to send an ACK or NACK.
+		resp := response{
+			resources: resources,
+			typeURL:   url,
+			version:   version,
+		}
+		var resourceNames []string
+		var nackErr error
+		s.fc.setPending(true)
+		resourceNames, nackErr = s.eventHandler.onResponse(resp, sync.OnceFunc(func() { s.fc.setPending(false) }))
+		if xdsresource.ErrType(nackErr) == xdsresource.ErrorTypeResourceTypeUnsupported {
+			// A general guiding principle is that if the server sends
+			// something the client didn't actually subscribe to, then the
+			// client ignores it. Here, we have received a response with
+			// resources of a type that we don't know about.
+			//
+			// Sending a NACK doesn't really seem appropriate here, since we're
+			// not actually validating what the server sent and therefore don't
+			// know that it's invalid.  But we shouldn't ACK either, because we
+			// don't know that it is valid.
+			s.logger.Warningf("%v", nackErr)
+			continue
+		}
+
+		s.onRecv(stream, resourceNames, url, version, nonce, nackErr)
+	}
+}
+
+func (s *adsStreamImpl) recvMessage(stream clients.Stream) (resources []*anypb.Any, url, version, nonce string, err error) {
+	r, err := stream.Recv()
+	if err != nil {
+		return nil, "", "", "", err
+	}
+	var resp v3discoverypb.DiscoveryResponse
+	if err := proto.Unmarshal(r, &resp); err != nil {
+		s.logger.Infof("Failed to unmarshal response to DiscoveryResponse: %v", err)
+		return nil, "", "", "", fmt.Errorf("unexpected message type %T", r)
+	}
+	if s.logger.V(perRPCVerbosityLevel) {
+		s.logger.Infof("ADS response received: %v", pretty.ToJSON(&resp))
+	} else if s.logger.V(2) {
+		s.logger.Infof("ADS response received for type %q, version %q, nonce %q", resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce())
+	}
+	return resp.GetResources(), resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), nil
+}
+
+// onRecv is invoked when a response is received from the server. The arguments
+// passed to this method correspond to the most recently received response.
+//
+// It performs the following actions:
+//   - updates resource type specific state
+//   - updates resource specific state for resources in the response
+//   - sends an ACK or NACK to the server based on the response
+func (s *adsStreamImpl) onRecv(stream clients.Stream, names []string, url, version, nonce string, nackErr error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Lookup the resource type specific state based on the type URL.
+	var typ ResourceType
+	for t := range s.resourceTypeState {
+		if t.TypeURL == url {
+			typ = t
+			break
+		}
+	}
+	typeState, ok := s.resourceTypeState[typ]
+	if !ok {
+		s.logger.Warningf("ADS stream received a response for type %q, but no state exists for it", url)
+		return
+	}
+
+	// Update the resource type specific state. This includes:
+	//   - updating the nonce unconditionally
+	//   - updating the version only if the response is to be ACKed
+	previousVersion := typeState.version
+	typeState.nonce = nonce
+	if nackErr == nil {
+		typeState.version = version
+	}
+
+	// Update the resource specific state. For all resources received as
+	// part of this response that are in state `started` or `requested`,
+	// this includes:
+	//   - setting the watch state to watchstateReceived
+	//   - stopping the expiry timer, if one exists
+	for _, name := range names {
+		rs, ok := typeState.subscribedResources[name]
+		if !ok {
+			s.logger.Warningf("ADS stream received a response for resource %q, but no state exists for it", name)
+			continue
+		}
+		if ws := rs.State; ws == xdsresource.ResourceWatchStateStarted || ws == xdsresource.ResourceWatchStateRequested {
+			rs.State = xdsresource.ResourceWatchStateReceived
+			if rs.ExpiryTimer != nil {
+				rs.ExpiryTimer.Stop()
+				rs.ExpiryTimer = nil
+			}
+		}
+	}
+
+	// Send an ACK or NACK.
+	subscribedResourceNames := resourceNames(typeState.subscribedResources)
+	if nackErr != nil {
+		s.logger.Warningf("Sending NACK for resource type: %q, version: %q, nonce: %q, reason: %v", url, version, nonce, nackErr)
+		s.sendMessageLocked(stream, subscribedResourceNames, url, previousVersion, nonce, nackErr)
+		return
+	}
+
+	if s.logger.V(2) {
+		s.logger.Infof("Sending ACK for resource type: %q, version: %q, nonce: %q", url, version, nonce)
+	}
+	s.sendMessageLocked(stream, subscribedResourceNames, url, version, nonce, nil)
+}
+
+// onError is called when an error occurs on the ADS stream. It stops any
+// outstanding resource timers and resets the watch state to started for any
+// resources that were in the requested state. It also handles the case where
+// the ADS stream was closed after receiving a response, which is not
+// considered an error.
+func (s *adsStreamImpl) onError(err error, msgReceived bool) {
+	// For resources that been requested but not yet responded to by the
+	// management server, stop the resource timers and reset the watch state to
+	// watchStateStarted. This is because we don't want the expiry timer to be
+	// running when we don't have a stream open to the management server.
+	s.mu.Lock()
+	for _, state := range s.resourceTypeState {
+		for _, rs := range state.subscribedResources {
+			if rs.State != xdsresource.ResourceWatchStateRequested {
+				continue
+			}
+			if rs.ExpiryTimer != nil {
+				rs.ExpiryTimer.Stop()
+				rs.ExpiryTimer = nil
+			}
+			rs.State = xdsresource.ResourceWatchStateStarted
+		}
+	}
+	s.mu.Unlock()
+
+	// Note that we do not consider it an error if the ADS stream was closed
+	// after having received a response on the stream. This is because there
+	// are legitimate reasons why the server may need to close the stream during
+	// normal operations, such as needing to rebalance load or the underlying
+	// connection hitting its max connection age limit.
+	// (see [gRFC A9](https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md)).
+	if msgReceived {
+		err = xdsresource.NewError(xdsresource.ErrTypeStreamFailedAfterRecv, err.Error())
+	}
+
+	s.eventHandler.onStreamError(err)
+}
+
+// startWatchTimersLocked starts the expiry timers for the given resource names
+// of the specified resource type.  For each resource name, if the resource
+// watch state is in the "started" state, it transitions the state to
+// "requested" and starts an expiry timer. When the timer expires, the resource
+// watch state is set to "timeout" and the event handler callback is called.
+//
+// The caller must hold the s.mu lock.
+func (s *adsStreamImpl) startWatchTimersLocked(typ ResourceType, names []string) {
+	typeState := s.resourceTypeState[typ]
+	for _, name := range names {
+		resourceState, ok := typeState.subscribedResources[name]
+		if !ok {
+			continue
+		}
+		if resourceState.State != xdsresource.ResourceWatchStateStarted {
+			continue
+		}
+		resourceState.State = xdsresource.ResourceWatchStateRequested
+
+		rs := resourceState
+		resourceState.ExpiryTimer = time.AfterFunc(s.watchExpiryTimeout, func() {
+			s.mu.Lock()
+			rs.State = xdsresource.ResourceWatchStateTimeout
+			rs.ExpiryTimer = nil
+			s.mu.Unlock()
+			s.eventHandler.onWatchExpiry(typ, name)
+		})
+	}
+}
+
+func (s *adsStreamImpl) adsResourceWatchStateForTesting(rType ResourceType, resourceName string) (xdsresource.ResourceWatchState, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	state, ok := s.resourceTypeState[rType]
+	if !ok {
+		return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource type: %v", rType)
+	}
+	resourceState, ok := state.subscribedResources[resourceName]
+	if !ok {
+		return xdsresource.ResourceWatchState{}, fmt.Errorf("unknown resource name: %v", resourceName)
+	}
+	return *resourceState, nil
+}
+
+func resourceNames(m map[string]*xdsresource.ResourceWatchState) []string {
+	ret := make([]string, len(m))
+	idx := 0
+	for name := range m {
+		ret[idx] = name
+		idx++
+	}
+	return ret
+}
+
+// adsFlowControl implements ADS stream level flow control that enables the ADS
+// stream to block the reading of the next message until the previous update is
+// consumed by all watchers.
+//
+// The lifetime of the flow control is tied to the lifetime of the stream. When
+// the stream is closed, it is the responsibility of the caller to stop the flow
+// control. This ensures that any goroutine blocked on the flow control's wait
+// method is unblocked.
+type adsFlowControl struct {
+	mu sync.Mutex
+	// cond is used to signal when the most recent update has been consumed, or
+	// the flow control has been stopped (in which case, waiters should be
+	// unblocked as well).
+	cond    *sync.Cond
+	pending bool // indicates if the most recent update is pending consumption
+	stopped bool // indicates if the ADS stream has been stopped
+}
+
+// newADSFlowControl returns a new adsFlowControl.
+func newADSFlowControl() *adsFlowControl {
+	fc := &adsFlowControl{}
+	fc.cond = sync.NewCond(&fc.mu)
+	return fc
+}
+
+// stop marks the flow control as stopped and signals the condition variable to
+// unblock any goroutine waiting on it.
+func (fc *adsFlowControl) stop() {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	fc.stopped = true
+	fc.cond.Broadcast()
+}
+
+// setPending changes the internal state to indicate whether there is an update
+// pending consumption by all watchers. If there is no longer a pending update,
+// the condition variable is signaled to allow the recv method to proceed.
+func (fc *adsFlowControl) setPending(pending bool) {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	if fc.stopped {
+		return
+	}
+
+	fc.pending = pending
+	if !pending {
+		fc.cond.Broadcast()
+	}
+}
+
+// wait blocks until all the watchers have consumed the most recent update.
+// Returns true if the flow control was stopped while waiting, false otherwise.
+func (fc *adsFlowControl) wait() bool {
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+
+	for fc.pending && !fc.stopped {
+		fc.cond.Wait()
+	}
+
+	return fc.stopped
+}
+
+func (s *adsStreamImpl) isStreamEstablished() bool {
+	return s.streamEstablished.Load()
+}

internal/xds/clients/xdsclient/authority.go

@@ -0,0 +1,996 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"google.golang.org/grpc/grpclog"
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/internal/syncutil"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+)
+
+type resourceState struct {
+	watchers          map[ResourceWatcher]bool       // Set of watchers for this resource.
+	cache             ResourceData                   // Most recent ACKed update for this resource.
+	md                xdsresource.UpdateMetadata     // Metadata for the most recent update.
+	deletionIgnored   bool                           // True, if resource deletion was ignored for a prior update.
+	xdsChannelConfigs map[*xdsChannelWithConfig]bool // Set of xdsChannels where this resource is subscribed.
+}
+
+// xdsChannelForADS is used to acquire a reference to an xdsChannel. This
+// functionality is provided by the xdsClient.
+//
+// The arguments to the function are as follows:
+//   - the server config for the xdsChannel
+//   - the calling authority on which a set of callbacks are invoked by the
+//     xdsChannel on ADS stream events
+//
+// Returns a reference to the xdsChannel and a function to release the same. A
+// non-nil error is returned if the channel creation fails and the first two
+// return values are meaningless in this case.
+type xdsChannelForADS func(*ServerConfig, *authority) (*xdsChannel, func(), error)
+
+// xdsChannelWithConfig is a struct that holds an xdsChannel and its associated
+// ServerConfig, along with a cleanup function to release the xdsChannel.
+type xdsChannelWithConfig struct {
+	channel      *xdsChannel
+	serverConfig *ServerConfig
+	cleanup      func()
+}
+
+// authority provides the functionality required to communicate with a
+// management server corresponding to an authority name specified in the
+// xDS client configuration.
+//
+// It holds references to one or more xdsChannels, one for each server
+// configuration in the config, to allow fallback from a primary management
+// server to a secondary management server. Authorities that contain similar
+// server configuration entries will end up sharing the xdsChannel for that
+// server configuration. The xdsChannels are owned and managed by the xdsClient.
+//
+// It also contains a cache of resource state for resources requested from
+// management server(s). This cache contains the list of registered watchers and
+// the most recent resource configuration received from the management server.
+type authority struct {
+	// The following fields are initialized at creation time and are read-only
+	// afterwards, and therefore don't need to be protected with a mutex.
+	name                      string                       // Name of the authority from xDS client configuration.
+	watcherCallbackSerializer *syncutil.CallbackSerializer // Serializer to run watcher callbacks, owned by the xDS client implementation.
+	getChannelForADS          xdsChannelForADS             // Function to get an xdsChannel for ADS, provided by the xDS client implementation.
+	xdsClientSerializer       *syncutil.CallbackSerializer // Serializer to run call ins from the xDS client, owned by this authority.
+	xdsClientSerializerClose  func()                       // Function to close the above serializer.
+	logger                    *igrpclog.PrefixLogger       // Logger for this authority.
+	target                    string                       // The gRPC Channel target.
+	metricsReporter           clients.MetricsReporter
+
+	// The below defined fields must only be accessed in the context of the
+	// serializer callback, owned by this authority.
+
+	// A two level map containing the state of all the resources being watched.
+	//
+	// The first level map key is the ResourceType (Listener, Route etc). This
+	// allows us to have a single map for all resources instead of having per
+	// resource-type maps.
+	//
+	// The second level map key is the resource name, with the value being the
+	// actual state of the resource.
+	resources map[ResourceType]map[string]*resourceState
+
+	// An ordered list of xdsChannels corresponding to the list of server
+	// configurations specified for this authority in the config. The
+	// ordering specifies the order in which these channels are preferred for
+	// fallback.
+	xdsChannelConfigs []*xdsChannelWithConfig
+
+	// The current active xdsChannel. Here, active does not mean that the
+	// channel has a working connection to the server. It simply points to the
+	// channel that we are trying to work with, based on fallback logic.
+	activeXDSChannel *xdsChannelWithConfig
+}
+
+// authorityBuildOptions wraps arguments required to create a new authority.
+type authorityBuildOptions struct {
+	serverConfigs    []ServerConfig               // Server configs for the authority
+	name             string                       // Name of the authority
+	serializer       *syncutil.CallbackSerializer // Callback serializer for invoking watch callbacks
+	getChannelForADS xdsChannelForADS             // Function to acquire a reference to an xdsChannel
+	logPrefix        string                       // Prefix for logging
+	target           string                       // Target for the gRPC Channel that owns xDS Client/Authority
+	metricsReporter  clients.MetricsReporter      // Metrics reporter for the authority
+}
+
+// newAuthority creates a new authority instance with the provided
+// configuration. The authority is responsible for managing the state of
+// resources requested from the management server, as well as acquiring and
+// releasing references to channels used to communicate with the management
+// server.
+//
+// Note that no channels to management servers are created at this time. Instead
+// a channel to the first server configuration is created when the first watch
+// is registered, and more channels are created as needed by the fallback logic.
+func newAuthority(args authorityBuildOptions) *authority {
+	ctx, cancel := context.WithCancel(context.Background())
+	l := grpclog.Component("xds")
+	logPrefix := args.logPrefix + fmt.Sprintf("[authority %q] ", args.name)
+	ret := &authority{
+		name:                      args.name,
+		watcherCallbackSerializer: args.serializer,
+		getChannelForADS:          args.getChannelForADS,
+		xdsClientSerializer:       syncutil.NewCallbackSerializer(ctx),
+		xdsClientSerializerClose:  cancel,
+		logger:                    igrpclog.NewPrefixLogger(l, logPrefix),
+		resources:                 make(map[ResourceType]map[string]*resourceState),
+		target:                    args.target,
+		metricsReporter:           args.metricsReporter,
+	}
+
+	// Create an ordered list of xdsChannels with their server configs. The
+	// actual channel to the first server configuration is created when the
+	// first watch is registered, and channels to other server configurations
+	// are created as needed to support fallback.
+	for _, sc := range args.serverConfigs {
+		ret.xdsChannelConfigs = append(ret.xdsChannelConfigs, &xdsChannelWithConfig{serverConfig: &sc})
+	}
+	return ret
+}
+
+// adsStreamFailure is called to notify the authority about an ADS stream
+// failure on an xdsChannel to the management server identified by the provided
+// server config. The error is forwarded to all the resource watchers.
+//
+// This method is called by the xDS client implementation (on all interested
+// authorities) when a stream error is reported by an xdsChannel.
+//
+// Errors of type xdsresource.ErrTypeStreamFailedAfterRecv are ignored.
+func (a *authority) adsStreamFailure(serverConfig *ServerConfig, err error) {
+	a.xdsClientSerializer.TrySchedule(func(context.Context) {
+		a.handleADSStreamFailure(serverConfig, err)
+	})
+}
+
+// Handles ADS stream failure by invoking watch callbacks and triggering
+// fallback if the associated conditions are met.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) handleADSStreamFailure(serverConfig *ServerConfig, err error) {
+	if a.logger.V(2) {
+		a.logger.Infof("Connection to server %s failed with error: %v", serverConfig, err)
+	}
+
+	// We do not consider it an error if the ADS stream was closed after having
+	// received a response on the stream. This is because there are legitimate
+	// reasons why the server may need to close the stream during normal
+	// operations, such as needing to rebalance load or the underlying
+	// connection hitting its max connection age limit. See gRFC A57 for more
+	// details.
+	if xdsresource.ErrType(err) == xdsresource.ErrTypeStreamFailedAfterRecv {
+		a.logger.Warningf("Watchers not notified since ADS stream failed after having received at least one response: %v", err)
+		return
+	}
+
+	// Two conditions need to be met for fallback to be triggered:
+	// 1. There is a connectivity failure on the ADS stream, as described in
+	//    gRFC A57. For us, this means that the ADS stream was closed before the
+	//    first server response was received. We already checked that condition
+	//    earlier in this method.
+	// 2. There is at least one watcher for a resource that is not cached.
+	//    Cached resources include ones that
+	//    - have been successfully received and can be used.
+	//    - are considered non-existent according to xDS Protocol Specification.
+	if !a.watcherExistsForUncachedResource() {
+		if a.logger.V(2) {
+			a.logger.Infof("No watchers for uncached resources. Not triggering fallback")
+		}
+		// Since we are not triggering fallback, propagate the connectivity
+		// error to all watchers and return early.
+		a.propagateConnectivityErrorToAllWatchers(err)
+		return
+	}
+
+	// Attempt to fallback to servers with lower priority than the failing one.
+	currentServerIdx := a.serverIndexForConfig(serverConfig)
+	for i := currentServerIdx + 1; i < len(a.xdsChannelConfigs); i++ {
+		if a.fallbackToServer(a.xdsChannelConfigs[i]) {
+			// Since we have successfully triggered fallback, we don't have to
+			// notify watchers about the connectivity error.
+			return
+		}
+	}
+
+	// Having exhausted all available servers, we must notify watchers of the
+	// connectivity error - A71.
+	a.propagateConnectivityErrorToAllWatchers(err)
+}
+
+// propagateConnectivityErrorToAllWatchers propagates the given connection error
+// to all watchers of all resources.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) propagateConnectivityErrorToAllWatchers(err error) {
+	for _, rType := range a.resources {
+		for _, state := range rType {
+			for watcher := range state.watchers {
+				if state.cache == nil {
+					a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
+						watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {})
+					})
+				} else {
+					a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
+						watcher.AmbientError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err), func() {})
+					})
+				}
+			}
+		}
+	}
+}
+
+// serverIndexForConfig returns the index of the xdsChannelConfig matching the
+// provided server config, panicking if no match is found (which indicates a
+// programming error).
+func (a *authority) serverIndexForConfig(sc *ServerConfig) int {
+	for i, cfg := range a.xdsChannelConfigs {
+		if isServerConfigEqual(sc, cfg.serverConfig) {
+			return i
+		}
+	}
+	panic(fmt.Sprintf("no server config matching %v found", sc))
+}
+
+// Determines the server to fallback to and triggers fallback to the same. If
+// required, creates an xdsChannel to that server, and re-subscribes to all
+// existing resources.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) fallbackToServer(xc *xdsChannelWithConfig) bool {
+	if a.logger.V(2) {
+		a.logger.Infof("Attempting to initiate fallback to server %q", xc.serverConfig)
+	}
+
+	if xc.channel != nil {
+		if a.logger.V(2) {
+			a.logger.Infof("Channel to the next server in the list %q already exists", xc.serverConfig)
+		}
+		return false
+	}
+
+	channel, cleanup, err := a.getChannelForADS(xc.serverConfig, a)
+	if err != nil {
+		a.logger.Errorf("Failed to create xDS channel: %v", err)
+		return false
+	}
+	xc.channel = channel
+	xc.cleanup = cleanup
+	a.activeXDSChannel = xc
+
+	// Subscribe to all existing resources from the new management server.
+	for typ, resources := range a.resources {
+		for name, state := range resources {
+			if a.logger.V(2) {
+				a.logger.Infof("Resubscribing to resource of type %q and name %q", typ.TypeName, name)
+			}
+			xc.channel.subscribe(typ, name)
+
+			// Add the new channel to the list of xdsChannels from which this
+			// resource has been requested from. Retain the cached resource and
+			// the set of existing watchers (and other metadata fields) in the
+			// resource state.
+			state.xdsChannelConfigs[xc] = true
+		}
+	}
+	return true
+}
+
+// adsResourceUpdate is called to notify the authority about a resource update
+// received on the ADS stream.
+//
+// This method is called by the xDS client implementation (on all interested
+// authorities) when a stream error is reported by an xdsChannel.
+func (a *authority) adsResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
+	a.xdsClientSerializer.TrySchedule(func(context.Context) {
+		a.handleADSResourceUpdate(serverConfig, rType, updates, md, onDone)
+	})
+}
+
+// handleADSResourceUpdate processes an update from the xDS client, updating the
+// resource cache and notifying any registered watchers of the update.
+//
+// If the update is received from a higher priority xdsChannel that was
+// previously down, we revert to it and close all lower priority xdsChannels.
+//
+// Once the update has been processed by all watchers, the authority is expected
+// to invoke the onDone callback.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) handleADSResourceUpdate(serverConfig *ServerConfig, rType ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
+	if !a.handleRevertingToPrimaryOnUpdate(serverConfig) {
+		return
+	}
+
+	// We build a list of callback funcs to invoke, and invoke them at the end
+	// of this method instead of inline (when handling the update for a
+	// particular resource), because we want to make sure that all calls to
+	// increment watcherCnt happen before any callbacks are invoked. This will
+	// ensure that the onDone callback is never invoked before all watcher
+	// callbacks are invoked, and the watchers have processed the update.
+	watcherCnt := new(atomic.Int64)
+	done := func() {
+		if watcherCnt.Add(-1) == 0 {
+			onDone()
+		}
+	}
+	var funcsToSchedule []func(context.Context)
+	defer func() {
+		if len(funcsToSchedule) == 0 {
+			// When there are no watchers for the resources received as part of
+			// this update, invoke onDone explicitly to unblock the next read on
+			// the ADS stream.
+			onDone()
+			return
+		}
+		for _, f := range funcsToSchedule {
+			a.watcherCallbackSerializer.ScheduleOr(f, onDone)
+		}
+	}()
+
+	resourceStates := a.resources[rType]
+	for name, uErr := range updates {
+		state, ok := resourceStates[name]
+		if !ok {
+			continue
+		}
+
+		if err := uErr.Err; err != nil {
+			if a.metricsReporter != nil {
+				a.metricsReporter.ReportMetric(&metrics.ResourceUpdateInvalid{
+					ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName,
+				})
+			}
+
+			// Notify watchers only if this is not a duplicated error from the previous update.
+			if errState := state.md.ErrState; errState == nil || errState.Err == nil || state.md.ErrState.Err.Error() != err.Error() {
+				for watcher := range state.watchers {
+					watcherCnt.Add(1)
+					if state.cache == nil {
+						funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceError(err, done) })
+					} else {
+						funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.AmbientError(err, done) })
+					}
+				}
+			}
+
+			// On error, keep previous version of the resource. But update
+			// status to NACKed and capture the specific per-resource error (and
+			// not the overall error for the update) in the ErrState field.
+			state.md.Status = xdsresource.ServiceStatusNACKed
+			state.md.ErrState = &xdsresource.UpdateErrorMetadata{
+				Version:   md.ErrState.Version,
+				Err:       err,
+				Timestamp: md.ErrState.Timestamp,
+			}
+
+			continue
+		}
+
+		if a.metricsReporter != nil {
+			a.metricsReporter.ReportMetric(&metrics.ResourceUpdateValid{
+				ServerURI: serverConfig.ServerIdentifier.ServerURI, ResourceType: rType.TypeName,
+			})
+		}
+
+		if state.deletionIgnored {
+			state.deletionIgnored = false
+			a.logger.Infof("A valid update was received for resource %q of type %q after previously ignoring a deletion", name, rType.TypeName)
+		}
+		// Notify watchers if any of these conditions are met:
+		//   - this is the first update for this resource
+		//   - this update is different from the one currently cached
+		//   - the previous update for this resource was NACKed, but the update
+		//     before that was the same as this update.
+		if state.cache == nil || !state.cache.Equal(uErr.Resource) || state.md.ErrState != nil {
+			// Update the resource cache.
+			if a.logger.V(2) {
+				a.logger.Infof("Resource type %q with name %q added to cache", rType.TypeName, name)
+			}
+			state.cache = uErr.Resource
+
+			for watcher := range state.watchers {
+				resource := uErr.Resource
+				watcherCnt.Add(1)
+				funcsToSchedule = append(funcsToSchedule, func(context.Context) { watcher.ResourceChanged(resource, done) })
+			}
+		}
+
+		// Set status to ACK, and clear error state. The metadata might be a
+		// NACK metadata because some other resources in the same response
+		// are invalid.
+		state.md = md
+		state.md.ErrState = nil
+		state.md.Status = xdsresource.ServiceStatusACKed
+		if md.ErrState != nil {
+			state.md.Version = md.ErrState.Version
+		}
+	}
+
+	// If this resource type requires that all resources be present in every
+	// SotW response from the server, a response that does not include a
+	// previously seen resource will be interpreted as a deletion of that
+	// resource unless ignore_resource_deletion option was set in the server
+	// config.
+	if !rType.AllResourcesRequiredInSotW {
+		return
+	}
+	for name, state := range resourceStates {
+		if state.cache == nil {
+			// If the resource state does not contain a cached update, which can
+			// happen when:
+			// - resource was newly requested but has not yet been received, or,
+			// - resource was removed as part of a previous update,
+			// we don't want to generate an error for the watchers.
+			//
+			// For the first of the above two conditions, this ADS response may
+			// be in reaction to an earlier request that did not yet request the
+			// new resource, so its absence from the response does not
+			// necessarily indicate that the resource does not exist. For that
+			// case, we rely on the request timeout instead.
+			//
+			// For the second of the above two conditions, we already generated
+			// an error when we received the first response which removed this
+			// resource. So, there is no need to generate another one.
+			continue
+		}
+		if _, ok := updates[name]; ok {
+			// If the resource was present in the response, move on.
+			continue
+		}
+		if state.md.Status == xdsresource.ServiceStatusNotExist {
+			// The metadata status is set to "ServiceStatusNotExist" if a
+			// previous update deleted this resource, in which case we do not
+			// want to repeatedly call the watch callbacks with a
+			// "resource-not-found" error.
+			continue
+		}
+		if serverConfig.SupportsServerFeature(ServerFeatureIgnoreResourceDeletion) {
+			// Per A53, resource deletions are ignored if the
+			// `ignore_resource_deletion` server feature is enabled through the
+			// xDS client configuration. If the resource deletion is to be
+			// ignored, the resource is not removed from the cache and the
+			// corresponding OnResourceDoesNotExist() callback is not invoked on
+			// the watchers.
+			if !state.deletionIgnored {
+				state.deletionIgnored = true
+				a.logger.Warningf("Ignoring resource deletion for resource %q of type %q", name, rType.TypeName)
+			}
+			continue
+		}
+
+		// If we get here, it means that the resource exists in cache, but not
+		// in the new update. Delete the resource from cache, and send a
+		// resource not found error to indicate that the resource has been
+		// removed. Metadata for the resource is still maintained, as this is
+		// required by CSDS.
+		state.cache = nil
+		state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}
+		for watcher := range state.watchers {
+			watcherCnt.Add(1)
+			funcsToSchedule = append(funcsToSchedule, func(context.Context) {
+				watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", name, rType.TypeName), done)
+			})
+		}
+	}
+}
+
+// adsResourceDoesNotExist is called by the xDS client implementation (on all
+// interested authorities) to notify the authority that a subscribed resource
+// does not exist.
+func (a *authority) adsResourceDoesNotExist(rType ResourceType, resourceName string) {
+	a.xdsClientSerializer.TrySchedule(func(context.Context) {
+		a.handleADSResourceDoesNotExist(rType, resourceName)
+	})
+}
+
+// handleADSResourceDoesNotExist is called when a subscribed resource does not
+// exist. It removes the resource from the cache, updates the metadata status
+// to ServiceStatusNotExist, and notifies all watchers that the resource does
+// not exist.
+func (a *authority) handleADSResourceDoesNotExist(rType ResourceType, resourceName string) {
+	if a.logger.V(2) {
+		a.logger.Infof("Watch for resource %q of type %s timed out", resourceName, rType.TypeName)
+	}
+
+	resourceStates := a.resources[rType]
+	if resourceStates == nil {
+		if a.logger.V(2) {
+			a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName)
+		}
+		return
+	}
+	state, ok := resourceStates[resourceName]
+	if !ok {
+		if a.logger.V(2) {
+			a.logger.Infof("Resource %q of type %s currently not being watched", resourceName, rType.TypeName)
+		}
+		return
+	}
+
+	state.cache = nil
+	state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}
+	for watcher := range state.watchers {
+		watcher := watcher
+		a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
+			watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {})
+		})
+	}
+}
+
+// handleRevertingToPrimaryOnUpdate is called when a resource update is received
+// from the xDS client.
+//
+// If the update is from the currently active server, nothing is done. Else, all
+// lower priority servers are closed and the active server is reverted to the
+// highest priority server that sent the update.
+//
+// The return value indicates whether subsequent processing of the resource
+// update should continue or not.
+//
+// This method is only executed in the context of a serializer callback.
+func (a *authority) handleRevertingToPrimaryOnUpdate(serverConfig *ServerConfig) bool {
+	if a.activeXDSChannel == nil {
+		// This can happen only when all watches on this authority have been
+		// removed, and the xdsChannels have been closed. This update should
+		// have been received prior to closing of the channel, and therefore
+		// must be ignored.
+		return false
+	}
+
+	if isServerConfigEqual(serverConfig, a.activeXDSChannel.serverConfig) {
+		// If the resource update is from the current active server, nothing
+		// needs to be done from fallback point of view.
+		return true
+	}
+
+	// If the resource update is not from the current active server, it means
+	// that we have received an update either from:
+	// - a server that has a higher priority than the current active server and
+	//   therefore we need to revert back to it and close all lower priority
+	//   servers, or,
+	// - a server that has a lower priority than the current active server. This
+	//   can happen when the server close and the response race against each
+	//   other. We can safely ignore this update, since we have already reverted
+	//   to the higher priority server, and closed all lower priority servers.
+	serverIdx := a.serverIndexForConfig(serverConfig)
+	activeServerIdx := a.serverIndexForConfig(a.activeXDSChannel.serverConfig)
+	if activeServerIdx < serverIdx {
+		if a.logger.V(2) {
+			a.logger.Infof("Ignoring update from lower priority server [%d] %q", serverIdx, serverConfig)
+		}
+		return false
+	}
+	if a.logger.V(2) {
+		a.logger.Infof("Received update from higher priority server [%d] %q", serverIdx, serverConfig)
+	}
+
+	// At this point, we are guaranteed that we have received a response from a
+	// higher priority server compared to the current active server. So, we
+	// revert to the higher priorty server and close all lower priority ones.
+	a.activeXDSChannel = a.xdsChannelConfigs[serverIdx]
+
+	// Close all lower priority channels.
+	//
+	// But before closing any channel, we need to unsubscribe from any resources
+	// that were subscribed to on this channel. Resources could be subscribed to
+	// from multiple channels as we fallback to lower priority servers. But when
+	// a higher priority one comes back up, we need to unsubscribe from all
+	// lower priority ones before releasing the reference to them.
+	for i := serverIdx + 1; i < len(a.xdsChannelConfigs); i++ {
+		cfg := a.xdsChannelConfigs[i]
+
+		for rType, rState := range a.resources {
+			for resourceName, state := range rState {
+				for xcc := range state.xdsChannelConfigs {
+					if xcc != cfg {
+						continue
+					}
+					// If the current resource is subscribed to on this channel,
+					// unsubscribe, and remove the channel from the list of
+					// channels that this resource is subscribed to.
+					xcc.channel.unsubscribe(rType, resourceName)
+					delete(state.xdsChannelConfigs, xcc)
+				}
+			}
+		}
+
+		// Release the reference to the channel.
+		if cfg.cleanup != nil {
+			if a.logger.V(2) {
+				a.logger.Infof("Closing lower priority server [%d] %q", i, cfg.serverConfig)
+			}
+			cfg.cleanup()
+			cfg.cleanup = nil
+		}
+		cfg.channel = nil
+	}
+	return true
+}
+
+// watchResource registers a new watcher for the specified resource type and
+// name. It returns a function that can be called to cancel the watch.
+//
+// If this is the first watch for any resource on this authority, an xdsChannel
+// to the first management server (from the list of server configurations) will
+// be created.
+//
+// If this is the first watch for the given resource name, it will subscribe to
+// the resource with the xdsChannel. If a cached copy of the resource exists, it
+// will immediately notify the new watcher. When the last watcher for a resource
+// is removed, it will unsubscribe the resource from the xdsChannel.
+func (a *authority) watchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() {
+	cleanup := func() {}
+	done := make(chan struct{})
+
+	a.xdsClientSerializer.ScheduleOr(func(context.Context) {
+		defer close(done)
+
+		if a.logger.V(2) {
+			a.logger.Infof("New watch for type %q, resource name %q", rType.TypeName, resourceName)
+		}
+
+		xdsChannel, err := a.xdsChannelToUse()
+		if err != nil {
+			a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) })
+			return
+		}
+
+		// Lookup the entry for the resource type in the top-level map. If there is
+		// no entry for this resource type, create one.
+		resources := a.resources[rType]
+		if resources == nil {
+			resources = make(map[string]*resourceState)
+			a.resources[rType] = resources
+		}
+
+		// Lookup the resource state for the particular resource name that the watch
+		// is being registered for. If this is the first watch for this resource
+		// name, request it from the management server.
+		state := resources[resourceName]
+		if state == nil {
+			if a.logger.V(2) {
+				a.logger.Infof("First watch for type %q, resource name %q", rType.TypeName, resourceName)
+			}
+			state = &resourceState{
+				watchers:          make(map[ResourceWatcher]bool),
+				md:                xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested},
+				xdsChannelConfigs: map[*xdsChannelWithConfig]bool{xdsChannel: true},
+			}
+			resources[resourceName] = state
+			xdsChannel.channel.subscribe(rType, resourceName)
+		}
+		// Always add the new watcher to the set of watchers.
+		state.watchers[watcher] = true
+
+		// If we have a cached copy of the resource, notify the new watcher
+		// immediately.
+		if state.cache != nil {
+			if a.logger.V(2) {
+				a.logger.Infof("Resource type %q with resource name %q found in cache: %v", rType.TypeName, resourceName, state.cache)
+			}
+			// state can only be accessed in the context of an
+			// xdsClientSerializer callback. Hence making a copy of the cached
+			// resource here for watchCallbackSerializer.
+			resource := state.cache
+			a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceChanged(resource, func() {}) })
+		}
+		// If last update was NACK'd, notify the new watcher of error
+		// immediately as well.
+		if state.md.Status == xdsresource.ServiceStatusNACKed {
+			if a.logger.V(2) {
+				a.logger.Infof("Resource type %q with resource name %q was NACKed", rType.TypeName, resourceName)
+			}
+			// state can only be accessed in the context of an
+			// xdsClientSerializer callback. Hence making a copy of the error
+			// here for watchCallbackSerializer.
+			err := state.md.ErrState.Err
+			if state.cache == nil {
+				a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.ResourceError(err, func() {}) })
+			} else {
+				a.watcherCallbackSerializer.TrySchedule(func(context.Context) { watcher.AmbientError(err, func() {}) })
+			}
+		}
+		// If the metadata field is updated to indicate that the management
+		// server does not have this resource, notify the new watcher.
+		if state.md.Status == xdsresource.ServiceStatusNotExist {
+			a.watcherCallbackSerializer.TrySchedule(func(context.Context) {
+				watcher.ResourceError(xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds: resource %q of type %q has been removed", resourceName, rType.TypeName), func() {})
+			})
+		}
+		cleanup = a.unwatchResource(rType, resourceName, watcher)
+	}, func() {
+		if a.logger.V(2) {
+			a.logger.Infof("Failed to schedule a watch for type %q, resource name %q, because the xDS client is closed", rType.TypeName, resourceName)
+		}
+		close(done)
+	})
+	<-done
+	return cleanup
+}
+
+func (a *authority) unwatchResource(rType ResourceType, resourceName string, watcher ResourceWatcher) func() {
+	return sync.OnceFunc(func() {
+		done := make(chan struct{})
+		a.xdsClientSerializer.ScheduleOr(func(context.Context) {
+			defer close(done)
+
+			if a.logger.V(2) {
+				a.logger.Infof("Canceling a watch for type %q, resource name %q", rType.TypeName, resourceName)
+			}
+
+			// Lookup the resource type from the resource cache. The entry is
+			// guaranteed to be present, since *we* were the ones who added it in
+			// there when the watch was registered.
+			resources := a.resources[rType]
+			state := resources[resourceName]
+
+			// Delete this particular watcher from the list of watchers, so that its
+			// callback will not be invoked in the future.
+			delete(state.watchers, watcher)
+			if len(state.watchers) > 0 {
+				if a.logger.V(2) {
+					a.logger.Infof("Other watchers exist for type %q, resource name %q", rType.TypeName, resourceName)
+				}
+				return
+			}
+
+			// There are no more watchers for this resource. Unsubscribe this
+			// resource from all channels where it was subscribed to and delete
+			// the state associated with it.
+			if a.logger.V(2) {
+				a.logger.Infof("Removing last watch for resource name %q", resourceName)
+			}
+			for xcc := range state.xdsChannelConfigs {
+				xcc.channel.unsubscribe(rType, resourceName)
+			}
+			delete(resources, resourceName)
+
+			// If there are no more watchers for this resource type, delete the
+			// resource type from the top-level map.
+			if len(resources) == 0 {
+				if a.logger.V(2) {
+					a.logger.Infof("Removing last watch for resource type %q", rType.TypeName)
+				}
+				delete(a.resources, rType)
+			}
+			// If there are no more watchers for any resource type, release the
+			// reference to the xdsChannels.
+			if len(a.resources) == 0 {
+				if a.logger.V(2) {
+					a.logger.Infof("Removing last watch for any resource type, releasing reference to the xdsChannel")
+				}
+				a.closeXDSChannels()
+			}
+		}, func() { close(done) })
+		<-done
+	})
+}
+
+// xdsChannelToUse returns the xdsChannel to use for communicating with the
+// management server. If an active channel is available, it returns that.
+// Otherwise, it creates a new channel using the first server configuration in
+// the list of configurations, and returns that.
+//
+// A non-nil error is returned if the channel creation fails.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) xdsChannelToUse() (*xdsChannelWithConfig, error) {
+	if a.activeXDSChannel != nil {
+		return a.activeXDSChannel, nil
+	}
+
+	sc := a.xdsChannelConfigs[0].serverConfig
+	xc, cleanup, err := a.getChannelForADS(sc, a)
+	if err != nil {
+		return nil, err
+	}
+	a.xdsChannelConfigs[0].channel = xc
+	a.xdsChannelConfigs[0].cleanup = cleanup
+	a.activeXDSChannel = a.xdsChannelConfigs[0]
+	return a.activeXDSChannel, nil
+}
+
+// closeXDSChannels closes all the xDS channels associated with this authority,
+// when there are no more watchers for any resource type.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) closeXDSChannels() {
+	for _, xcc := range a.xdsChannelConfigs {
+		if xcc.cleanup != nil {
+			xcc.cleanup()
+			xcc.cleanup = nil
+		}
+		xcc.channel = nil
+	}
+	a.activeXDSChannel = nil
+}
+
+// watcherExistsForUncachedResource returns true if there is at least one
+// watcher for a resource that has not yet been cached.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) watcherExistsForUncachedResource() bool {
+	for _, resourceStates := range a.resources {
+		for _, state := range resourceStates {
+			if state.md.Status == xdsresource.ServiceStatusRequested {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// dumpResources returns a dump of the resource configuration cached by this
+// authority, for CSDS purposes.
+func (a *authority) dumpResources() []*v3statuspb.ClientConfig_GenericXdsConfig {
+	var ret []*v3statuspb.ClientConfig_GenericXdsConfig
+	done := make(chan struct{})
+
+	a.xdsClientSerializer.ScheduleOr(func(context.Context) {
+		defer close(done)
+		ret = a.resourceConfig()
+	}, func() { close(done) })
+	<-done
+	return ret
+}
+
+// resourceConfig returns a slice of GenericXdsConfig objects representing the
+// current state of all resources managed by this authority. This is used for
+// reporting the current state of the xDS client.
+//
+// Only executed in the context of a serializer callback.
+func (a *authority) resourceConfig() []*v3statuspb.ClientConfig_GenericXdsConfig {
+	var ret []*v3statuspb.ClientConfig_GenericXdsConfig
+	for rType, resourceStates := range a.resources {
+		typeURL := rType.TypeURL
+		for name, state := range resourceStates {
+			var raw *anypb.Any
+			if state.cache != nil {
+				raw = &anypb.Any{TypeUrl: typeURL, Value: state.cache.Bytes()}
+			}
+			config := &v3statuspb.ClientConfig_GenericXdsConfig{
+				TypeUrl:      typeURL,
+				Name:         name,
+				VersionInfo:  state.md.Version,
+				XdsConfig:    raw,
+				LastUpdated:  timestamppb.New(state.md.Timestamp),
+				ClientStatus: serviceStatusToProto(state.md.Status),
+			}
+			if errState := state.md.ErrState; errState != nil {
+				config.ErrorState = &v3adminpb.UpdateFailureState{
+					LastUpdateAttempt: timestamppb.New(errState.Timestamp),
+					Details:           errState.Err.Error(),
+					VersionInfo:       errState.Version,
+				}
+			}
+			ret = append(ret, config)
+		}
+	}
+	return ret
+}
+
+func (a *authority) close() {
+	a.xdsClientSerializerClose()
+	<-a.xdsClientSerializer.Done()
+	if a.logger.V(2) {
+		a.logger.Infof("Closed")
+	}
+}
+
+func serviceStatusToProto(serviceStatus xdsresource.ServiceStatus) v3adminpb.ClientResourceStatus {
+	switch serviceStatus {
+	case xdsresource.ServiceStatusUnknown:
+		return v3adminpb.ClientResourceStatus_UNKNOWN
+	case xdsresource.ServiceStatusRequested:
+		return v3adminpb.ClientResourceStatus_REQUESTED
+	case xdsresource.ServiceStatusNotExist:
+		return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST
+	case xdsresource.ServiceStatusACKed:
+		return v3adminpb.ClientResourceStatus_ACKED
+	case xdsresource.ServiceStatusNACKed:
+		return v3adminpb.ClientResourceStatus_NACKED
+	default:
+		return v3adminpb.ClientResourceStatus_UNKNOWN
+	}
+}
+
+func (a *authority) resourceWatchStateForTesting(rType ResourceType, resourceName string) (state xdsresource.ResourceWatchState, err error) {
+	done := make(chan struct{})
+	a.xdsClientSerializer.ScheduleOr(func(context.Context) {
+		state, err = a.activeXDSChannel.channel.ads.adsResourceWatchStateForTesting(rType, resourceName)
+		close(done)
+	}, func() {
+		err = errors.New("failed to retrieve resource watch state because the xDS client is closed")
+		close(done)
+	})
+	<-done
+
+	return state, err
+}
+
+// resourceStats returns a snapshot of the current state of all resources watched
+// by this authority. The return value is a nested map where:
+//   - The outer map's key is the resource type name (e.g., "ListenerResource").
+//   - The inner map's key is the cache state of the resource (e.g., "requested",
+//     "acked", "nacked", "does_not_exist").
+//   - The inner map's value is the total count of resources in that specific state.
+func (a *authority) resourceStats() map[string]map[string]int {
+	ret := make(chan map[string]map[string]int, 1)
+	op := func(context.Context) {
+		summary := make(map[string]map[string]int)
+		for rType, resourceMap := range a.resources {
+			typeName := rType.TypeName
+			if _, ok := summary[typeName]; !ok {
+				summary[typeName] = make(map[string]int)
+			}
+			for _, state := range resourceMap {
+				s := cacheState(state)
+				summary[typeName][s]++
+			}
+		}
+
+		ret <- summary
+	}
+	a.xdsClientSerializer.ScheduleOr(op, func() {
+		ret <- nil
+	})
+
+	return <-ret
+}
+
+// cacheState determines the metrics label string for a given resource state.
+func cacheState(r *resourceState) string {
+	switch r.md.Status {
+	case xdsresource.ServiceStatusRequested:
+		return "requested"
+	case xdsresource.ServiceStatusNotExist:
+		return "does_not_exist"
+	case xdsresource.ServiceStatusACKed:
+		return "acked"
+	case xdsresource.ServiceStatusNACKed:
+		// If the status is NACKed, it means the *latest* update failed.
+		// However, if 'r.cache' is not nil, it means we are still holding onto
+		// a previously ACKed version of the resource.
+		if r.cache != nil {
+			return "nacked_but_cached"
+		}
+		return "nacked"
+	default:
+		// Fallback for initialization states
+		return "requested"
+	}
+}

internal/xds/clients/xdsclient/channel.go

@@ -0,0 +1,329 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/envconfig"
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/internal"
+	"google.golang.org/grpc/internal/xds/clients/internal/backoff"
+	"google.golang.org/grpc/internal/xds/clients/internal/syncutil"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+)
+
+const (
+	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
+	clientFeatureResourceWrapper    = "xds.config.resource-in-sotw"
+)
+
+// xdsChannelEventHandler wraps callbacks used to notify the xDS client about
+// events on the xdsChannel. Methods in this interface may be invoked
+// concurrently and the xDS client implementation needs to handle them in a
+// thread-safe manner.
+type xdsChannelEventHandler interface {
+	// adsStreamFailure is called when the xdsChannel encounters an ADS stream
+	// failure.
+	adsStreamFailure(error)
+
+	// adsResourceUpdate is called when the xdsChannel receives an ADS response
+	// from the xDS management server. The callback is provided with the
+	// following:
+	//   - the resource type of the resources in the response
+	//   - a map of resources in the response, keyed by resource name
+	//   - the metadata associated with the response
+	//   - a callback to be invoked when the updated is processed
+	adsResourceUpdate(ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, func())
+
+	// adsResourceDoesNotExist is called when the xdsChannel determines that a
+	// requested ADS resource does not exist.
+	adsResourceDoesNotExist(ResourceType, string)
+}
+
+// xdsChannelOpts holds the options for creating a new xdsChannel.
+type xdsChannelOpts struct {
+	transport          clients.Transport       // Takes ownership of this transport.
+	serverConfig       *ServerConfig           // Configuration of the server to connect to.
+	clientConfig       *Config                 // Complete xDS client configuration, used to decode resources.
+	eventHandler       xdsChannelEventHandler  // Callbacks for ADS stream events.
+	backoff            func(int) time.Duration // Backoff function to use for stream retries. Defaults to exponential backoff, if unset.
+	watchExpiryTimeout time.Duration           // Timeout for ADS resource watch expiry.
+	logPrefix          string                  // Prefix to use for logging.
+}
+
+// newXDSChannel creates a new xdsChannel instance with the provided options.
+// It performs basic validation on the provided options and initializes the
+// xdsChannel with the necessary components.
+func newXDSChannel(opts xdsChannelOpts) (*xdsChannel, error) {
+	switch {
+	case opts.transport == nil:
+		return nil, errors.New("xdsclient: transport is nil")
+	case opts.serverConfig == nil:
+		return nil, errors.New("xdsclient: serverConfig is nil")
+	case opts.clientConfig == nil:
+		return nil, errors.New("xdsclient: clientConfig is nil")
+	case opts.eventHandler == nil:
+		return nil, errors.New("xdsclient: eventHandler is nil")
+	}
+
+	xc := &xdsChannel{
+		transport:    opts.transport,
+		serverConfig: opts.serverConfig,
+		clientConfig: opts.clientConfig,
+		eventHandler: opts.eventHandler,
+		closed:       syncutil.NewEvent(),
+	}
+
+	l := grpclog.Component("xds")
+	logPrefix := opts.logPrefix + fmt.Sprintf("[xds-channel %p] ", xc)
+	xc.logger = igrpclog.NewPrefixLogger(l, logPrefix)
+
+	if opts.backoff == nil {
+		opts.backoff = backoff.DefaultExponential.Backoff
+	}
+	np := internal.NodeProto(opts.clientConfig.Node)
+	np.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}
+	xc.ads = newADSStreamImpl(adsStreamOpts{
+		transport:          opts.transport,
+		eventHandler:       xc,
+		backoff:            opts.backoff,
+		nodeProto:          np,
+		watchExpiryTimeout: opts.watchExpiryTimeout,
+		logPrefix:          logPrefix,
+	})
+	if xc.logger.V(2) {
+		xc.logger.Infof("xdsChannel is created for ServerConfig %v", opts.serverConfig)
+	}
+	return xc, nil
+}
+
+// xdsChannel represents a client channel to a management server, and is
+// responsible for managing the lifecycle of the ADS and LRS streams. It invokes
+// callbacks on the registered event handler for various ADS stream events.
+//
+// It is safe for concurrent use.
+type xdsChannel struct {
+	// The following fields are initialized at creation time and are read-only
+	// after that, and hence need not be guarded by a mutex.
+	transport    clients.Transport      // Takes ownership of this transport (used to make streaming calls).
+	ads          *adsStreamImpl         // An ADS stream to the management server.
+	serverConfig *ServerConfig          // Configuration of the server to connect to.
+	clientConfig *Config                // Complete xDS client configuration, used to decode resources.
+	eventHandler xdsChannelEventHandler // Callbacks for ADS stream events.
+	logger       *igrpclog.PrefixLogger // Logger to use for logging.
+	closed       *syncutil.Event        // Fired when the channel is closed.
+}
+
+func (xc *xdsChannel) close() {
+	xc.closed.Fire()
+	xc.ads.Stop()
+	xc.transport.Close()
+	xc.logger.Infof("Shutdown")
+}
+
+// subscribe adds a subscription for the given resource name of the given
+// resource type on the ADS stream.
+func (xc *xdsChannel) subscribe(typ ResourceType, name string) {
+	if xc.closed.HasFired() {
+		if xc.logger.V(2) {
+			xc.logger.Infof("Attempt to subscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name)
+		}
+		return
+	}
+	xc.ads.subscribe(typ, name)
+}
+
+// unsubscribe removes the subscription for the given resource name of the given
+// resource type from the ADS stream.
+func (xc *xdsChannel) unsubscribe(typ ResourceType, name string) {
+	if xc.closed.HasFired() {
+		if xc.logger.V(2) {
+			xc.logger.Infof("Attempt to unsubscribe to an xDS resource of type %s and name %q on a closed channel", typ.TypeName, name)
+		}
+		return
+	}
+	xc.ads.unsubscribe(typ, name)
+}
+
+// The following onADSXxx() methods implement the StreamEventHandler interface
+// and are invoked by the ADS stream implementation.
+
+// onStreamError is invoked when an error occurs on the ADS stream. It
+// propagates the update to the xDS client.
+func (xc *xdsChannel) onStreamError(err error) {
+	if xc.closed.HasFired() {
+		if xc.logger.V(2) {
+			xc.logger.Infof("Received ADS stream error on a closed xdsChannel: %v", err)
+		}
+		return
+	}
+	xc.eventHandler.adsStreamFailure(err)
+}
+
+// onWatchExpiry is invoked when a watch for a resource expires. It
+// propagates the update to the xDS client.
+func (xc *xdsChannel) onWatchExpiry(typ ResourceType, name string) {
+	if xc.closed.HasFired() {
+		if xc.logger.V(2) {
+			xc.logger.Infof("Received ADS resource watch expiry for resource %q on a closed xdsChannel", name)
+		}
+		return
+	}
+	xc.eventHandler.adsResourceDoesNotExist(typ, name)
+}
+
+// onResponse is invoked when a response is received on the ADS stream. It
+// decodes the resources in the response, and propagates the updates to the xDS
+// client.
+//
+// It returns the list of resource names in the response and any errors
+// encountered during decoding.
+func (xc *xdsChannel) onResponse(resp response, onDone func()) ([]string, error) {
+	if xc.closed.HasFired() {
+		if xc.logger.V(2) {
+			xc.logger.Infof("Received an update from the ADS stream on closed ADS stream")
+		}
+		return nil, errors.New("xdsChannel is closed")
+	}
+
+	// Lookup the resource parser based on the resource type.
+	rType, ok := xc.clientConfig.ResourceTypes[resp.typeURL]
+	if !ok {
+		return nil, xdsresource.NewErrorf(xdsresource.ErrorTypeResourceTypeUnsupported, "Resource type URL %q unknown in response from server", resp.typeURL)
+	}
+
+	updates, md, err := xc.decodeResponse(&rType, resp)
+	var names []string
+	for name := range updates {
+		names = append(names, name)
+	}
+
+	xc.eventHandler.adsResourceUpdate(rType, updates, md, onDone)
+	return names, err
+}
+
+// decodeResponse decodes the resources in the given ADS response.
+//
+// The opts parameter provides configuration options for decoding the resources.
+// The rType parameter specifies the resource type parser to use for decoding
+// the resources.
+//
+// The returned map contains a key for each resource in the response, with the
+// value being either the decoded resource data or an error if decoding failed.
+// The returned metadata includes the version of the response, the timestamp of
+// the update, and the status of the update (ACKed or NACKed).
+//
+// If there are any errors decoding the resources, the metadata will indicate
+// that the update was NACKed, and the returned error will contain information
+// about all errors encountered by this function.
+func (xc *xdsChannel) decodeResponse(rType *ResourceType, resp response) (map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) {
+	timestamp := time.Now()
+	md := xdsresource.UpdateMetadata{
+		Version:   resp.version,
+		Timestamp: timestamp,
+	}
+	opts := &DecodeOptions{
+		Config:       xc.clientConfig,
+		ServerConfig: xc.serverConfig,
+	}
+
+	topLevelErrors := make([]error, 0)          // Tracks deserialization errors, where we don't have a resource name.
+	perResourceErrors := make(map[string]error) // Tracks resource validation errors, where we have a resource name.
+	ret := make(map[string]dataAndErrTuple)     // Return result, a map from resource name to either resource data or error.
+	for _, r := range resp.resources {
+		result, err := func() (res *DecodeResult, err error) {
+			defer func() {
+				if envconfig.XDSRecoverPanicInResourceParsing {
+					if p := recover(); p != nil {
+						err = fmt.Errorf("recovered from panic during resource parsing, resource: %v, panic: %v", r, p)
+					}
+				}
+			}()
+			return rType.Decoder.Decode(NewAnyProto(r), *opts)
+		}()
+
+		// Name field of the result is left unpopulated only when resource
+		// deserialization fails.
+		name := ""
+		if result == nil && err == nil {
+			xc.logger.Errorf("Decode() returned nil result and nil error for resource: %v", r)
+			continue
+		}
+		if result != nil {
+			name = xdsresource.ParseName(result.Name).String()
+		}
+		if err == nil {
+			ret[name] = dataAndErrTuple{Resource: result.Resource}
+			continue
+		}
+		if name == "" {
+			topLevelErrors = append(topLevelErrors, err)
+			continue
+		}
+		perResourceErrors[name] = err
+		// Add place holder in the map so we know this resource name was in
+		// the response.
+		ret[name] = dataAndErrTuple{Err: xdsresource.NewError(xdsresource.ErrorTypeNACKed, err.Error())}
+	}
+
+	if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 {
+		md.Status = xdsresource.ServiceStatusACKed
+		return ret, md, nil
+	}
+
+	md.Status = xdsresource.ServiceStatusNACKed
+	errRet := combineErrors(rType.TypeName, topLevelErrors, perResourceErrors)
+	md.ErrState = &xdsresource.UpdateErrorMetadata{
+		Version:   resp.version,
+		Err:       xdsresource.NewError(xdsresource.ErrorTypeNACKed, errRet.Error()),
+		Timestamp: timestamp,
+	}
+	return ret, md, errRet
+}
+
+func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error {
+	var errStrB strings.Builder
+	errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType))
+	if len(topLevelErrors) > 0 {
+		errStrB.WriteString("top level errors: ")
+		for i, err := range topLevelErrors {
+			if i != 0 {
+				errStrB.WriteString(";\n")
+			}
+			errStrB.WriteString(err.Error())
+		}
+	}
+	if len(perResourceErrors) > 0 {
+		var i int
+		for name, err := range perResourceErrors {
+			if i != 0 {
+				errStrB.WriteString(";\n")
+			}
+			i++
+			errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error()))
+		}
+	}
+	return errors.New(errStrB.String())
+}

internal/xds/clients/xdsclient/channel_test.go

@@ -0,0 +1,821 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	xdstestutils "google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// xdsChannelForTest creates an xdsChannel to the specified serverURI for
+// testing purposes.
+func xdsChannelForTest(t *testing.T, serverURI, nodeID string, watchExpiryTimeout time.Duration) *xdsChannel {
+	t.Helper()
+
+	// Create a grpc transport to the above management server.
+	si := clients.ServerIdentifier{
+		ServerURI:  serverURI,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	tr, err := (grpctransport.NewBuilder(configs)).Build(si)
+	if err != nil {
+		t.Fatalf("Failed to create a transport for server config %v: %v", si, err)
+	}
+
+	serverCfg := ServerConfig{
+		ServerIdentifier: si,
+	}
+	clientConfig := Config{
+		Servers:       []ServerConfig{serverCfg},
+		Node:          clients.Node{ID: nodeID},
+		ResourceTypes: map[string]ResourceType{xdsresource.V3ListenerURL: listenerType},
+	}
+	// Create an xdsChannel that uses everything set up above.
+	xc, err := newXDSChannel(xdsChannelOpts{
+		transport:          tr,
+		serverConfig:       &serverCfg,
+		clientConfig:       &clientConfig,
+		eventHandler:       newTestEventHandler(),
+		watchExpiryTimeout: watchExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xdsChannel: %v", err)
+	}
+	t.Cleanup(func() { xc.close() })
+	return xc
+}
+
+// verifyUpdateAndMetadata verifies that the event handler received the expected
+// updates and metadata.  It checks that the received resource type matches the
+// expected type, and that the received updates and metadata match the expected
+// values. The function ignores the timestamp fields in the metadata, as those
+// are expected to be different.
+func verifyUpdateAndMetadata(ctx context.Context, t *testing.T, eh *testEventHandler, wantUpdates map[string]dataAndErrTuple, wantMD xdsresource.UpdateMetadata) {
+	t.Helper()
+
+	gotTyp, gotUpdates, gotMD, err := eh.waitForUpdate(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for update callback to be invoked on the event handler")
+	}
+
+	if gotTyp != listenerType {
+		t.Fatalf("Got resource type %v, want %v", gotTyp, listenerType)
+	}
+	opts := cmp.Options{
+		protocmp.Transform(),
+		cmpopts.EquateEmpty(),
+		cmpopts.EquateErrors(),
+		cmpopts.IgnoreFields(xdsresource.UpdateMetadata{}, "Timestamp"),
+		cmpopts.IgnoreFields(xdsresource.UpdateErrorMetadata{}, "Timestamp"),
+	}
+	if diff := cmp.Diff(wantUpdates, gotUpdates, opts); diff != "" {
+		t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %+v\n got: %+v", diff, wantUpdates, gotUpdates)
+	}
+	if diff := cmp.Diff(wantMD, gotMD, opts); diff != "" {
+		t.Fatalf("Got unexpected diff in update (-want +got):\n%s\n want: %v\n got: %v", diff, wantMD, gotMD)
+	}
+}
+
+// Tests different failure cases when creating a new xdsChannel. It checks that
+// the xdsChannel creation fails when any of the required options (transport,
+// serverConfig, bootstrapConfig, or resourceTypeGetter) are missing or nil.
+func (s) TestChannel_New_FailureCases(t *testing.T) {
+	type fakeTransport struct {
+		clients.Transport
+	}
+
+	tests := []struct {
+		name       string
+		opts       xdsChannelOpts
+		wantErrStr string
+	}{
+		{
+			name:       "emptyTransport",
+			opts:       xdsChannelOpts{},
+			wantErrStr: "transport is nil",
+		},
+		{
+			name:       "emptyServerConfig",
+			opts:       xdsChannelOpts{transport: &fakeTransport{}},
+			wantErrStr: "serverConfig is nil",
+		},
+		{
+			name: "emptyCConfig",
+			opts: xdsChannelOpts{
+				transport:    &fakeTransport{},
+				serverConfig: &ServerConfig{},
+			},
+			wantErrStr: "clientConfig is nil",
+		},
+		{
+			name: "emptyEventHandler",
+			opts: xdsChannelOpts{
+				transport:    &fakeTransport{},
+				serverConfig: &ServerConfig{},
+				clientConfig: &Config{},
+			},
+			wantErrStr: "eventHandler is nil",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if _, err := newXDSChannel(test.opts); err == nil || !strings.Contains(err.Error(), test.wantErrStr) {
+				t.Fatalf("newXDSChannel() = %v, want %q", err, test.wantErrStr)
+			}
+		})
+	}
+}
+
+// Tests different scenarios of the xdsChannel receiving a response from the
+// management server. In all scenarios, the xdsChannel is expected to pass the
+// received responses as-is to the resource parsing functionality specified by
+// the resourceTypeGetter.
+func (s) TestChannel_ADS_HandleResponseFromManagementServer(t *testing.T) {
+	const (
+		listenerName1 = "listener-name-1"
+		listenerName2 = "listener-name-2"
+		routeName     = "route-name"
+		clusterName   = "cluster-name"
+	)
+	var (
+		badlyMarshaledResource = &anypb.Any{
+			TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener",
+			Value:   []byte{1, 2, 3, 4},
+		}
+		apiListener = &v3listenerpb.ApiListener{
+			ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name: routeName},
+				},
+			}),
+		}
+		listener1 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
+			Name:        listenerName1,
+			ApiListener: apiListener,
+		})
+		listener2 = xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
+			Name:        listenerName2,
+			ApiListener: apiListener,
+		})
+	)
+
+	tests := []struct {
+		desc                     string
+		resourceNamesToRequest   []string
+		managementServerResponse *v3discoverypb.DiscoveryResponse
+		wantUpdates              map[string]dataAndErrTuple
+		wantMD                   xdsresource.UpdateMetadata
+		wantErr                  error
+	}{
+		{
+			desc:                   "one bad resource - deserialization failure",
+			resourceNamesToRequest: []string{listenerName1},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources:   []*anypb.Any{badlyMarshaledResource},
+			},
+			wantUpdates: nil, // No updates expected as the response runs into unmarshaling errors.
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusNACKed,
+				Version: "0",
+				ErrState: &xdsresource.UpdateErrorMetadata{
+					Version: "0",
+					Err:     cmpopts.AnyError,
+				},
+			},
+			wantErr: cmpopts.AnyError,
+		},
+		{
+			desc:                   "one bad resource - validation failure",
+			resourceNamesToRequest: []string{listenerName1},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources: []*anypb.Any{xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
+					Name: listenerName1,
+					ApiListener: &v3listenerpb.ApiListener{
+						ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+							RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+						}),
+					},
+				})},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName1: {
+					Err: cmpopts.AnyError,
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusNACKed,
+				Version: "0",
+				ErrState: &xdsresource.UpdateErrorMetadata{
+					Version: "0",
+					Err:     cmpopts.AnyError,
+				},
+			},
+		},
+		{
+			desc:                   "two bad resources",
+			resourceNamesToRequest: []string{listenerName1, listenerName2},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources: []*anypb.Any{
+					badlyMarshaledResource,
+					xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
+						Name: listenerName2,
+						ApiListener: &v3listenerpb.ApiListener{
+							ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+							}),
+						},
+					}),
+				},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName2: {
+					Err: cmpopts.AnyError,
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusNACKed,
+				Version: "0",
+				ErrState: &xdsresource.UpdateErrorMetadata{
+					Version: "0",
+					Err:     cmpopts.AnyError,
+				},
+			},
+		},
+		{
+			desc:                   "one good resource",
+			resourceNamesToRequest: []string{listenerName1},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources:   []*anypb.Any{listener1},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName1: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener1.GetValue(),
+					}},
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusACKed,
+				Version: "0",
+			},
+		},
+		{
+			desc:                   "one good and one bad - deserialization failure",
+			resourceNamesToRequest: []string{listenerName1, listenerName2},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources: []*anypb.Any{
+					badlyMarshaledResource,
+					listener2,
+				},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName2: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener2.GetValue(),
+					}},
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusNACKed,
+				Version: "0",
+				ErrState: &xdsresource.UpdateErrorMetadata{
+					Version: "0",
+					Err:     cmpopts.AnyError,
+				},
+			},
+		},
+		{
+			desc:                   "one good and one bad - validation failure",
+			resourceNamesToRequest: []string{listenerName1, listenerName2},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources: []*anypb.Any{
+					xdstestutils.MarshalAny(t, &v3listenerpb.Listener{
+						Name: listenerName1,
+						ApiListener: &v3listenerpb.ApiListener{
+							ApiListener: xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+								RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{},
+							}),
+						},
+					}),
+					listener2,
+				},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName1: {Err: cmpopts.AnyError},
+				listenerName2: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener2.GetValue(),
+					}},
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusNACKed,
+				Version: "0",
+				ErrState: &xdsresource.UpdateErrorMetadata{
+					Version: "0",
+					Err:     cmpopts.AnyError,
+				},
+			},
+		},
+		{
+			desc:                   "two good resources",
+			resourceNamesToRequest: []string{listenerName1, listenerName2},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources:   []*anypb.Any{listener1, listener2},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName1: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener1.GetValue(),
+					}},
+				},
+				listenerName2: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener2.GetValue(),
+					}},
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusACKed,
+				Version: "0",
+			},
+		},
+		{
+			desc:                   "two resources when we requested one",
+			resourceNamesToRequest: []string{listenerName1},
+			managementServerResponse: &v3discoverypb.DiscoveryResponse{
+				VersionInfo: "0",
+				TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+				Resources:   []*anypb.Any{listener1, listener2},
+			},
+			wantUpdates: map[string]dataAndErrTuple{
+				listenerName1: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener1.GetValue(),
+					}},
+				},
+				listenerName2: {
+					Resource: &listenerResourceData{Resource: listenerUpdate{
+						RouteConfigName: routeName,
+						Raw:             listener2.GetValue(),
+					}},
+				},
+			},
+			wantMD: xdsresource.UpdateMetadata{
+				Status:  xdsresource.ServiceStatusACKed,
+				Version: "0",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			// Start a fake xDS management server and configure the response it
+			// would send to its client.
+			mgmtServer, cleanup, err := fakeserver.StartServer(nil)
+			if err != nil {
+				t.Fatalf("Failed to start fake xDS server: %v", err)
+			}
+			defer cleanup()
+			t.Logf("Started xDS management server on %s", mgmtServer.Address)
+			mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse}
+
+			// Create an xdsChannel for the test with a long watch expiry timer
+			// to ensure that watches don't expire for the duration of the test.
+			nodeID := uuid.New().String()
+			xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout)
+			defer xc.close()
+
+			// Subscribe to the resources specified in the test table.
+			for _, name := range test.resourceNamesToRequest {
+				xc.subscribe(listenerType, name)
+			}
+
+			// Wait for an update callback on the event handler and verify the
+			// contents of the update and the metadata.
+			verifyUpdateAndMetadata(ctx, t, xc.eventHandler.(*testEventHandler), test.wantUpdates, test.wantMD)
+		})
+	}
+}
+
+// Tests that the xdsChannel correctly handles the expiry of a watch for a
+// resource by ensuring that the watch expiry callback is invoked on the event
+// handler with the expected resource type and name.
+func (s) TestChannel_ADS_HandleResponseWatchExpiry(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server, but do not configure any resources on it.
+	// This will result in the watch for a resource to timeout.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create an xdsChannel for the test with a short watch expiry timer to
+	// ensure that the test does not run very long, as it needs to wait for the
+	// watch to expire.
+	nodeID := uuid.New().String()
+	xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestShortTimeout)
+	defer xc.close()
+
+	// Subscribe to a listener resource.
+	const listenerName = "listener-name"
+	xc.subscribe(listenerType, listenerName)
+
+	// Wait for the watch expiry callback on the authority to be invoked and
+	// verify that the watch expired for the expected resource name and type.
+	eventHandler := xc.eventHandler.(*testEventHandler)
+	gotTyp, gotName, err := eventHandler.waitForResourceDoesNotExist(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the watch expiry callback to be invoked on the xDS client")
+	}
+
+	if gotTyp != listenerType {
+		t.Fatalf("Got type %v, want %v", gotTyp, listenerType)
+	}
+	if gotName != listenerName {
+		t.Fatalf("Got name %v, want %v", gotName, listenerName)
+	}
+}
+
+// Tests that the xdsChannel correctly handles stream failures by ensuring that
+// the stream failure callback is invoked on the event handler.
+func (s) TestChannel_ADS_StreamFailure(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 20000*defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server with a restartable listener to simulate
+	// connection failures.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := xdstestutils.NewRestartableListener(l)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
+
+	// Configure a listener resource on the management server.
+	const listenerResourceName = "test-listener-resource"
+	const routeConfigurationName = "test-route-configuration-resource"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Create an xdsChannel for the test with a long watch expiry timer
+	// to ensure that watches don't expire for the duration of the test.
+	xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2000*defaultTestTimeout)
+	defer xc.close()
+
+	// Subscribe to the resource created above.
+	xc.subscribe(listenerType, listenerResourceName)
+
+	// Wait for an update callback on the event handler and verify the
+	// contents of the update and the metadata.
+	hcm := xdstestutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{
+			ConfigSource: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+			},
+			RouteConfigName: routeConfigurationName,
+		}},
+	})
+	listenerResource, err := anypb.New(&v3listenerpb.Listener{
+		Name:        listenerResourceName,
+		ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create listener resource: %v", err)
+	}
+
+	wantUpdates := map[string]dataAndErrTuple{
+		listenerResourceName: {
+			Resource: &listenerResourceData{
+				Resource: listenerUpdate{
+					RouteConfigName: routeConfigurationName,
+					Raw:             listenerResource.GetValue(),
+				},
+			},
+		},
+	}
+	wantMD := xdsresource.UpdateMetadata{
+		Status:  xdsresource.ServiceStatusACKed,
+		Version: "1",
+	}
+
+	eventHandler := xc.eventHandler.(*testEventHandler)
+	verifyUpdateAndMetadata(ctx, t, eventHandler, wantUpdates, wantMD)
+
+	lis.Stop()
+	if err := eventHandler.waitForStreamFailure(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for the stream failure callback to be invoked on the xDS client: %v", err)
+	}
+}
+
+// Tests the behavior of the xdsChannel when a resource is unsubscribed.
+// Verifies that when a previously subscribed resource is unsubscribed, a
+// request is sent without the previously subscribed resource name.
+func (s) TestChannel_ADS_ResourceUnsubscribe(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server that uses a channel to inform the test
+	// about the specific LDS resource names being requested.
+	ldsResourcesCh := make(chan []string, 1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
+
+			if req.TypeUrl != xdsresource.V3ListenerURL {
+				return fmt.Errorf("unexpected resource type URL: %q", req.TypeUrl)
+			}
+
+			// Make the most recently requested names available to the test.
+			ldsResourcesCh <- req.GetResourceNames()
+			return nil
+		},
+	})
+
+	// Configure two listener resources on the management server.
+	const listenerResourceName1 = "test-listener-resource-1"
+	const routeConfigurationName1 = "test-route-configuration-resource-1"
+	const listenerResourceName2 = "test-listener-resource-2"
+	const routeConfigurationName2 = "test-route-configuration-resource-2"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1),
+			e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2),
+		},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Create an xdsChannel for the test with a long watch expiry timer
+	// to ensure that watches don't expire for the duration of the test.
+	xc := xdsChannelForTest(t, mgmtServer.Address, nodeID, 2*defaultTestTimeout)
+	defer xc.close()
+
+	// Subscribe to the resources created above and verify that a request is
+	// sent for the same.
+	xc.subscribe(listenerType, listenerResourceName1)
+	xc.subscribe(listenerType, listenerResourceName2)
+	if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the above resources to be ACKed.
+	if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName1, listenerResourceName2}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Unsubscribe to one of the resources created above, and ensure that the
+	// other resource is still being requested.
+	xc.unsubscribe(listenerType, listenerResourceName1)
+	if err := waitForResourceNames(ctx, ldsResourcesCh, []string{listenerResourceName2}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Since the version on the management server for the above resource is not
+	// changed, we will not receive an update from it for the one resource that
+	// we are still requesting.
+
+	// Unsubscribe to the remaining resource, and ensure that no more resources
+	// are being requested.
+	xc.unsubscribe(listenerType, listenerResourceName2)
+	if err := waitForResourceNames(ctx, ldsResourcesCh, []string{}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// waitForResourceNames waits for the wantNames to be received on namesCh.
+// Returns a non-nil error if the context expires before that.
+func waitForResourceNames(ctx context.Context, namesCh chan []string, wantNames []string) error {
+	var lastRequestedNames []string
+	for ; ; <-time.After(defaultTestShortTimeout) {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames)
+		case gotNames := <-namesCh:
+			if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) {
+				return nil
+			}
+			lastRequestedNames = gotNames
+		}
+	}
+}
+
+// newTestEventHandler creates a new testEventHandler instance with the
+// necessary channels for testing the xdsChannel.
+func newTestEventHandler() *testEventHandler {
+	return &testEventHandler{
+		typeCh:    make(chan ResourceType, 1),
+		updateCh:  make(chan map[string]dataAndErrTuple, 1),
+		mdCh:      make(chan xdsresource.UpdateMetadata, 1),
+		nameCh:    make(chan string, 1),
+		connErrCh: make(chan error, 1),
+	}
+}
+
+// testEventHandler is a struct that implements the xdsChannelEventhandler
+// interface.  It is used to receive events from an xdsChannel, and has multiple
+// channels on which it makes these events available to the test.
+type testEventHandler struct {
+	typeCh    chan ResourceType               // Resource type of an update or resource-does-not-exist error.
+	updateCh  chan map[string]dataAndErrTuple // Resource updates.
+	mdCh      chan xdsresource.UpdateMetadata // Metadata from an update.
+	nameCh    chan string                     // Name of the non-existent resource.
+	connErrCh chan error                      // Connectivity error.
+
+}
+
+func (ta *testEventHandler) adsStreamFailure(err error) {
+	ta.connErrCh <- err
+}
+
+func (ta *testEventHandler) waitForStreamFailure(ctx context.Context) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-ta.connErrCh:
+	}
+	return nil
+}
+
+func (ta *testEventHandler) adsResourceUpdate(typ ResourceType, updates map[string]dataAndErrTuple, md xdsresource.UpdateMetadata, onDone func()) {
+	ta.typeCh <- typ
+	ta.updateCh <- updates
+	ta.mdCh <- md
+	onDone()
+}
+
+// waitForUpdate waits for the next resource update event from the xdsChannel.
+// It returns the resource type, the resource updates, and the update metadata.
+// If the context is canceled, it returns an error.
+func (ta *testEventHandler) waitForUpdate(ctx context.Context) (ResourceType, map[string]dataAndErrTuple, xdsresource.UpdateMetadata, error) {
+	var typ ResourceType
+	var updates map[string]dataAndErrTuple
+	var md xdsresource.UpdateMetadata
+
+	select {
+	case typ = <-ta.typeCh:
+	case <-ctx.Done():
+		return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
+	}
+
+	select {
+	case updates = <-ta.updateCh:
+	case <-ctx.Done():
+		return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
+	}
+
+	select {
+	case md = <-ta.mdCh:
+	case <-ctx.Done():
+		return ResourceType{}, nil, xdsresource.UpdateMetadata{}, ctx.Err()
+	}
+	return typ, updates, md, nil
+}
+
+func (ta *testEventHandler) adsResourceDoesNotExist(typ ResourceType, name string) {
+	ta.typeCh <- typ
+	ta.nameCh <- name
+}
+
+// waitForResourceDoesNotExist waits for the next resource-does-not-exist event
+// from the xdsChannel. It returns the resource type and the resource name. If
+// the context is canceled, it returns an error.
+func (ta *testEventHandler) waitForResourceDoesNotExist(ctx context.Context) (ResourceType, string, error) {
+	var typ ResourceType
+	var name string
+
+	select {
+	case typ = <-ta.typeCh:
+	case <-ctx.Done():
+		return ResourceType{}, "", ctx.Err()
+	}
+
+	select {
+	case name = <-ta.nameCh:
+	case <-ctx.Done():
+		return ResourceType{}, "", ctx.Err()
+	}
+	return typ, name, nil
+}
+
+type panicDecoder struct{}
+
+func (panicDecoder) Decode(*AnyProto, DecodeOptions) (*DecodeResult, error) {
+	panic("simulate panic")
+}
+
+// TestDecodeResponse_PanicRecoveryEnabled tests the panic recovery mechanism
+// in decodeResponse. It verifies that if XDSRecoverPanicInResourceParsing
+// env variable is enabled, panics during unmarshaling are caught and returned
+// as errors.
+func (s) TestDecodeResponse_PanicRecoveryEnabled(t *testing.T) {
+	rType := &ResourceType{
+		TypeName: "resourceType",
+		Decoder:  panicDecoder{},
+	}
+	resp := response{resources: []*anypb.Any{{Value: []byte("test")}}}
+	wantErr := "recovered from panic during resource parsing"
+
+	xc := &xdsChannel{}
+	if _, _, err := xc.decodeResponse(rType, resp); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("decodeResponse() failed with err: %v, want %q", err, wantErr)
+	}
+}
+
+// TestDecodeResponse_PanicRecoveryDisabled tests the panic recovery mechanism
+// in decodeResponse. It verifies that when XDSRecoverPanicInResourceParsing
+// env variable is disabled, panics during unmarshaling propagate.
+func (s) TestDecodeResponse_PanicRecoveryDisabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSRecoverPanicInResourceParsing, false)
+	rType := &ResourceType{
+		TypeName: "resourceType",
+		Decoder:  panicDecoder{},
+	}
+	resp := response{resources: []*anypb.Any{{Value: []byte("test")}}}
+	wantErr := "simulate panic"
+	xc := &xdsChannel{}
+
+	defer func() {
+		if r := recover(); r == nil || !strings.Contains(fmt.Sprint(r), wantErr) {
+			t.Fatalf("Expected panic in decodeResponse, got: %v, want: %q", r, wantErr)
+		}
+	}()
+	xc.decodeResponse(rType, resp)
+}

internal/xds/clients/xdsclient/clientimpl_watchers.go

@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient
+
+import (
+	"context"
+	"fmt"
+
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+)
+
+// wrappingWatcher is a wrapper around an xdsresource.ResourceWatcher that adds
+// the node ID to the error messages reported to the watcher.
+type wrappingWatcher struct {
+	ResourceWatcher
+	nodeID string
+}
+
+func (w *wrappingWatcher) AmbientError(err error, done func()) {
+	w.ResourceWatcher.AmbientError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done)
+}
+
+func (w *wrappingWatcher) ResourceError(err error, done func()) {
+	w.ResourceWatcher.ResourceError(fmt.Errorf("[xDS node id: %v]: %w", w.nodeID, err), done)
+}
+
+// WatchResource starts watching the specified resource.
+//
+// The watch fails to start if:
+//   - There is no ResourceType implementation for the given typeURL in the
+//     ResourceTypes field of the Config struct used to create the XDSClient.
+//   - The provided resourceName contains an authority that is not present in the
+//     Authorities field.
+//
+// The returned function cancels the watch and prevents future calls to the
+// watcher.
+func (c *XDSClient) WatchResource(typeURL, resourceName string, watcher ResourceWatcher) (cancel func()) {
+	// Return early if the client is already closed.
+	if c.done.HasFired() {
+		logger.Warningf("Watch registered for type %q, but client is closed", typeURL)
+		return func() {}
+	}
+
+	watcher = &wrappingWatcher{
+		ResourceWatcher: watcher,
+		nodeID:          c.config.Node.ID,
+	}
+
+	rType, ok := c.config.ResourceTypes[typeURL]
+	if !ok {
+		logger.Warningf("ResourceType implementation for resource type url %q is not found", rType.TypeURL)
+		c.serializer.TrySchedule(func(context.Context) {
+			watcher.ResourceError(fmt.Errorf("no ResourceType implementation found for typeURL %q", rType.TypeURL), func() {})
+		})
+		return func() {}
+	}
+
+	n := xdsresource.ParseName(resourceName)
+	a := c.getAuthorityForResource(n)
+	if a == nil {
+		logger.Warningf("Watch registered for name %q of type %q, authority %q is not found", rType.TypeName, resourceName, n.Authority)
+		c.serializer.TrySchedule(func(context.Context) {
+			watcher.ResourceError(fmt.Errorf("authority %q not found in the config for resource %q", n.Authority, resourceName), func() {})
+		})
+		return func() {}
+	}
+	// The watchResource method on the authority is invoked with n.String()
+	// instead of resourceName because n.String() canonicalizes the given name.
+	// So, two resource names which don't differ in the query string, but only
+	// differ in the order of context params will result in the same resource
+	// being watched by the authority.
+	return a.watchResource(rType, n.String(), watcher)
+}
+
+// Gets the authority for the given resource name.
+//
+// See examples in this section of the gRFC:
+// https://github.com/grpc/proposal/blob/master/A47-xds-federation.md#bootstrap-config-changes
+func (c *XDSClient) getAuthorityForResource(name *xdsresource.Name) *authority {
+	// For new-style resource names, always lookup the authorities map. If the
+	// name does not specify an authority, we will end up looking for an entry
+	// in the map with the empty string as the key.
+	if name.Scheme == xdsresource.FederationScheme {
+		return c.authorities[name.Authority]
+	}
+
+	// For old-style resource names, we use the top-level authority if the name
+	// does not specify an authority.
+	if name.Authority == "" {
+		return c.topLevelAuthority
+	}
+	return c.authorities[name.Authority]
+}