snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/balancer/clusterimpl/balancer_test.go

@@ -0,0 +1,356 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpctest"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/testutils"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/testutils/fakeclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+)
+
+const (
+	defaultTestTimeout      = 5 * time.Second
+	defaultShortTestTimeout = 100 * time.Microsecond
+
+	testClusterName = "test-cluster"
+	testServiceName = "test-eds-service"
+)
+
+var (
+	testBackendEndpoints = []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: "1.1.1.1:1"}}}}
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func init() {
+	NewRandomWRR = testutils.NewTestWRR
+}
+
+// TestPickerUpdateAfterClose covers the case where a child policy sends a
+// picker update after the cluster_impl policy is closed. Because picker updates
+// are handled in the run() goroutine, which exits before Close() returns, we
+// expect the above picker update to be dropped.
+func (s) TestPickerUpdateAfterClose(t *testing.T) {
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a subConn state
+	// change).
+	closeCh := make(chan struct{})
+	const childPolicyName = "stubBalancer-TestPickerUpdateAfterClose"
+	stub.Register(childPolicyName, stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			// Create a subConn which will be used later on to test the race
+			// between StateListener() and Close().
+			sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{
+				StateListener: func(balancer.SubConnState) {
+					go func() {
+						// Wait for Close() to be called on the parent policy before
+						// sending the picker update.
+						<-closeCh
+						bd.ClientConn.UpdateState(balancer.State{
+							Picker: base.NewErrPicker(errors.New("dummy error picker")),
+						})
+					}()
+				},
+			})
+			if err != nil {
+				return err
+			}
+			sc.Connect()
+			return nil
+		},
+	})
+
+	var maxRequest uint32 = 50
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+						MaxRequests:    &maxRequest,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: childPolicyName,
+			},
+		},
+	}); err != nil {
+		b.Close()
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	// Send a subConn state change to trigger a picker update. The stub balancer
+	// that we use as the child policy will not send a picker update until the
+	// parent policy is closed.
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	b.Close()
+	close(closeCh)
+
+	select {
+	case <-cc.NewPickerCh:
+		t.Fatalf("unexpected picker update after balancer is closed")
+	case <-time.After(defaultShortTestTimeout):
+	}
+}
+
+// TestClusterNameInAddressAttributes covers the case that cluster name is
+// attached to the subconn address attributes.
+func (s) TestClusterNameInAddressAttributes(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	sc1 := <-cc.NewSubConnCh
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
+	// This should get the connecting picker.
+	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	addrs1 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs1[0].Addr, testBackendEndpoints[0].Addresses[0].Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	cn, ok := xdsinternal.GetXDSHandshakeClusterName(addrs1[0].Attributes)
+	if !ok || cn != testClusterName {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName)
+	}
+
+	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
+	// Test pick with one backend.
+	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	const testClusterName2 = "test-cluster-2"
+	var addr2 = resolver.Address{Addr: "2.2.2.2"}
+	state2 := xdsclient.SetClient(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{addr2}}}}, xdsC)
+	state2 = xdsresource.SetXDSConfig(state2, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName2: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						ClusterName:    testClusterName2,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state2,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName2,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: roundrobin.Name,
+			},
+		},
+	}); err != nil {
+		t.Fatalf("unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	addrs2 := <-cc.NewSubConnAddrsCh
+	if got, want := addrs2[0].Addr, addr2.Addr; got != want {
+		t.Fatalf("sc is created with addr %v, want %v", got, want)
+	}
+	// New addresses should have the new cluster name.
+	cn2, ok := xdsinternal.GetXDSHandshakeClusterName(addrs2[0].Attributes)
+	if !ok || cn2 != testClusterName2 {
+		t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2)
+	}
+}
+
+// Test verify that the case picker is updated synchronously on receipt of
+// configuration update.
+func (s) TestPickerUpdatedSynchronouslyOnConfigUpdate(t *testing.T) {
+	// Override the pickerUpdateHook to be notified that picker was updated.
+	pickerUpdated := make(chan struct{}, 1)
+	origNewPickerUpdated := pickerUpdateHook
+	pickerUpdateHook = func() {
+		pickerUpdated <- struct{}{}
+	}
+	defer func() { pickerUpdateHook = origNewPickerUpdated }()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Override the clientConnUpdateHook to ensure client conn was updated.
+	clientConnUpdateDone := make(chan struct{}, 1)
+	origClientConnUpdateHook := clientConnUpdateHook
+	clientConnUpdateHook = func() {
+		// Verify that picker was updated before the completion of
+		// client conn update.
+		select {
+		case <-pickerUpdated:
+		case <-ctx.Done():
+			t.Fatal("Client conn update completed before picker update.")
+		}
+		clientConnUpdateDone <- struct{}{}
+	}
+	defer func() { clientConnUpdateHook = origClientConnUpdateHook }()
+
+	builder := balancer.Get(Name)
+	cc := testutils.NewBalancerClientConn(t)
+	b := builder.Build(cc, balancer.BuildOptions{})
+	defer b.Close()
+
+	// Create a stub balancer which waits for the cluster_impl policy to be
+	// closed before sending a picker update (upon receipt of a resolver
+	// update).
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			bd.ClientConn.UpdateState(balancer.State{
+				Picker: base.NewErrPicker(errors.New("dummy error picker")),
+			})
+			return nil
+		},
+	})
+
+	xdsC := fakeclient.NewClient()
+	state := xdsclient.SetClient(resolver.State{Endpoints: testBackendEndpoints}, xdsC)
+	state = xdsresource.SetXDSConfig(state, &xdsresource.XDSConfig{
+		Clusters: map[string]*xdsresource.ClusterResult{
+			testClusterName: {
+				Config: xdsresource.ClusterConfig{
+					Cluster: &xdsresource.ClusterUpdate{
+						ClusterName:    testClusterName,
+						ClusterType:    xdsresource.ClusterTypeEDS,
+						EDSServiceName: testServiceName,
+					},
+					EndpointConfig: &xdsresource.EndpointConfig{EDSUpdate: &xdsresource.EndpointsUpdate{}},
+				},
+			},
+		},
+	})
+	if err := b.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState: state,
+		BalancerConfig: &LBConfig{
+			Cluster: testClusterName,
+			ChildPolicy: &internalserviceconfig.BalancerConfig{
+				Name: t.Name(),
+			},
+		},
+	}); err != nil {
+		t.Fatalf("Unexpected error from UpdateClientConnState: %v", err)
+	}
+
+	select {
+	case <-clientConnUpdateDone:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for client conn update to be completed.")
+	}
+}
+
+func (s) TestDropRequestsPerMillion(t *testing.T) {
+	tests := []struct {
+		name        string
+		numerator   uint32
+		denominator uint32
+		want        uint32
+	}{
+		{name: "zero", numerator: 0, denominator: million, want: 0},
+		{name: "five percent hundred", numerator: 5, denominator: 100, want: 50000},
+		// numerator*million overflows a uint32 once numerator reaches 4295 with
+		// the million denominator, which is only a 0.43% drop.
+		{name: "point four three percent million", numerator: 4295, denominator: million, want: 4295},
+		{name: "one percent million", numerator: 10000, denominator: million, want: 10000},
+		{name: "fifty percent million", numerator: 500000, denominator: million, want: 500000},
+		{name: "hundred percent million", numerator: million, denominator: million, want: million},
+		// A fraction above 100% is capped at a million.
+		{name: "over hundred percent", numerator: 150, denominator: 100, want: million},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := dropRequestsPerMillion(tt.numerator, tt.denominator); got != tt.want {
+				t.Errorf("dropRequestsPerMillion(%d, %d) = %d, want %d", tt.numerator, tt.denominator, got, tt.want)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/clusterimpl.go

@@ -0,0 +1,644 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package clusterimpl implements the xds_cluster_impl balancing policy. It
+// handles the cluster features (e.g. circuit_breaking, RPC dropping).
+//
+// Note that it doesn't handle name resolution, which is done by policy
+// xds_cluster_resolver.
+package clusterimpl
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"slices"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/weightedroundrobin"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal/balancer/gracefulswitch"
+	"google.golang.org/grpc/internal/credentials/xds"
+	"google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/pretty"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+
+	"google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/balancer/loadstore"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+const (
+	// Name is the name of the cluster_impl balancer.
+	Name                   = "xds_cluster_impl_experimental"
+	defaultRequestCountMax = 1024
+	loadStoreStopTimeout   = 1 * time.Second
+)
+
+var (
+	// Below function is no-op in actual code, but can be overridden in
+	// tests to give tests visibility into exactly when certain events happen.
+	clientConnUpdateHook = func() {}
+	pickerUpdateHook     = func() {}
+	buildProvider        = buildProviderFunc
+)
+
+func init() {
+	balancer.Register(bb{})
+}
+
+type bb struct{}
+
+func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	b := &clusterImplBalancer{
+		ClientConn:      cc,
+		loadWrapper:     loadstore.NewWrapper(),
+		requestCountMax: defaultRequestCountMax,
+	}
+	b.xdsHIPtr.Store(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+	b.logger = prefixLogger(b)
+	b.child = gracefulswitch.NewBalancer(b, bOpts)
+	b.logger.Infof("Created")
+
+	var creds credentials.TransportCredentials
+	switch {
+	case bOpts.DialCreds != nil:
+		creds = bOpts.DialCreds
+	case bOpts.CredsBundle != nil:
+		creds = bOpts.CredsBundle.TransportCredentials()
+	}
+	if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() {
+		b.xdsCredsInUse = true
+	}
+	b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse)
+	return b
+}
+
+func (bb) Name() string {
+	return Name
+}
+
+func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+	return parseConfig(c)
+}
+
+type clusterImplBalancer struct {
+	balancer.ClientConn
+
+	// The following fields are set at creation time, and are read-only after
+	// that, and therefore need not be protected by a mutex.
+	logger *grpclog.PrefixLogger
+	// TODO: #8366 -  Refactor usage of loadWrapper to easily plugin a test
+	// load reporter from tests.
+	loadWrapper *loadstore.Wrapper
+
+	// The following fields are only accessed from balancer API methods, which
+	// are guaranteed to be called serially by gRPC.
+	xdsClient        xdsclient.XDSClient     // Sent down in ResolverState attributes.
+	cancelLoadReport func(context.Context)   // To stop reporting load through the above xDS client.
+	edsServiceName   string                  // EDS service name to report load for.
+	lrsServer        *bootstrap.ServerConfig // Load reporting server configuration.
+	dropCategories   []DropConfig            // The categories for drops.
+	child            *gracefulswitch.Balancer
+	xdsHIPtr         atomic.Pointer[grpcsync.RefCounted[*xds.HandshakeInfo]] // Accessed atomically as it is shared between the balancer and the transport.
+	xdsCredsInUse    bool
+
+	// The active security configuration received from the control plane.
+	securityConfig *xdsresource.SecurityConfig
+
+	// The following fields are protected by mu, since they are accessed in
+	// balancer API methods and in methods called from the child policy.
+	mu                       sync.Mutex
+	clusterName              string                                      // The cluster name for credentials handshaking.
+	inhibitPickerUpdates     bool                                        // Inhibits state updates from child policy when processing an update from the parent.
+	pendingPickerUpdates     bool                                        // True if a picker update from the child policy was inhibited when processing an update from the parent.
+	childState               balancer.State                              // Most recent state update from the child policy.
+	drops                    []*dropper                                  // Drops implementation.
+	requestCounterCluster    string                                      // The cluster name for the request counter, from LB config.
+	requestCounterService    string                                      // The service name for the request counter, from LB config.
+	requestCountMax          uint32                                      // Max concurrent requests, from LB config.
+	requestCounter           *xdsclient.ClusterRequestsCounter           // Tracks total inflight requests for a given service.
+	telemetryLabels          map[string]string                           // Telemetry labels to set on picks, from LB config.
+	lrsReportEndpointMetrics *xdsresource.LRSReportEndpointMetricsConfig // LRS metrics to propagate.
+}
+
+// dropRequestsPerMillion scales a drop overload's numerator and denominator to
+// a number of requests to drop per million. numerator can be as large as a
+// million (a 100% drop), so the multiplication is done in uint64 to avoid
+// overflowing a uint32, and the result is capped at a million because a drop
+// ratio above 100% is treated as 100%.
+func dropRequestsPerMillion(numerator, denominator uint32) uint32 {
+	rpm := uint64(numerator) * million / uint64(denominator)
+	if rpm > million {
+		rpm = million
+	}
+	return uint32(rpm)
+}
+
+// handleClusterConfigLocked updates the internal state of the balancer with the
+// new cluster configuration. It returns true if a new picker needs to be
+// generated as a result of these changes. It must be called with b.mu held.
+func (b *clusterImplBalancer) handleClusterConfigLocked(clusterConfig xdsresource.ClusterConfig) bool {
+	clusterUpdate := clusterConfig.Cluster
+	var updatePicker bool
+
+	b.telemetryLabels = clusterUpdate.TelemetryLabels
+
+	if !b.lrsReportEndpointMetrics.Equal(clusterUpdate.LRSReportEndpointMetrics) {
+		b.lrsReportEndpointMetrics = clusterUpdate.LRSReportEndpointMetrics
+		updatePicker = true
+	}
+
+	var newDrops []DropConfig
+	if clusterUpdate.ClusterType == xdsresource.ClusterTypeEDS {
+		edsUpdate := clusterConfig.EndpointConfig.EDSUpdate
+		newDrops = make([]DropConfig, 0, len(edsUpdate.Drops))
+		for _, d := range edsUpdate.Drops {
+			newDrops = append(newDrops, DropConfig{
+				Category:           d.Category,
+				RequestsPerMillion: dropRequestsPerMillion(d.Numerator, d.Denominator),
+			})
+		}
+		if !slices.Equal(b.dropCategories, newDrops) {
+			b.dropCategories = newDrops
+			b.drops = make([]*dropper, 0, len(newDrops))
+			for _, c := range newDrops {
+				b.drops = append(b.drops, newDropper(c))
+			}
+			updatePicker = true
+		}
+	}
+
+	if b.requestCounterCluster != clusterUpdate.ClusterName || b.requestCounterService != clusterUpdate.EDSServiceName {
+		b.requestCounterCluster = clusterUpdate.ClusterName
+		b.requestCounterService = clusterUpdate.EDSServiceName
+		b.requestCounter = xdsclient.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName)
+		updatePicker = true
+	}
+	var newRequestCountMax uint32 = 1024
+	if clusterUpdate.MaxRequests != nil {
+		newRequestCountMax = *clusterUpdate.MaxRequests
+	}
+	if b.requestCountMax != newRequestCountMax {
+		b.requestCountMax = newRequestCountMax
+		updatePicker = true
+	}
+
+	return updatePicker
+}
+
+func (b *clusterImplBalancer) newPickerLocked() *picker {
+	return &picker{
+		drops:           b.drops,
+		s:               b.childState,
+		loadStore:       b.loadWrapper,
+		counter:         b.requestCounter,
+		countMax:        b.requestCountMax,
+		telemetryLabels: b.telemetryLabels,
+		clusterName:     b.clusterName,
+		metrics:         b.lrsReportEndpointMetrics,
+	}
+}
+
+// updateLoadStore checks the config for load store, and decides whether it
+// needs to restart the load reporting stream.
+func (b *clusterImplBalancer) updateLoadStore(clusterUpdate *xdsresource.ClusterUpdate) error {
+	var updateLoadClusterAndService bool
+
+	// ClusterName is different, restart. ClusterName is from ClusterName and
+	// EDSServiceName.
+	clusterName := b.getClusterName()
+	if clusterName != clusterUpdate.ClusterName {
+		updateLoadClusterAndService = true
+		b.setClusterName(clusterUpdate.ClusterName)
+		clusterName = clusterUpdate.ClusterName
+	}
+	if b.edsServiceName != clusterUpdate.EDSServiceName {
+		updateLoadClusterAndService = true
+		b.edsServiceName = clusterUpdate.EDSServiceName
+	}
+	if updateLoadClusterAndService {
+		// This updates the clusterName and serviceName that will be reported
+		// for the loads. The update here is too early, the perfect timing is
+		// when the picker is updated with the new connection. But from this
+		// balancer's point of view, it's impossible to tell.
+		//
+		// On the other hand, this will almost never happen. Each LRS policy
+		// shouldn't get updated config. The parent should do a graceful switch
+		// when the clusterName or serviceName is changed.
+		b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName)
+	}
+
+	var (
+		stopOldLoadReport  bool
+		startNewLoadReport bool
+	)
+
+	// Check if it's necessary to restart load report.
+	if b.lrsServer == nil {
+		if clusterUpdate.LRSServerConfig != nil {
+			// Old is nil, new is not nil, start new LRS.
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			startNewLoadReport = true
+		}
+		// Old is nil, new is nil, do nothing.
+	} else if clusterUpdate.LRSServerConfig == nil {
+		// Old is not nil, new is nil, stop old, don't start new.
+		b.lrsServer = nil
+		stopOldLoadReport = true
+	} else {
+		// Old is not nil, new is not nil, compare string values, if
+		// different, stop old and start new.
+		if !b.lrsServer.Equal(clusterUpdate.LRSServerConfig) {
+			b.lrsServer = clusterUpdate.LRSServerConfig
+			stopOldLoadReport = true
+			startNewLoadReport = true
+		}
+	}
+
+	if stopOldLoadReport {
+		if b.cancelLoadReport != nil {
+			stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+			defer stopCancel()
+			b.cancelLoadReport(stopCtx)
+			b.cancelLoadReport = nil
+			if !startNewLoadReport {
+				// If a new LRS stream will be started later, no need to update
+				// it to nil here.
+				b.loadWrapper.UpdateLoadStore(nil)
+			}
+		}
+	}
+	if startNewLoadReport {
+		var loadStore *lrsclient.LoadStore
+		if b.xdsClient != nil {
+			loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(b.lrsServer)
+		}
+		b.loadWrapper.UpdateLoadStore(loadStore)
+	}
+
+	return nil
+}
+
+func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) {
+	cfg := configs[instanceName]
+	provider, err := cfg.Build(certprovider.BuildOptions{
+		CertName:     certName,
+		WantIdentity: wantIdentity,
+		WantRoot:     wantRoot,
+	})
+	if err != nil {
+		// This error is not expected since the bootstrap process parses the
+		// config and makes sure that it is acceptable to the plugin. Still, it
+		// is possible that the plugin parses the config successfully, but its
+		// Build() method errors out.
+		return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err)
+	}
+	return provider, nil
+}
+
+func (b *clusterImplBalancer) buildProviders(config *xdsresource.SecurityConfig) (certprovider.Provider, certprovider.Provider, error) {
+	cpc := b.xdsClient.BootstrapConfig().CertProviderConfigs()
+	var rootProvider certprovider.Provider
+	if config.UseSystemRootCerts {
+		rootProvider = systemRootCertsProvider{}
+	} else {
+		rp, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		rootProvider = rp
+	}
+
+	var identityProvider certprovider.Provider
+	if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" {
+		ip, err := buildProvider(cpc, name, cert, true, false)
+		if err != nil {
+			if rootProvider != nil {
+				rootProvider.Close()
+			}
+			return nil, nil, err
+		}
+		identityProvider = ip
+	}
+	return rootProvider, identityProvider, nil
+}
+
+// handleSecurityConfig processes the security configuration received from the
+// management server, creates appropriate certificate provider plugins, and
+// updates the HandshakeInfo which is added as an address attribute in
+// NewSubConn() calls.
+func (b *clusterImplBalancer) handleSecurityConfig(config *xdsresource.SecurityConfig) error {
+	// If xdsCredentials are not in use, i.e, the user did not want to get
+	// security configuration from an xDS server, we should not be acting on the
+	// received security config here. Doing so poses a security threat.
+	if !b.xdsCredsInUse || config.Equal(b.securityConfig) {
+		return nil
+	}
+
+	// Security config being nil is a valid case where the management server has
+	// not sent any security configuration. The xdsCredentials implementation
+	// handles this by delegating to its fallback credentials.
+	if config == nil {
+		b.securityConfig = nil
+		oldHI := b.xdsHIPtr.Swap(xds.NewHandshakeInfo(nil, nil, nil, "", false, false, false))
+		if oldHI != nil {
+			oldHI.Decrement()
+		}
+		return nil
+	}
+
+	rootProvider, identityProvider, err := b.buildProviders(config)
+	if err != nil {
+		return err
+	}
+
+	newHI := xds.NewHandshakeInfo(rootProvider, identityProvider, config.SubjectAltNameMatchers, config.SNI, false, config.AutoSNISANValidation, config.UseAutoHostSNI)
+	b.securityConfig = config
+	oldHI := b.xdsHIPtr.Swap(newHI)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	return nil
+}
+
+func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error {
+	defer clientConnUpdateHook()
+
+	b.mu.Lock()
+	b.inhibitPickerUpdates = true
+	b.mu.Unlock()
+	if b.logger.V(2) {
+		b.logger.Infof("Received configuration: %s", pretty.ToJSON(s.BalancerConfig))
+	}
+	newConfig, ok := s.BalancerConfig.(*LBConfig)
+	if !ok {
+		return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig)
+	}
+
+	// Need to check for potential errors at the beginning of this function, so
+	// that on errors, we reject the whole config, instead of applying part of
+	// it.
+	bb := balancer.Get(newConfig.ChildPolicy.Name)
+	if bb == nil {
+		return fmt.Errorf("child policy %q not registered", newConfig.ChildPolicy.Name)
+	}
+
+	if b.xdsClient == nil {
+		c := xdsclient.FromResolverState(s.ResolverState)
+		if c == nil {
+			return balancer.ErrBadResolverState
+		}
+		b.xdsClient = c
+	}
+
+	xdsConfig := xdsresource.XDSConfigFromResolverState(s.ResolverState)
+	if xdsConfig == nil {
+		b.logger.Warningf("Received balancer config with no xDS config")
+		return balancer.ErrBadResolverState
+	}
+	clusterCfg := xdsConfig.Clusters[newConfig.Cluster]
+	clusterUpdate := clusterCfg.Config.Cluster
+	if err := b.handleSecurityConfig(clusterUpdate.SecurityCfg); err != nil {
+		// If the security config is invalid, for example, if the provider
+		// instance is not found in the bootstrap config, we need to put the
+		// channel in transient failure.
+		return fmt.Errorf("received Cluster resource that contains invalid security config: %v", err)
+
+	}
+	// Update load reporting config. This needs to be done before updating the
+	// child policy because we need the loadStore from the updated client to be
+	// passed to the ccWrapper, so that the next picker from the child policy
+	// will pick up the new loadStore.
+	if err := b.updateLoadStore(clusterUpdate); err != nil {
+		return err
+	}
+
+	// Build config for the gracefulswitch balancer. It is safe to ignore JSON
+	// marshaling errors here, since the config was already validated as part of
+	// ParseConfig().
+	cfg := []map[string]any{{newConfig.ChildPolicy.Name: newConfig.ChildPolicy.Config}}
+	cfgJSON, _ := json.Marshal(cfg)
+	parsedCfg, err := gracefulswitch.ParseConfig(cfgJSON)
+	if err != nil {
+		return err
+	}
+
+	// Addresses and sub-balancer config are sent to sub-balancer.
+	err = b.child.UpdateClientConnState(balancer.ClientConnState{
+		ResolverState:  weightedroundrobin.SetBackendService(s.ResolverState, b.clusterName),
+		BalancerConfig: parsedCfg,
+	})
+
+	b.mu.Lock()
+	updatePicker := b.handleClusterConfigLocked(clusterCfg.Config)
+	// We want to send a picker update to the parent if one of the two
+	// conditions are met:
+	// - drop/request count config or LRS metrics config has changed *and* there
+	//   is already a picker from the child, or
+	// - there is a pending picker update from the child (and this covers the
+	//   case where the drop/request config has not changed, but the child sent
+	//   a picker update while we were still processing config from our parent).
+	if (updatePicker && b.childState.Picker != nil) || b.pendingPickerUpdates {
+		b.pendingPickerUpdates = false
+		b.ClientConn.UpdateState(balancer.State{
+			ConnectivityState: b.childState.ConnectivityState,
+			Picker:            b.newPickerLocked(),
+		})
+	}
+	b.inhibitPickerUpdates = false
+	b.mu.Unlock()
+	pickerUpdateHook()
+	return err
+}
+
+func (b *clusterImplBalancer) ResolverError(err error) {
+	b.child.ResolverError(err)
+}
+
+func (b *clusterImplBalancer) updateSubConnState(_ balancer.SubConn, s balancer.SubConnState, cb func(balancer.SubConnState)) {
+	// Trigger re-resolution when a SubConn turns transient failure. This is
+	// necessary for the LogicalDNS in cluster_resolver policy to re-resolve.
+	//
+	// Note that this happens not only for the addresses from DNS, but also for
+	// EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent
+	// knows). The parent priority policy is configured to ignore re-resolution
+	// signal from the EDS children.
+	if s.ConnectivityState == connectivity.TransientFailure {
+		b.ClientConn.ResolveNow(resolver.ResolveNowOptions{})
+	}
+
+	if cb != nil {
+		cb(s)
+	}
+}
+
+func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) {
+	b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)
+}
+
+func (b *clusterImplBalancer) Close() {
+	b.child.Close()
+	b.childState = balancer.State{}
+
+	if b.cancelLoadReport != nil {
+		stopCtx, stopCancel := context.WithTimeout(context.Background(), loadStoreStopTimeout)
+		defer stopCancel()
+		b.cancelLoadReport(stopCtx)
+		b.cancelLoadReport = nil
+	}
+	oldHI := b.xdsHIPtr.Swap(nil)
+	if oldHI != nil {
+		oldHI.Decrement()
+	}
+	b.securityConfig = nil
+	b.logger.Infof("Shutdown")
+}
+
+func (b *clusterImplBalancer) ExitIdle() {
+	b.child.ExitIdle()
+}
+
+// Override methods to accept updates from the child LB.
+
+func (b *clusterImplBalancer) UpdateState(state balancer.State) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	// Inhibit sending a picker update to our parent as part of handling new
+	// state from the child, if we are currently handling an update from our
+	// parent. Update the childState field regardless.
+	b.childState = state
+	if b.inhibitPickerUpdates {
+		b.pendingPickerUpdates = true
+		if b.logger.V(2) {
+			b.logger.Infof("Received a picker update from the child when processing an update from the parent")
+		}
+		return
+	}
+
+	b.ClientConn.UpdateState(balancer.State{
+		ConnectivityState: state.ConnectivityState,
+		Picker:            b.newPickerLocked(),
+	})
+	pickerUpdateHook()
+}
+
+func (b *clusterImplBalancer) setClusterName(n string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	b.clusterName = n
+}
+
+func (b *clusterImplBalancer) getClusterName() string {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.clusterName
+}
+
+// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be
+// retrieved from the addresses when creating SubConn.
+//
+// All SubConns passed to the child policies are wrapped in this, so that the
+// picker can get the localityID from the picked SubConn, and do load reporting.
+//
+// After wrapping, all SubConns to and from the parent ClientConn (e.g. for
+// SubConn state update, update/remove SubConn) must be the original SubConns.
+// All SubConns to and from the child policy (NewSubConn, forwarding SubConn
+// state update) must be the wrapper. The balancer keeps a map from the original
+// SubConn to the wrapper for this purpose.
+type scWrapper struct {
+	balancer.SubConn
+
+	localityID clients.Locality
+
+	hostname string
+}
+
+func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	clusterName := b.getClusterName()
+	newAddrs := make([]resolver.Address, len(addrs))
+	for i, addr := range addrs {
+		newAddrs[i] = xdsinternal.SetXDSHandshakeClusterName(addr, clusterName)
+		newAddrs[i] = xds.SetHandshakeInfo(newAddrs[i], &b.xdsHIPtr)
+
+		host := xdsresource.Hostname(addr)
+		// If the hostname contains a port, strip it. Per [RFC 6066, Section
+		// 3](https://www.rfc-editor.org/rfc/rfc6066.html#section-3), the SNI
+		// may only contain a qualified DNS hostname, which excludes port
+		// numbers.
+		h, _, err := net.SplitHostPort(host)
+		if err == nil {
+			host = h
+		}
+		// Store hostname in the address attributes, so that it can be used in
+		// the client handshake.
+		newAddrs[i] = xds.SetAddressHostname(newAddrs[i], host)
+	}
+	var sc balancer.SubConn
+	scw := &scWrapper{}
+	if len(addrs) > 0 {
+		scw.hostname = xdsresource.Hostname(addrs[0])
+		scw.localityID = xdsinternal.GetLocalityID(addrs[0])
+	}
+	oldListener := opts.StateListener
+	opts.StateListener = func(state balancer.SubConnState) {
+		b.updateSubConnState(sc, state, oldListener)
+	}
+	sc, err := b.ClientConn.NewSubConn(newAddrs, opts)
+	if err != nil {
+		return nil, err
+	}
+	scw.SubConn = sc
+	return scw, nil
+}
+
+func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) {
+	b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc)
+}
+
+func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, _ []resolver.Address) {
+	b.logger.Errorf("UpdateAddresses(%v) called unexpectedly", sc)
+}
+
+// systemRootCertsProvider implements a certprovider.Provider that returns the
+// system default root certificates for validation.
+type systemRootCertsProvider struct{}
+
+func (systemRootCertsProvider) Close() {}
+
+func (systemRootCertsProvider) KeyMaterial(context.Context) (*certprovider.KeyMaterial, error) {
+	rootCAs, err := internal.X509SystemCertPoolFunc()
+	if err != nil {
+		return nil, err
+	}
+	return &certprovider.KeyMaterial{Roots: rootCAs}, nil
+}

internal/xds/balancer/clusterimpl/config.go

@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"encoding/json"
+
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// DropConfig contains the category, and drop ratio.
+type DropConfig struct {
+	Category           string
+	RequestsPerMillion uint32
+}
+
+// LBConfig is the balancer config for cluster_impl balancer.
+type LBConfig struct {
+	serviceconfig.LoadBalancingConfig `json:"-"`
+
+	Cluster     string                                `json:"cluster,omitempty"`
+	ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"`
+}
+
+func parseConfig(c json.RawMessage) (*LBConfig, error) {
+	var cfg LBConfig
+	if err := json.Unmarshal(c, &cfg); err != nil {
+		return nil, err
+	}
+	return &cfg, nil
+}

internal/xds/balancer/clusterimpl/config_test.go

@@ -0,0 +1,132 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/balancer"
+	_ "google.golang.org/grpc/balancer/roundrobin"
+	_ "google.golang.org/grpc/balancer/weightedtarget"
+	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
+)
+
+const (
+	testJSONConfig = `{
+  "cluster": "test_cluster",
+  "edsServiceName": "test-eds",
+  "lrsLoadReportingServer": {
+    "server_uri": "trafficdirector.googleapis.com:443",
+    "channel_creds": [ { "type": "google_default" } ]
+  },
+  "maxConcurrentRequests": 123,
+  "dropCategories": [
+    {
+      "category": "drop-1",
+      "requestsPerMillion": 314
+    },
+    {
+      "category": "drop-2",
+      "requestsPerMillion": 159
+    }
+  ],
+  "childPolicy": [
+    {
+      "weighted_target_experimental": {
+        "targets": {
+          "wt-child-1": {
+            "weight": 75,
+            "childPolicy":[{"round_robin":{}}]
+          },
+          "wt-child-2": {
+            "weight": 25,
+            "childPolicy":[{"round_robin":{}}]
+          }
+        }
+      }
+    }
+  ]
+}`
+
+	wtName = "weighted_target_experimental"
+)
+
+var (
+	wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser)
+	wtConfigJSON   = `{
+  "targets": {
+    "wt-child-1": {
+      "weight": 75,
+      "childPolicy":[{"round_robin":{}}]
+    },
+    "wt-child-2": {
+      "weight": 25,
+      "childPolicy":[{"round_robin":{}}]
+    }
+  }
+}`
+
+	wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON))
+)
+
+func (s) TestParseConfig(t *testing.T) {
+	tests := []struct {
+		name    string
+		js      string
+		want    *LBConfig
+		wantErr bool
+	}{
+		{
+			name:    "empty json",
+			js:      "",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name:    "bad json",
+			js:      "{",
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "OK",
+			js:   testJSONConfig,
+			want: &LBConfig{
+				Cluster: "test_cluster",
+				ChildPolicy: &internalserviceconfig.BalancerConfig{
+					Name:   wtName,
+					Config: wtConfig,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig([]byte(tt.js))
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if diff := cmp.Diff(tt.want, got); diff != "" {
+				t.Errorf("parseConfig() got unexpected diff (-want, +got): %v", diff)
+			}
+		})
+	}
+}

internal/xds/balancer/clusterimpl/internal/internal.go

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package internal contains code internal to the clusterimpl package.
+package internal
+
+import "crypto/x509"
+
+// X509SystemCertPoolFunc is used for overriding the system cert pool for
+// tests.
+var X509SystemCertPoolFunc = x509.SystemCertPool

internal/xds/balancer/clusterimpl/logging.go

@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+const prefix = "[xds-cluster-impl-lb %p] "
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p))
+}

internal/xds/balancer/clusterimpl/picker.go

@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package clusterimpl
+
+import (
+	"context"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/wrr"
+	xdsinternal "google.golang.org/grpc/internal/xds"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+)
+
+// NewRandomWRR is used when calculating drops. It's exported so that tests can
+// override it.
+var NewRandomWRR = wrr.NewRandom
+
+const million = 1000000
+
+type dropper struct {
+	category string
+	w        wrr.WRR
+}
+
+// greatest common divisor (GCD) via Euclidean algorithm
+func gcd(a, b uint32) uint32 {
+	for b != 0 {
+		t := b
+		b = a % b
+		a = t
+	}
+	return a
+}
+
+func newDropper(c DropConfig) *dropper {
+	w := NewRandomWRR()
+	gcdv := gcd(c.RequestsPerMillion, million)
+	// Return true for RequestPerMillion, false for the rest.
+	w.Add(true, int64(c.RequestsPerMillion/gcdv))
+	w.Add(false, int64((million-c.RequestsPerMillion)/gcdv))
+
+	return &dropper{
+		category: c.Category,
+		w:        w,
+	}
+}
+
+func (d *dropper) drop() (ret bool) {
+	return d.w.Next().(bool)
+}
+
+// loadReporter wraps the methods from the loadStore that are used here.
+type loadReporter interface {
+	CallStarted(locality clients.Locality)
+	CallFinished(locality clients.Locality, err error)
+	CallServerLoad(locality clients.Locality, name string, val float64)
+	CallDropped(category string)
+}
+
+// Picker implements RPC drop, circuit breaking drop and load reporting.
+type picker struct {
+	drops           []*dropper
+	s               balancer.State
+	loadStore       loadReporter
+	counter         *xdsclient.ClusterRequestsCounter
+	countMax        uint32
+	telemetryLabels map[string]string
+	clusterName     string
+	metrics         *xdsresource.LRSReportEndpointMetricsConfig
+}
+
+func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// Unconditionally update labels if present, even dropped or queued RPC's can
+	// use these labels.
+	stats.UpdateLabels(info.Ctx, d.telemetryLabels)
+
+	// Don't drop unless the inner picker is READY. Similar to
+	// https://github.com/grpc/grpc-go/issues/2622.
+	if d.s.ConnectivityState == connectivity.Ready {
+		// Check if this RPC should be dropped by category.
+		for _, dp := range d.drops {
+			if dp.drop() {
+				if d.loadStore != nil {
+					d.loadStore.CallDropped(dp.category)
+				}
+				return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped")
+			}
+		}
+	}
+
+	// Check if this RPC should be dropped by circuit breaking.
+	if d.counter != nil {
+		if err := d.counter.StartRequest(d.countMax); err != nil {
+			// Drops by circuit breaking are reported with empty category. They
+			// will be reported only in total drops, but not in per category.
+			if d.loadStore != nil {
+				d.loadStore.CallDropped("")
+			}
+			return balancer.PickResult{}, status.Error(codes.Unavailable, err.Error())
+		}
+	}
+
+	var lID clients.Locality
+	pr, err := d.s.Picker.Pick(info)
+	if scw, ok := pr.SubConn.(*scWrapper); ok {
+		// This OK check also covers the case err!=nil, because SubConn will be
+		// nil.
+		pr.SubConn = scw.SubConn
+		// If locality ID isn't found in the wrapper, an empty locality ID will
+		// be used.
+		lID = scw.localityID
+
+		if scw.hostname != "" && autoHostRewriteEnabled(info.Ctx) {
+			if pr.Metadata == nil {
+				pr.Metadata = metadata.Pairs(":authority", scw.hostname)
+			} else {
+				pr.Metadata.Set(":authority", scw.hostname)
+			}
+		}
+	}
+
+	if err != nil {
+		if d.counter != nil {
+			// Release one request count if this pick fails.
+			d.counter.EndRequest()
+		}
+		return pr, err
+	}
+
+	stats.UpdateLabels(
+		info.Ctx,
+		map[string]string{
+			"grpc.lb.locality":        xdsinternal.LocalityString(lID),
+			"grpc.lb.backend_service": d.clusterName,
+		},
+	)
+
+	if d.loadStore != nil {
+		locality := clients.Locality{Region: lID.Region, Zone: lID.Zone, SubZone: lID.SubZone}
+		d.loadStore.CallStarted(locality)
+		oldDone := pr.Done
+		pr.Done = func(info balancer.DoneInfo) {
+			if oldDone != nil {
+				oldDone(info)
+			}
+			d.loadStore.CallFinished(locality, info.Err)
+
+			load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport)
+			if !ok || load == nil {
+				return
+			}
+
+			if envconfig.XDSORCAToLRSPropEnabled {
+				if d.metrics != nil {
+					if d.metrics.CPUUtilization {
+						d.loadStore.CallServerLoad(locality, "cpu_utilization", load.CpuUtilization)
+					}
+					if d.metrics.MemUtilization {
+						d.loadStore.CallServerLoad(locality, "mem_utilization", load.MemUtilization)
+					}
+					if d.metrics.ApplicationUtilization {
+						d.loadStore.CallServerLoad(locality, "application_utilization", load.ApplicationUtilization)
+					}
+					for n, c := range load.NamedMetrics {
+						_, ok := d.metrics.NamedMetrics[n]
+						if d.metrics.NamedMetricsAll || ok {
+							d.loadStore.CallServerLoad(locality, "named_metrics."+n, c)
+						}
+					}
+				}
+			} else {
+				for n, c := range load.NamedMetrics {
+					d.loadStore.CallServerLoad(locality, n, c)
+				}
+			}
+		}
+	}
+
+	if d.counter != nil {
+		// Update Done() so that when the RPC finishes, the request count will
+		// be released.
+		oldDone := pr.Done
+		pr.Done = func(doneInfo balancer.DoneInfo) {
+			d.counter.EndRequest()
+			if oldDone != nil {
+				oldDone(doneInfo)
+			}
+		}
+	}
+
+	return pr, err
+}
+
+// autoHostRewriteKey is the context key used to store the value of
+// route's autoHostRewrite in the RPC context.
+type autoHostRewriteKey struct{}
+
+// autoHostRewriteEnabled retrieves the autoHostRewrite value from the provided context.
+func autoHostRewriteEnabled(ctx context.Context) bool {
+	v, _ := ctx.Value(autoHostRewriteKey{}).(bool)
+	return v
+}
+
+// AutoHostRewriteEnabledForTesting returns the value of autoHostRewrite field;
+// to be used for testing only.
+func AutoHostRewriteEnabledForTesting(ctx context.Context) bool {
+	return autoHostRewriteEnabled(ctx)
+}
+
+// EnableAutoHostRewrite adds the autoHostRewrite value to the context for
+// the xds_cluster_impl LB policy to pick.
+func EnableAutoHostRewrite(ctx context.Context) context.Context {
+	return context.WithValue(ctx, autoHostRewriteKey{}, true)
+}

internal/xds/balancer/clusterimpl/tests/clusterimpl_security_test.go

@@ -0,0 +1,783 @@
+/*
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package clusterimpl_test
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	iclusterimpl "google.golang.org/grpc/internal/xds/balancer/clusterimpl/internal"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/testdata"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin.
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router"       // Register the router filter.
+	_ "google.golang.org/grpc/internal/xds/resolver"                // Register the xds resolver
+)
+
+// Common setup for security tests:
+//   - creates an xDS client with the specified bootstrap configuration
+//   - creates a gRPC channel that uses the passed in client creds
+//   - creates a test server that uses the passed in server creds
+//
+// Returns the following:
+// - a client channel to make RPCs
+// - address of the test backend server
+func setupForSecurityTests(t *testing.T, bootstrapContents []byte, clientCreds, serverCreds credentials.TransportCredentials) (*grpc.ClientConn, string) {
+	t.Helper()
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Create a ClientConn with the specified transport credentials.
+	cc, err := grpc.NewClient(r.Scheme()+":///test.service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	t.Cleanup(func() { cc.Close() })
+
+	// Start a test service backend with the specified transport credentials.
+	sOpts := []grpc.ServerOption{}
+	if serverCreds != nil {
+		sOpts = append(sOpts, grpc.Creds(serverCreds))
+	}
+	server := stubserver.StartTestService(t, nil, sOpts...)
+	t.Cleanup(server.Stop)
+
+	return cc, server.Address
+}
+
+// Creates transport credentials to be used on the client side that rely on xDS
+// to provide the security configuration. It falls back to insecure creds if no
+// security information is received from the management server.
+func xdsClientCredsWithInsecureFallback(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	xdsCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create xDS credentials: %v", err)
+	}
+	return xdsCreds
+}
+
+// Creates transport credentials to be used on the server side from certificate
+// files in testdata/x509.
+//
+// The certificate returned by this function has a CommonName of "test-server1".
+func tlsServerCreds(t *testing.T) credentials.TransportCredentials {
+	t.Helper()
+
+	cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("Failed to load server cert and key: %v", err)
+
+	}
+	pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem"))
+	if err != nil {
+		t.Fatalf("Failed to read client CA cert: %v", err)
+	}
+	roots := x509.NewCertPool()
+	roots.AppendCertsFromPEM(pemData)
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ClientCAs:    roots,
+	}
+	return credentials.NewTLS(cfg)
+}
+
+// Checks the AuthInfo available in the peer if it matches the expected security
+// level of the connection.
+func verifySecurityInformationFromPeer(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel) {
+	// This is not a true helper in the Go sense, because it does not perform
+	// setup or cleanup tasks. Marking it a helper is to ensure that when the
+	// test fails, the line information of the caller is outputted instead of
+	// from here.
+	//
+	// And this function directly calls t.Fatalf() instead of returning an error
+	// and letting the caller decide what to do with it. This is also OK since
+	// all callers will simply end up calling t.Fatalf() with the returned
+	// error, and can't add any contextual information of value to the error
+	// message.
+	t.Helper()
+
+	switch wantSecLevel {
+	case e2e.SecurityLevelNone:
+		if pr.AuthInfo.AuthType() != "insecure" {
+			t.Fatalf("AuthType() is %s, want insecure", pr.AuthInfo.AuthType())
+		}
+	case e2e.SecurityLevelMTLS:
+		ai, ok := pr.AuthInfo.(credentials.TLSInfo)
+		if !ok {
+			t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{})
+		}
+		if len(ai.State.PeerCertificates) != 1 {
+			t.Fatalf("Number of peer certificates is %d, want 1", len(ai.State.PeerCertificates))
+		}
+		cert := ai.State.PeerCertificates[0]
+		const wantCommonName = "test-server1"
+		if cn := cert.Subject.CommonName; cn != wantCommonName {
+			t.Fatalf("Common name in peer certificate is %s, want %s", cn, wantCommonName)
+		}
+	}
+}
+
+// makeAggregateClusterResource returns an aggregate cluster resource with the
+// given name and list of child names.
+func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster {
+	return e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: name,
+		Type:        e2e.ClusterTypeAggregate,
+		ChildNames:  childNames,
+	})
+}
+
+// Tests the case where xDS credentials are not in use, but the cds LB policy
+// receives a Cluster update with security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with insecure creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, insecure.NewCredentials(), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where xDS credentials are in use, but the cds LB policy
+// receives a Cluster update without security configuration. Verifies that the
+// connection between client and server is insecure.
+func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with
+	// insecure credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure default resources in the management server. The
+	// cluster resource is not configured to return any security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made.
+	peer := &peer.Peer{}
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the security config returned by the management server
+// cannot be resolved based on the contents of the bootstrap config. Verifies
+// that the cds LB policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server,
+	// and one that does not have certificate providers configuration.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       8080,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// A certificate provider builder that returns a nil Provider from the starter
+// func passed to certprovider.NewBuildableConfig().
+type errCertProviderBuilder struct{}
+
+const errCertProviderName = "err-cert-provider"
+
+func (e errCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) {
+	// Returning a nil Provider simulates the case where an error is encountered
+	// at the time of building the Provider.
+	bc := certprovider.NewBuildableConfig(errCertProviderName, nil, func(certprovider.BuildOptions) certprovider.Provider { return nil })
+	return bc, nil
+}
+
+func (e errCertProviderBuilder) Name() string {
+	return errCertProviderName
+}
+
+func init() {
+	certprovider.Register(errCertProviderBuilder{})
+}
+
+// Tests the case where the certprovider.Store returns an error when the cds LB
+// policy attempts to build a certificate provider. Verifies that the cds LB
+// policy puts the channel in TRANSIENT_FAILURE.
+func (s) TestCertproviderStoreError(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration for
+	// errCertProviderBuilder.
+	nodeID := uuid.New().String()
+	providerCfg := json.RawMessage(fmt.Sprintf(`{
+		"plugin_name": "%s",
+		"config": {}
+	}`, errCertProviderName))
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+		CertificateProviders:               map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create a grpc channel with xDS creds.
+	cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil)
+
+	// Configure a cluster resource that contains security configuration, in the
+	// management server.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure.
+func (s) TestGoodSecurityConfig(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that contains a certificate provider instance
+// that is missing in the bootstrap file. Verifies that the channel moves to
+// TRANSIENT_FAILURE. Subsequently, the cds LB policy receives a cluster
+// resource that contains a certificate provider that is present in the
+// bootstrap file.  Verifies that the connection between the client and the
+// server is secure.
+func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) {
+	var (
+		target      = "test.service"
+		routeName   = "route1"
+		clusterName = "cluster1"
+		serviceName = "service1"
+	)
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)
+	cluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with initial resources: %v", err)
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Update the management server with a Cluster resource that contains a
+	// certificate provider instance that is present in the bootstrap
+	// configuration.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with valid resources: %v", err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource without security configuration. Verifies that this results in the
+// use of fallback credentials, which in this case is insecure creds.
+func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) {
+	target := "test.service"
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// Start a test service backend that does not expect a secure connection.
+	insecureServer := stubserver.StartTestService(t, nil)
+	t.Cleanup(insecureServer.Stop)
+
+	// Update the resources in the management server to contain no security
+	// configuration. This should result in the use of fallback credentials,
+	// which is insecure in our case.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: target,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, insecureServer.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to move to the new backend that expects
+	// connections without security.
+	for ctx.Err() == nil {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+			t.Logf("EmptyCall() failed: %v", err)
+		}
+		if peer.Addr.String() == insecureServer.Address {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatal("Timed out when waiting for connection to switch to second backend")
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that can be successfully resolved using the
+// bootstrap file contents. Verifies that the connection between the client and
+// the server is secure. Subsequently, the cds LB policy receives a cluster
+// resource that is NACKed by the xDS client. Test verifies that the cds LB
+// policy continues to use the previous good configuration.
+func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The
+	// cluster resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	// The cluster resource contains security configuration with a certificate
+	// provider instance that is missing in the bootstrap configuration.
+	resources.Clusters[0] = e2e.DefaultCluster(resources.Clusters[0].Name, resources.Endpoints[0].ClusterName, e2e.SecurityLevelNone)
+	resources.Clusters[0].TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName: "unknown-certificate-provider-instance",
+						},
+					},
+				},
+			}),
+		},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}
+
+// Tests the case where the cds LB policy receives security configuration as
+// part of the Cluster resource that specifies the use system root certs.
+// Verifies that the connection between the client and the server is secure.
+func (s) TestSystemRootCertsSecurityConfig(t *testing.T) {
+	origFlag := envconfig.XDSSystemRootCertsEnabled
+	origSRCF := iclusterimpl.X509SystemCertPoolFunc
+	defer func() {
+		envconfig.XDSSystemRootCertsEnabled = origFlag
+		iclusterimpl.X509SystemCertPoolFunc = origSRCF
+	}()
+	envconfig.XDSSystemRootCertsEnabled = true
+
+	systemRootCertsFuncCalled := false
+	iclusterimpl.X509SystemCertPoolFunc = func() (*x509.CertPool, error) {
+		certData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem"))
+		if err != nil {
+			return nil, fmt.Errorf("failed to read certificate file: %w", err)
+		}
+		certPool := x509.NewCertPool()
+
+		if ok := certPool.AppendCertsFromPEM(certData); !ok {
+			return nil, fmt.Errorf("failed to append certificate to cert pool")
+		}
+		systemRootCertsFuncCalled = true
+		return certPool, nil
+	}
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configure default resources in the management server. The cluster
+	// resource is configured to return security configuration.
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: "test.service",
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, serverAddress),
+		SecLevel:   e2e.SecurityLevelTLSWithSystemRootCerts,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+
+	if systemRootCertsFuncCalled != true {
+		t.Errorf("System root certs were not used during the test.")
+	}
+}
+
+// TestAggregateClusterSecurityConfig tests the case where the CDS LB policy
+// receives a top-level aggregate cluster pointing to a child EDS cluster. The
+// test verifies that the CDS LB policy correctly ignores the aggregate
+// cluster's security configuration and uses the child EDS cluster's security
+// configuration to establish the mTLS connection.
+func (s) TestAggregateClusterSecurityConfig(t *testing.T) {
+	var (
+		target         = "test.service"
+		routeName      = "route1"
+		clusterName    = "cluster1"
+		serviceName    = "service1"
+		edsClusterName = clusterName + "-eds"
+	)
+
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server
+	// and one that includes certificate providers configuration.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create a grpc channel with xDS creds talking to a test server with TLS
+	// credentials.
+	cc, serverAddress := setupForSecurityTests(t, bc, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t))
+
+	// Configures an aggregate cluster with a security config that is not NACKed
+	// by the xdsclient but would fail the handshake as it has an invalid SAN
+	// matcher.
+	aggCluster := makeAggregateClusterResource(clusterName, []string{edsClusterName})
+	aggCluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "bad-cn"}},
+								},
+							},
+							ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+						},
+					},
+				},
+			}),
+		},
+	}
+
+	// Create the child EDS cluster with a valid mTLS security configuration.
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(target, routeName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeName, target, clusterName)},
+		Clusters: []*v3clusterpb.Cluster{
+			aggCluster,
+			e2e.DefaultCluster(edsClusterName, serviceName, e2e.SecurityLevelMTLS),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that a successful RPC can be made over a secure connection since
+	// the EDS cluster has a valid mTLS configuration even though the top level
+	// cluster has an incorrect security configuration.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS)
+}