snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

test/xds/xds_client_ack_nack_test.go

@@ -0,0 +1,202 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/resolver"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// We are interested in LDS, RDS, CDS and EDS resources as part of the regular
+// xDS flow on the client.
+const wantResources = 4
+
+// seenAllACKs returns true if the provided ackVersions map contains valid acks
+// for all the resources that we are interested in. If `wantNonEmpty` is true,
+// only non-empty ack versions are considered valid.
+func seenAllACKs(acksVersions map[string]string, wantNonEmpty bool) bool {
+	if len(acksVersions) != wantResources {
+		return false
+	}
+	for _, ack := range acksVersions {
+		if wantNonEmpty && ack == "" {
+			return false
+		}
+	}
+	return true
+}
+
+// TestClientResourceVersionAfterStreamRestart tests the scenario where the
+// xdsClient's ADS stream to the management server gets broken. This test
+// verifies that the version number on the initial request on the new stream
+// indicates the most recent version seen by the client on the previous stream.
+func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) {
+	// Create a restartable listener which can close existing connections.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	// We depend on the fact that the management server assigns monotonically
+	// increasing stream IDs starting at 1.
+	const (
+		idBeforeRestart = 1
+		idAfterRestart  = 2
+	)
+
+	// Events of importance in the test, in the order in which they are expected
+	// to happen.
+	acksReceivedBeforeRestart := grpcsync.NewEvent()
+	streamRestarted := grpcsync.NewEvent()
+	acksReceivedAfterRestart := grpcsync.NewEvent()
+
+	// Map from stream id to a map of resource type to resource version.
+	ackVersionsMap := make(map[int64]map[string]string)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error {
+			// Return early under the following circumstances:
+			// - Received all the requests we wanted to see. This is to avoid
+			//   any stray requests leading to test flakes.
+			// - Request contains no resource names. Such requests are usually
+			//   seen when the xdsclient is shutting down and is no longer
+			//   interested in the resources that it had subscribed to earlier.
+			if acksReceivedAfterRestart.HasFired() || len(req.GetResourceNames()) == 0 {
+				return nil
+			}
+			// Create a stream specific map to store ack versions if this is the
+			// first time we are seeing this stream id.
+			if ackVersionsMap[id] == nil {
+				ackVersionsMap[id] = make(map[string]string)
+			}
+			ackVersionsMap[id][req.GetTypeUrl()] = req.GetVersionInfo()
+			// Prior to stream restart, we are interested only in non-empty
+			// resource versions. The xdsclient first sends out requests with an
+			// empty version string. After receipt of requested resource, it
+			// sends out another request for the same resource, but this time
+			// with a non-empty version string, to serve as an ACK.
+			if seenAllACKs(ackVersionsMap[idBeforeRestart], true) {
+				acksReceivedBeforeRestart.Fire()
+			}
+			// After stream restart, we expect the xdsclient to send out
+			// requests with version string set to the previously ACKed
+			// versions. If it sends out requests with empty version string, it
+			// is a bug and we want this test to catch it.
+			if seenAllACKs(ackVersionsMap[idAfterRestart], false) {
+				acksReceivedAfterRestart.Fire()
+			}
+			return nil
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			streamRestarted.Fire()
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// A successful RPC means that the xdsclient received all requested
+	// resources. The ACKs from the xdsclient may get a little delayed. So, we
+	// need to wait for all ACKs to be received on the management server before
+	// restarting the stream.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for all resources to be ACKed prior to stream restart")
+	case <-acksReceivedBeforeRestart.Done():
+	}
+
+	// Stop the listener on the management server. This will cause the client to
+	// backoff and recreate the stream.
+	lis.Stop()
+
+	// Wait for the stream to be closed on the server.
+	<-streamRestarted.Done()
+
+	// Restart the listener on the management server to be able to accept
+	// reconnect attempts from the client.
+	lis.Restart()
+
+	// Wait for all the previously sent resources to be re-requested.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for all resources to be ACKed post stream restart")
+	case <-acksReceivedAfterRestart.Done():
+	}
+
+	if diff := cmp.Diff(ackVersionsMap[idBeforeRestart], ackVersionsMap[idAfterRestart]); diff != "" {
+		t.Fatalf("unexpected diff in ack versions before and after stream restart (-want, +got):\n%s", diff)
+	}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_affinity_test.go

@@ -0,0 +1,128 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// hashRouteConfig returns a RouteConfig resource with hash policy set to
+// header "session_id".
+func hashRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration {
+	return &v3routepb.RouteConfiguration{
+		Name: routeName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{ldsTarget},
+			Routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+				Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+					ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+					HashPolicy: []*v3routepb.RouteAction_HashPolicy{{
+						PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{
+							Header: &v3routepb.RouteAction_HashPolicy_Header{
+								HeaderName: "session_id",
+							},
+						},
+						Terminal: true,
+					}},
+				}},
+			}},
+		}},
+	}
+}
+
+// ringhashCluster returns a Cluster resource that picks ringhash as the lb
+// policy.
+func ringhashCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster {
+	return &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: edsServiceName,
+		},
+		LbPolicy: v3clusterpb.Cluster_RING_HASH,
+	}
+}
+
+// TestClientSideAffinitySanityCheck tests that the affinity config can be
+// propagated to pick the ring_hash policy. It doesn't test the affinity
+// behavior in ring_hash policy.
+func (s) TestClientSideAffinitySanityCheck(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	// Replace RDS and CDS resources with ringhash config, but keep the resource
+	// names.
+	resources.Routes = []*v3routepb.RouteConfiguration{hashRouteConfig(
+		resources.Routes[0].Name,
+		resources.Listeners[0].Name,
+		resources.Clusters[0].Name,
+	)}
+	resources.Clusters = []*v3clusterpb.Cluster{ringhashCluster(
+		resources.Clusters[0].Name,
+		resources.Clusters[0].EdsClusterConfig.ServiceName,
+	)}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_certificate_providers_test.go

@@ -0,0 +1,508 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with no security configuration. The test verifies that the gRPC client is
+// able to make RPCs to the backend which is configured to accept plaintext
+// connections. This ensures that the insecure fallback credentials are getting
+// used on the client.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t)
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with no
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with an mTLS security configuration. The test verifies that the gRPC client
+// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and
+// string. This ensures that when the certificate provider instance name
+// specified in the security configuration is not present in the bootstrap,
+// channel creation does not fail, but it moves to TRANSIENT_FAILURE and
+// subsequent rpcs fail.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) {
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server,
+	// with no certificate providers.
+	nodeID := uuid.New().String()
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with mTLS
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Make an RPC and ensure that expected error is returned.
+	wantErr := fmt.Sprintf("identity certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance)
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains one certificate
+// provider, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server responds with three clusters:
+//  1. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap
+//  2. contains no security configuration, hence should use insecure fallback
+//  3. contains invalid security configuration pointing to a non-existent
+//     certificate provider instance
+//
+// The test verifies that RPCs to the first two clusters succeed, while RPCs to
+// the third cluster fails with an appropriate code and error message.
+func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var xdsResolver resolver.Builder
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		var err error
+		xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Create test backends for all three clusters
+	// backend1 configured with TLS creds, represents cluster1
+	// backend2 configured with insecure creds, represents cluster2
+	// backend3 configured with insecure creds, represents cluster3
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const clusterName3 = "cluster3-" + serviceName
+	const endpointsName1 = "endpoints1-" + serviceName
+	const endpointsName2 = "endpoints2-" + serviceName
+	const endpointsName3 = "endpoints3-" + serviceName
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
+					}},
+				},
+			},
+		}},
+	}}
+	// Clusters:
+	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance.
+	// - cluster2 with no security configuration.
+	// - cluster3 with non-existent cert provider name.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+		func() *v3clusterpb.Cluster {
+			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
+			cluster3.TransportSocket = &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName: "non-existent-certificate-provider-instance-name",
+								},
+							},
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: "non-existent-certificate-provider-instance-name",
+							},
+						},
+					}),
+				},
+			}
+			return cluster3
+		}(),
+	}
+	// Endpoints for each of the above clusters with backends created earlier.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC to be routed to cluster1 and verify that it succeeds.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server1.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+
+	}
+
+	// Make an RPC to be routed to cluster2 and verify that it succeeds.
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("UnaryCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server2.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// Make an RPC to be routed to cluster3 and verify that it fails.
+	const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
+	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains one certificate
+// provider configured with SPIFFE Bundle Map roots on the client side, and xDS
+// credentials with an insecure fallback is specified at dial time. The
+// management server responds with three clusters:
+//  1. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap, and the server uses a SPIFFE cert.
+//  2. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap, and the server uses a SPIFFE cert chain.
+//  3. contains invalid security configuration pointing to a non-existent
+//     certificate provider instance
+//
+// The test verifies that RPCs to the first two clusters succeed, while RPCs to
+// the third cluster fails with an appropriate code and error message.
+func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfigurationSPIFFE(t *testing.T) {
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t)
+
+	// Create test backends for all three clusters
+	// backend1 configured with a SPIFFE cert, represents cluster1
+	// backend2 configured with a SPIFFE cert chain, represents cluster2
+	// backend3 configured with insecure creds, represents cluster3
+	serverCreds := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	serverCreds2 := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds2))
+	defer server2.Stop()
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const clusterName3 = "cluster3-" + serviceName
+	const endpointsName1 = "endpoints1-" + serviceName
+	const endpointsName2 = "endpoints2-" + serviceName
+	const endpointsName3 = "endpoints3-" + serviceName
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
+					}},
+				},
+			},
+		}},
+	}}
+	// Clusters:
+	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance and mTLS.
+	// - cluster2 with cert provider name e2e.ClientSideCertProviderInstance and mTLS.
+	// - cluster3 with non-existent cert provider name.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelMTLS),
+		func() *v3clusterpb.Cluster {
+			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
+			cluster3.TransportSocket = &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName: "non-existent-certificate-provider-instance-name",
+								},
+							},
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: "non-existent-certificate-provider-instance-name",
+							},
+						},
+					}),
+				},
+			}
+			return cluster3
+		}(),
+	}
+	// Endpoints for each of the above clusters with backends created earlier.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+		e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC to be routed to cluster1 and verify that it succeeds.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server1.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+	verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 1)
+
+	// Make an RPC to be routed to cluster2 and verify that it succeeds.
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("UnaryCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server2.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+	// In this call the server contains a peer chain of length 2
+	verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 2)
+
+	// Make an RPC to be routed to cluster3 and verify that it fails.
+	const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
+	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}

test/xds/xds_client_custom_lb_test.go

@@ -0,0 +1,269 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	_ "google.golang.org/grpc/balancer/leastrequest"       // To register least_request
+	_ "google.golang.org/grpc/balancer/weightedroundrobin" // To register weighted_round_robin
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
+	v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
+	v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3"
+	v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// wrrLocality is a helper that takes a proto message and returns a
+// WrrLocalityProto with the proto message marshaled into a proto.Any as a
+// child.
+func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality {
+	return &v3wrrlocalitypb.WrrLocality{
+		EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						TypedConfig: testutils.MarshalAny(t, m),
+					},
+				},
+			},
+		},
+	}
+}
+
+// clusterWithLBConfiguration returns a cluster resource with the proto message
+// passed Marshaled to an any and specified through the load_balancing_policy
+// field.
+func clusterWithLBConfiguration(t *testing.T, clusterName, edsServiceName string, secLevel e2e.SecurityLevel, m proto.Message) *v3clusterpb.Cluster {
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel)
+	cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{
+		Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+			{
+				TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+					TypedConfig: testutils.MarshalAny(t, m),
+				},
+			},
+		},
+	}
+	return cluster
+}
+
+// TestWRRLocality tests RPC distribution across a scenario with 5 backends,
+// with 2 backends in a locality with weight 1, and 3 backends in a second
+// locality with weight 2. Through xDS, the test configures a
+// wrr_locality_balancer with either a round robin or custom (specifying pick
+// first) child load balancing policy, and asserts the correct distribution
+// based on the locality weights and the endpoint picking policy specified.
+func (s) TestWrrLocality(t *testing.T) {
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+	backend3 := stubserver.StartTestService(t, nil)
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+	backend4 := stubserver.StartTestService(t, nil)
+	port4 := testutils.ParsePort(t, backend4.Address)
+	defer backend4.Stop()
+	backend5 := stubserver.StartTestService(t, nil)
+	port5 := testutils.ParsePort(t, backend5.Address)
+	defer backend5.Stop()
+	const serviceName = "my-service-client-side-xds"
+
+	tests := []struct {
+		name string
+		// Configuration will be specified through load_balancing_policy field.
+		wrrLocalityConfiguration *v3wrrlocalitypb.WrrLocality
+		addressDistributionWant  []struct {
+			addr  string
+			count int
+		}
+	}{
+		{
+			name:                     "rr_child",
+			wrrLocalityConfiguration: wrrLocality(t, &v3roundrobinpb.RoundRobin{}),
+			// Each addresses expected probability is locality weight of
+			// locality / total locality weights multiplied by 1 / number of
+			// endpoints in each locality (due to round robin across endpoints
+			// in a locality). Thus, address 1 and address 2 have 1/3 * 1/2
+			// probability, and addresses 3 4 5 have 2/3 * 1/3 probability of
+			// being routed to.
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+		// This configures custom lb as the child of wrr_locality, which points
+		// to our pick_first implementation. Thus, the expected distribution of
+		// addresses is locality weight of locality / total locality weights as
+		// the probability of picking the first backend within the locality
+		// (e.g. Address 1 for locality 1, and Address 3 for locality 2).
+		{
+			name: "custom_lb_child_pick_first",
+			wrrLocalityConfiguration: wrrLocality(t, &v3xdsxdstypepb.TypedStruct{
+				TypeUrl: "type.googleapis.com/pick_first",
+				Value:   &structpb.Struct{},
+			}),
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 1},
+				{addr: backend3.Address, count: 2},
+			},
+		},
+		// Sanity check for weighted round robin. Don't need to test super
+		// specific behaviors, as that is covered in unit tests. Set up weighted
+		// round robin as the endpoint picking policy with per RPC load reports
+		// enabled. Due the server not sending trailers with load reports, the
+		// weighted round robin policy should essentially function as round
+		// robin, and thus should have the same distribution as round robin
+		// above.
+		{
+			name: "custom_lb_child_wrr/",
+			wrrLocalityConfiguration: wrrLocality(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{
+				EnableOobLoadReport: &wrapperspb.BoolValue{
+					Value: false,
+				},
+				// BlackoutPeriod long enough to cause load report weights to
+				// trigger in the scope of test case, but no load reports
+				// configured anyway.
+				BlackoutPeriod:          durationpb.New(10 * time.Second),
+				WeightExpirationPeriod:  durationpb.New(10 * time.Second),
+				WeightUpdatePeriod:      durationpb.New(time.Second),
+				ErrorUtilizationPenalty: &wrapperspb.FloatValue{Value: 1},
+			}),
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+		{
+			name: "custom_lb_least_request",
+			wrrLocalityConfiguration: wrrLocality(t, &v3leastrequestpb.LeastRequest{
+				ChoiceCount: wrapperspb.UInt32(2),
+			}),
+			// The test performs a Unary RPC, and blocks until the RPC returns,
+			// and then makes the next Unary RPC. Thus, over iterations, no RPC
+			// counts are present. This causes least request's randomness of
+			// indexes to sample to converge onto a round robin distribution per
+			// locality. Thus, expect the same distribution as round robin
+			// above.
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Start an xDS management server.
+			managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			routeConfigName := "route-" + serviceName
+			clusterName := "cluster-" + serviceName
+			endpointsName := "endpoints-" + serviceName
+			resources := e2e.UpdateOptions{
+				NodeID:    nodeID,
+				Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+				Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+				Clusters:  []*v3clusterpb.Cluster{clusterWithLBConfiguration(t, clusterName, endpointsName, e2e.SecurityLevelNone, test.wrrLocalityConfiguration)},
+				Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+					ClusterName: endpointsName,
+					Host:        "localhost",
+					Localities: []e2e.LocalityOptions{
+						{
+							Backends: []e2e.BackendOptions{{Ports: []uint32{port1}}, {Ports: []uint32{port2}}},
+							Weight:   1,
+						},
+						{
+							Backends: []e2e.BackendOptions{{Ports: []uint32{port3}}, {Ports: []uint32{port4}}, {Ports: []uint32{port5}}},
+							Weight:   2,
+						},
+					},
+				})},
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("Failed to dial local test server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testgrpc.NewTestServiceClient(cc)
+			var addrDistWant []resolver.Address
+			for _, addrAndCount := range test.addressDistributionWant {
+				for i := 0; i < addrAndCount.count; i++ {
+					addrDistWant = append(addrDistWant, resolver.Address{Addr: addrAndCount.addr})
+				}
+			}
+			if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, t, client, addrDistWant); err != nil {
+				t.Fatalf("Error in expected round robin: %v", err)
+			}
+		})
+	}
+}

test/xds/xds_client_federation_test.go

@@ -0,0 +1,343 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestClientSideFederation tests that federation is supported.
+//
+// In this test, some xDS responses contain resource names in another authority
+// (in the new resource name style):
+// - LDS: old style, no authority (default authority)
+// - RDS: new style, in a different authority
+// - CDS: old style, no authority (default authority)
+// - EDS: new style, in a different authority
+func (s) TestClientSideFederation(t *testing.T) {
+	// Start a management server as the default authority.
+	serverDefaultAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Start another management server as the other authority.
+	const nonDefaultAuth = "non-default-auth"
+	serverAnotherAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create a bootstrap file in a temporary directory.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverDefaultAuth.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+		// Specify the address of the non-default authority.
+		Authorities: map[string]json.RawMessage{
+			nonDefaultAuth: []byte(fmt.Sprintf(`{
+				"xds_servers": [
+					{
+						"server_uri": %q,
+						"channel_creds": [{"type": "insecure"}]
+					}
+				]
+			}`, serverAnotherAuth.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	// LDS is old style name.
+	ldsName := serviceName
+	// RDS is new style, with the non default authority.
+	rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", nonDefaultAuth, "route-"+serviceName)
+	// CDS is old style name.
+	cdsName := "cluster-" + serviceName
+	// EDS is new style, with the non default authority.
+	edsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.ClusterLoadAssignment/%s", nonDefaultAuth, "endpoints-"+serviceName)
+
+	// Split resources, put LDS/CDS in the default authority, and put RDS/EDS in
+	// the other authority.
+	resourcesDefault := e2e.UpdateOptions{
+		NodeID: nodeID,
+		// This has only LDS and CDS.
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
+		SkipValidation: true,
+	}
+	resourcesAnother := e2e.UpdateOptions{
+		NodeID: nodeID,
+		// This has only RDS and EDS.
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// This has only LDS and CDS.
+	if err := serverDefaultAuth.Update(ctx, resourcesDefault); err != nil {
+		t.Fatal(err)
+	}
+	// This has only RDS and EDS.
+	if err := serverAnotherAuth.Update(ctx, resourcesAnother); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestClientSideFederationWithOnlyXDSTPStyleLDS tests that federation is
+// supported with new xdstp style names for LDS only while using the old style
+// for other resources. This test in addition also checks that when service name
+// contains escapable characters, we "fully" encode it for looking up
+// VirtualHosts in xDS RouteConfiguration.
+func (s) TestClientSideFederationWithOnlyXDSTPStyleLDS(t *testing.T) {
+	// Start a management server as a sophisticated authority.
+	const authority = "traffic-manager.xds.notgoogleapis.com"
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create a bootstrap file in a temporary directory.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ClientDefaultListenerResourceNameTemplate: fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%%s", authority),
+		// Specify the address of the non-default authority.
+		Authorities: map[string]json.RawMessage{
+			authority: []byte(fmt.Sprintf(`{
+				"xds_servers": [
+					{
+						"server_uri": %q,
+						"channel_creds": [{"type": "insecure"}]
+					}
+				]
+			}`, mgmtServer.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// serviceName with escapable characters - ' ', and '/'.
+	const serviceName = "my-service-client-side-xds/2nd component"
+
+	// All other resources are with old style name.
+	const rdsName = "route-" + serviceName
+	const cdsName = "cluster-" + serviceName
+	const edsName = "endpoints-" + serviceName
+
+	// Resource update sent to go-control-plane mgmt server.
+	resourceUpdate := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: func() []*v3listenerpb.Listener {
+			// LDS is new style xdstp name. Since the LDS resource name is prefixed
+			// with xdstp, the string will be %-encoded excluding '/'s. See
+			// bootstrap.PopulateResourceTemplate().
+			const specialEscapedServiceName = "my-service-client-side-xds/2nd%20component" // same as bootstrap.percentEncode(serviceName)
+			ldsName := fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%s", authority, specialEscapedServiceName)
+			return []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}
+		}(),
+		Routes: func() []*v3routepb.RouteConfiguration {
+			// RouteConfiguration will has one entry in []VirtualHosts that contains the
+			// "fully" escaped service name in []Domains. This is to assert that gRPC
+			// uses the escaped service name to lookup VirtualHosts. RDS is also with
+			// old style name.
+			const fullyEscapedServiceName = "my-service-client-side-xds%2F2nd%20component" // same as url.PathEscape(serviceName)
+			return []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, fullyEscapedServiceName, cdsName)}
+		}(),
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resourceUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestFederation_UnknownAuthorityInDialTarget tests the case where a ClientConn
+// is created with a dial target containing an authority which is not specified
+// in the bootstrap configuration. The test verifies that RPCs on the ClientConn
+// fail with an appropriate error.
+func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) {
+	// Setting up the management server is not *really* required for this test
+	// case. All we need is a bootstrap configuration which does not contain the
+	// authority mentioned in the dial target. But setting up the management
+	// server and actually making an RPC ensures that the xDS client is
+	// configured properly, and when we dial with an unknown authority in the
+	// next step, we can be sure that the error we receive is legitimate.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	target := fmt.Sprintf("xds:///%s", serviceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed %q: %v", target, err)
+	}
+	defer cc.Close()
+	t.Log("Created ClientConn to test service")
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() RPC: %v", err)
+	}
+	t.Log("Successfully performed an EmptyCall RPC")
+
+	target = fmt.Sprintf("xds://unknown-authority/%s", serviceName)
+	t.Logf("Creating a channel with unknown authority %q, expecting failure", target)
+	wantErr := fmt.Sprintf("authority \"unknown-authority\" specified in dial target %q is not found in the bootstrap file", target)
+	cc, err = grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Unexpected error while creating ClientConn: %v", err)
+	}
+	defer cc.Close()
+	client = testgrpc.NewTestServiceClient(cc)
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall(_, _) = _, %v; want _, %q", err, wantErr)
+	}
+}
+
+// TestFederation_UnknownAuthorityInReceivedResponse tests the case where the
+// LDS resource associated with the dial target contains an RDS resource name
+// with an authority which is not specified in the bootstrap configuration. The
+// test verifies that RPCs fail with an appropriate error.
+func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) {
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// LDS is old style name.
+	// RDS is new style, with an unknown authority.
+	const serviceName = "my-service-client-side-xds"
+	const unknownAuthority = "unknown-authority"
+	ldsName := serviceName
+	rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", unknownAuthority, "route-"+serviceName)
+
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "cluster-"+serviceName)},
+		SkipValidation: true, // This update has only LDS and RDS resources.
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	target := fmt.Sprintf("xds:///%s", serviceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed %q: %v", target, err)
+	}
+	defer cc.Close()
+	t.Log("Created ClientConn to test service")
+
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if err == nil {
+		t.Fatal("EmptyCall RPC succeeded for target with unknown authority when expected to fail")
+	}
+	if got, want := status.Code(err), codes.Unavailable; got != want {
+		t.Fatalf("EmptyCall RPC returned status code: %v, want %v", got, want)
+	}
+}

test/xds/xds_client_filter_e2e_test.go

@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Tests the interaction between the defaultStreamInterceptor and xDS filters.
+// It verifies that the SendMsg, RecvMsg, and CloseSend methods are called the
+// expected number of times for different RPC types (unary, client streaming,
+// server streaming, and bidi streaming) when an xDS filter is registered and
+// used in the call chain.
+func (s) TestDefaultStreamInterceptor_InteractionWithXDSFilters(t *testing.T) {
+	// Register a custom xDS filter builder for the test.
+	testFilterTypeURL := t.Name()
+	filterBuilder := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	httpfilter.Register(filterBuilder)
+	defer httpfilter.UnregisterForTesting(testFilterTypeURL)
+
+	// Setup a test backend implementing both all four RPC types.
+	testServer := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
+			return stream.Send(&testpb.StreamingOutputCallResponse{})
+		},
+		StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error {
+			for {
+				_, err := stream.Recv()
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					return err
+				}
+			}
+			return stream.SendAndClose(&testpb.StreamingInputCallResponse{})
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				_, err := stream.Recv()
+				if err == io.EOF {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+				if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	if err := testServer.Start(nil); err != nil {
+		t.Fatal("Error starting server:", err)
+	}
+	defer testServer.Stop()
+
+	// Start an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	const serviceName = "my-service-xds"
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: "cluster-A"}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "tracking-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster("cluster-A", "cluster-A", e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("cluster-A", "localhost", []uint32{testutils.ParsePort(t, testServer.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+	client := testgrpc.NewTestServiceClient(cc)
+
+	tests := []struct {
+		name        string
+		run         func(ctx context.Context, client testgrpc.TestServiceClient)
+		wantSendMsg int32
+		wantRecvMsg int32
+	}{
+		{
+			name: "unary",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+					t.Fatalf("EmptyCall() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 2, // 1 response + 1 trailers
+		},
+		{
+			name: "client_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.StreamingInputCall(ctx)
+				if err != nil {
+					t.Fatalf("StreamingInputCall() failed: %v", err)
+				}
+				for i := 0; i < 2; i++ {
+					if err := stream.Send(&testpb.StreamingInputCallRequest{}); err != nil {
+						t.Fatalf("Send() failed: %v", err)
+					}
+				}
+				if _, err := stream.CloseAndRecv(); err != nil {
+					t.Fatalf("CloseAndRecv() failed: %v", err)
+				}
+			},
+			wantSendMsg: 2, // 2 requests
+			wantRecvMsg: 2, // 1 response + 1 trailers
+		},
+		{
+			name: "server_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{})
+				if err != nil {
+					t.Fatalf("StreamingOutputCall() failed: %v", err)
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("Recv() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 1, // 1 response (trailers not yet consumed since stream not read to EOF)
+		},
+		{
+			name: "bidi_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.FullDuplexCall(ctx)
+				if err != nil {
+					t.Fatalf("FullDuplexCall() failed: %v", err)
+				}
+				if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+					t.Fatalf("Send() failed: %v", err)
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("Recv() failed: %v", err)
+				}
+				if err := stream.CloseSend(); err != nil {
+					t.Fatalf("CloseSend() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 1, // 1 response
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			filterBuilder.sendMsgCount.Store(0)
+			filterBuilder.recvMsgCount.Store(0)
+			filterBuilder.closeSendCount.Store(0)
+
+			tc.run(ctx, client)
+
+			if got := filterBuilder.sendMsgCount.Load(); got != tc.wantSendMsg {
+				t.Fatalf("SendMsg() count = %d, want %d", got, tc.wantSendMsg)
+			}
+			if got := filterBuilder.recvMsgCount.Load(); got != tc.wantRecvMsg {
+				t.Fatalf("RecvMsg() count = %d, want %d", got, tc.wantRecvMsg)
+			}
+			if got := filterBuilder.closeSendCount.Load(); got != 1 {
+				t.Fatalf("CloseSend() count = %d, want 1", got)
+			}
+		})
+	}
+}

test/xds/xds_client_http_connect_test.go

@@ -0,0 +1,642 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/proxyserver"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+)
+
+func enableXDSHTTPConnect(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, true)
+	cleanup, err := xdsresource.RegisterMetadataConverterForTesting(version.V3AddressURL)
+	if err != nil {
+		t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err)
+	}
+	t.Cleanup(cleanup)
+}
+
+func splitHostPort(t *testing.T, addr string) (string, int) {
+	t.Helper()
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("Failed to split host and port for address %q: %v", addr, err)
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		t.Fatalf("Failed to parse port %q: %v", portStr, err)
+	}
+	return host, port
+}
+
+// Test veifies client-side HTTP CONNECT proxying support where the proxy
+// address is configured via endpoint metadata. It verifies that gRPC requests
+// are successfully routed through the HTTP CONNECT proxy to the correct
+// backend server.
+func (s) TestXDSHTTPConnect(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support where the proxy
+// address is configured via metadata at the locality level. It verifies that
+// the client successfully fallback to using the locality-level proxy address
+// if no endpoint-level metadata is present.
+func (s) TestXDSHTTPConnect_LocalityFallback(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				// Locality-level metadata contains the proxy address.
+				Metadata: &v3corepb.Metadata{
+					TypedFilterMetadata: map[string]*anypb.Any{
+						"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+							Address: &v3corepb.Address_SocketAddress{
+								SocketAddress: &v3corepb.SocketAddress{
+									Address:       proxyHost,
+									PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+								},
+							},
+						}),
+					},
+				},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when the client
+// uses XDS credentials. It verifies that when a plaintext Http11Proxy wrapper
+// is present in CDS, the XDS client successfully falls back to insecure
+// credentials and establishes a proxy CONNECT tunnel to the correct backend
+// server without security handshaking.
+func (s) TestXDSHTTPConnect_WithInsecureXDSCredentials(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create client XDS credentials: %v", err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when the cluster
+// uses an Http11ProxyUpstreamTransport wrapping an inner UpstreamTlsContext.
+// It verifies that a secure TLS session is established with the backend over
+// the HTTP CONNECT proxy connection.
+func (s) TestXDSHTTPConnect_WithUpstreamTLSContext(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName: e2e.ClientSideCertProviderInstance,
+										},
+									},
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							}),
+						},
+					},
+				}),
+			},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create client XDS credentials: %v", err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s  want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when cluster
+// specifies an HTTP CONNECT proxy transport socket, but no proxy address
+// metadata is supplied. It verifies that the client gracefully fallback
+// to dialing the backend directly.
+func (s) TestXDSHTTPConnect_NoMetadata(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_ignore_resource_deletion_test.go

@@ -0,0 +1,502 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const (
+	serviceName = "my-service-xds"
+	rdsName     = "route-" + serviceName
+	cdsName1    = "cluster1-" + serviceName
+	cdsName2    = "cluster2-" + serviceName
+	edsName1    = "eds1-" + serviceName
+	edsName2    = "eds2-" + serviceName
+)
+
+var (
+	// This route configuration resource contains two routes:
+	// - a route for the EmptyCall rpc, to be sent to cluster1
+	// - a route for the UnaryCall rpc, to be sent to cluster2
+	defaultRouteConfigWithTwoRoutes = &routepb.RouteConfiguration{
+		Name: rdsName,
+		VirtualHosts: []*routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*routepb.Route{
+				{
+					Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &routepb.Route_Route{Route: &routepb.RouteAction{
+						ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName1},
+					}},
+				},
+				{
+					Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &routepb.Route_Route{Route: &routepb.RouteAction{
+						ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName2},
+					}},
+				},
+			},
+		}},
+	}
+)
+
+// This test runs subtest each for a Listener resource and a Cluster resource deletion
+// in the response from the server for the following cases:
+//   - testResourceDeletionIgnored: When ignore_resource_deletion is set, the
+//     xDSClient should not delete the resource.
+//   - testResourceDeletionNotIgnored: When ignore_resource_deletion is unset,
+//     the xDSClient should delete the resource.
+//
+// Resource deletion is only applicable to Listener and Cluster resources.
+func (s) TestIgnoreResourceDeletionOnClient(t *testing.T) {
+	server1 := stubserver.StartTestService(t, nil)
+	t.Cleanup(server1.Stop)
+
+	server2 := stubserver.StartTestService(t, nil)
+	t.Cleanup(server2.Stop)
+
+	initialResourceOnServer := func(nodeID string) e2e.UpdateOptions {
+		return e2e.UpdateOptions{
+			NodeID:    nodeID,
+			Listeners: []*listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)},
+			Routes:    []*routepb.RouteConfiguration{defaultRouteConfigWithTwoRoutes},
+			Clusters: []*clusterpb.Cluster{
+				e2e.DefaultCluster(cdsName1, edsName1, e2e.SecurityLevelNone),
+				e2e.DefaultCluster(cdsName2, edsName2, e2e.SecurityLevelNone),
+			},
+			Endpoints: []*endpointpb.ClusterLoadAssignment{
+				e2e.DefaultEndpoint(edsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+				e2e.DefaultEndpoint(edsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+			},
+			SkipValidation: true,
+		}
+	}
+
+	tests := []struct {
+		name           string
+		updateResource func(r *e2e.UpdateOptions)
+	}{
+		{
+			name: "listener",
+			updateResource: func(r *e2e.UpdateOptions) {
+				r.Listeners = nil
+			},
+		},
+		{
+			name: "cluster",
+			updateResource: func(r *e2e.UpdateOptions) {
+				r.Clusters = nil
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s resource deletion ignored", test.name), func(t *testing.T) {
+			testResourceDeletionIgnored(t, initialResourceOnServer, test.updateResource)
+		})
+		t.Run(fmt.Sprintf("%s resource deletion not ignored", test.name), func(t *testing.T) {
+			testResourceDeletionNotIgnored(t, initialResourceOnServer, test.updateResource)
+		})
+	}
+}
+
+// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion"
+// set in "server_features" field. This subtest verifies that the resource was
+// not deleted by the xDSClient when a resource is missing the xDS response and
+// RPCs continue to succeed.
+func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	t.Cleanup(cancel)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources := initialResource(nodeID)
+
+	// Update the management server with initial resources setup.
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v.", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Mutate resource and update on the server.
+	updateResource(&resources)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make an RPC every 50ms for the next 500ms. This is to ensure that the
+	// updated resource is received from the management server and is processed by
+	// gRPC. Since resource deletions are ignored by the xDS client, we expect RPCs
+	// to all endpoints to keep succeeding.
+	timer := time.NewTimer(500 * time.Millisecond)
+	ticker := time.NewTicker(50 * time.Millisecond)
+	t.Cleanup(ticker.Stop)
+	for {
+		if err := verifyRPCtoAllEndpoints(cc); err != nil {
+			t.Fatal(err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion"
+// not set in "server_features" field. This subtest verifies that the resource was
+// deleted by the xDSClient when a resource is missing the xDS response and subsequent
+// RPCs fail.
+func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	t.Cleanup(cancel)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources := initialResource(nodeID)
+
+	// Update the management server with initial resources setup.
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Mutate resource and update on the server.
+	updateResource(&resources)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Spin up go routines to verify RPCs fail after the update. The xDS node ID
+	// needs to be part of the error seen by the RPC caller.
+	client := testgrpc.NewTestServiceClient(cc)
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+			_, err := client.EmptyCall(ctx, &testpb.Empty{})
+			if err == nil {
+				continue
+			}
+			if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) {
+				return
+			}
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+			_, err := client.UnaryCall(ctx, &testpb.SimpleRequest{})
+			if err == nil {
+				continue
+			}
+			if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) {
+				return
+			}
+		}
+	}()
+
+	wg.Wait()
+	if ctx.Err() != nil {
+		t.Fatal("Context expired before RPCs failed.")
+	}
+}
+
+// This helper generates a custom bootstrap config for the test.
+func generateBootstrapContents(t *testing.T, serverURI string, ignoreResourceDeletion bool, nodeID string) []byte {
+	t.Helper()
+	var serverCfgs json.RawMessage
+	if ignoreResourceDeletion {
+		serverCfgs = []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}],
+			"server_features": ["ignore_resource_deletion"]
+		}]`, serverURI))
+	} else {
+		serverCfgs = []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverURI))
+
+	}
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers:                            serverCfgs,
+		Node:                               fmt.Appendf(nil, `{"id": "%s"}`, nodeID),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return bootstrapContents
+}
+
+// This helper creates an XDS resolver Builder from the bootstrap config passed
+// as parameter.
+func xdsResolverBuilder(t *testing.T, bs []byte) resolver.Builder {
+	t.Helper()
+	xdsR, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bs)
+	if err != nil {
+		t.Fatalf("Creating xDS resolver for testing failed for config %q: %v", string(bs), err)
+	}
+	return xdsR
+}
+
+// This helper creates an xDS-enabled gRPC server using the listener and the
+// bootstrap config passed. It then registers the test service on the newly
+// created gRPC server and starts serving.
+func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContents []byte, lis net.Listener) chan connectivity.ServingMode {
+	t.Helper()
+	updateCh := make(chan connectivity.ServingMode, 1)
+
+	// Create a server option to get notified about serving mode changes.
+	modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
+		t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
+		updateCh <- args.Mode
+	})
+	stub := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+	}
+	server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
+	if err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+	stub.S = server
+	t.Cleanup(stub.S.Stop)
+
+	stubserver.StartTestService(t, stub)
+
+	return updateCh
+}
+
+// This helper creates a new TCP listener. This helper also uses this listener to
+// create a resource update with a listener resource. This helper returns the
+// resource update and the TCP listener.
+func resourceWithListenerForGRPCServer(t *testing.T, nodeID string) (e2e.UpdateOptions, net.Listener) {
+	t.Helper()
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	t.Cleanup(func() { lis.Close() })
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("Failed to retrieve host and port of listener at %q: %v", lis.Addr(), err)
+	}
+	listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{listener},
+	}
+	return resources, lis
+}
+
+// This test creates a gRPC server which provides server-side xDS functionality
+// by talking to a custom management server. This tests the scenario where bootstrap
+// config with "server_features" includes "ignore_resource_deletion". In which
+// case, when the listener resource is deleted on the management server, the gRPC
+// server should continue to serve RPCs.
+func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources, lis := resourceWithListenerForGRPCServer(t, nodeID)
+	modeChangeCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the server to update to ServingModeServing mode.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Test timed out waiting for a server to change to ServingModeServing.")
+	case mode := <-modeChangeCh:
+		if mode != connectivity.ServingModeServing {
+			t.Fatalf("Server switched to mode %v, want %v", mode, connectivity.ServingModeServing)
+		}
+	}
+
+	// Create a ClientConn and make a successful RPCs.
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update without a listener resource.
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Perform RPCs every 100 ms for 1s and verify that the serving mode does not
+	// change on gRPC server.
+	timer := time.NewTimer(500 * time.Millisecond)
+	ticker := time.NewTicker(50 * time.Millisecond)
+	t.Cleanup(ticker.Stop)
+	for {
+		if err := verifyRPCtoAllEndpoints(cc); err != nil {
+			t.Fatal(err)
+		}
+		select {
+		case <-timer.C:
+			return
+		case mode := <-modeChangeCh:
+			t.Fatalf("Server switched to mode: %v when no switch was expected", mode)
+		case <-ticker.C:
+		}
+	}
+}
+
+// This test creates a gRPC server which provides server-side xDS functionality
+// by talking to a custom management server. This tests the scenario where bootstrap
+// config with "server_features" does not include "ignore_resource_deletion". In
+// which case, when the listener resource is deleted on the management server, the
+// gRPC server should stop serving RPCs and switch mode to ServingModeNotServing.
+func (s) TestListenerResourceDeletionOnServerNotIgnored(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources, lis := resourceWithListenerForGRPCServer(t, nodeID)
+	updateCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the listener to move to "serving" mode.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Test timed out waiting for a mode change update.")
+	case mode := <-updateCh:
+		if mode != connectivity.ServingModeServing {
+			t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing)
+		}
+	}
+
+	// Create a ClientConn and make a successful RPCs.
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{}, // empty listener resource
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timed out waiting for a mode change update: %v", err)
+	case mode := <-updateCh:
+		if mode != connectivity.ServingModeNotServing {
+			t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing)
+		}
+	}
+}
+
+// This helper makes both UnaryCall and EmptyCall RPCs using the ClientConn that
+// is passed to this function. This helper panics for any failed RPCs.
+func verifyRPCtoAllEndpoints(cc grpc.ClientConnInterface) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("rpc EmptyCall() failed: %v", err)
+	}
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+		return fmt.Errorf("rpc UnaryCall() failed: %v", err)
+	}
+	return nil
+}

test/xds/xds_client_integration_test.go

@@ -0,0 +1,244 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/peer"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+func (s) TestClientSideXDS(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestClient_ConcurrentRPC ensures thread safety for xDS clients executing
+// concurrent RPCs, particularly verifying that the regex matchers do not cause
+// data races.
+func (s) TestClient_ConcurrentRPC(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{{
+			Name: routeConfigName,
+			VirtualHosts: []*v3routepb.VirtualHost{{
+				Domains: []string{serviceName},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_SafeRegex{
+							SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/.*"},
+						},
+					},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+					}},
+				}},
+			}},
+		}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, clusterName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	const numRPCs = 20
+	wg := sync.WaitGroup{}
+	for i := 0; i < numRPCs; i++ {
+		wg.Go(func() {
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Errorf("EmptyCall() failed: %v", err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
+// Test verifies that client-side xDS routing strictly evaluates
+// OutgoingContext metadata and ignores IncomingContext metadata. This
+// prevents incoming headers on a server handler context from leaking
+// into and contaminating outgoing client routing decisions.
+func (s) TestClientSideXDS_RouteMatching_IgnoreIncomingMetadata(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName     = "my-service-client"
+		routeConfigName = "route-" + serviceName
+		clusterA        = "cluster-A-" + serviceName
+		clusterB        = "cluster-B-" + serviceName
+		headerName      = "x-route-target"
+		headerValue     = "cluster-a"
+	)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{{
+			Name: routeConfigName,
+			VirtualHosts: []*v3routepb.VirtualHost{{
+				Domains: []string{serviceName},
+				Routes: []*v3routepb.Route{
+					{
+						Match: &v3routepb.RouteMatch{
+							PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+							Headers: []*v3routepb.HeaderMatcher{{
+								Name: headerName,
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{
+									ExactMatch: headerValue,
+								},
+							}},
+						},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA},
+						}},
+					},
+					{
+						Match: &v3routepb.RouteMatch{
+							PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+						},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB},
+						}},
+					},
+				},
+			}},
+		}},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone),
+			e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
+			e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)}),
+			e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}),
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Simulate an intermediary server that receives an incoming RPC containing
+	// the header "x-route-target: cluster-a" in its IncomingContext, and then
+	// initiates an outgoing client RPC using that same context without setting
+	// any OutgoingContext metadata. Client-side routing  must strictly evaluate
+	// OutgoingContext (which is empty) and select Route 2.
+	incomingCtx := metadata.NewIncomingContext(ctx, metadata.Pairs(headerName, headerValue))
+	var p peer.Peer
+	if _, err := client.EmptyCall(incomingCtx, &testpb.Empty{}, grpc.Peer(&p)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if p.Addr.String() != serverB.Address {
+		t.Fatalf("Unexpected client routing to peer %s, want fallback Route 2 (clusterB at %s)", p.Addr.String(), serverB.Address)
+	}
+}

test/xds/xds_client_outlier_detection_test.go

@@ -0,0 +1,323 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// TestOutlierDetection_NoopConfig tests the scenario where the Outlier
+// Detection feature is enabled on the gRPC client, but it receives no Outlier
+// Detection configuration from the management server. This should result in a
+// no-op Outlier Detection configuration being used to configure the Outlier
+// Detection balancer. This test verifies that an RPC is able to proceed
+// normally with this configuration.
+func (s) TestOutlierDetection_NoopConfig(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	server.StartServer()
+	t.Logf("Started test service backend at %q", server.Address)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// clientResourcesMultipleBackendsAndOD returns xDS resources which correspond
+// to multiple upstreams, corresponding different backends listening on
+// different localhost:port combinations. The resources also configure an
+// Outlier Detection Balancer configured through the passed in Outlier Detection
+// proto.
+func clientResourcesMultipleBackendsAndOD(params e2e.ResourceParams, ports []uint32, od *v3clusterpb.OutlierDetection) e2e.UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
+	return e2e.UpdateOptions{
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{clusterWithOutlierDetection(clusterName, endpointsName, params.SecLevel, od)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, ports)},
+	}
+}
+
+func clusterWithOutlierDetection(clusterName, edsServiceName string, secLevel e2e.SecurityLevel, od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster {
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel)
+	cluster.OutlierDetection = od
+	return cluster
+}
+
+// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn,
+// connected to a server exposing the test.grpc_testing.TestService, are
+// roundrobined across the given backend addresses.
+//
+// Returns a non-nil error if context deadline expires before RPCs start to get
+// roundrobined across the given backends.
+func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error {
+	wantAddrCount := make(map[string]int)
+	for _, addr := range addrs {
+		wantAddrCount[addr.Addr]++
+	}
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		// Perform 3 iterations.
+		var iterations [][]string
+		for i := 0; i < 3; i++ {
+			iteration := make([]string, len(addrs))
+			for c := 0; c < len(addrs); c++ {
+				var peer peer.Peer
+				client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer))
+				if peer.Addr != nil {
+					iteration[c] = peer.Addr.String()
+				}
+			}
+			iterations = append(iterations, iteration)
+		}
+		// Ensure the first iteration contains all addresses in addrs.
+		gotAddrCount := make(map[string]int)
+		for _, addr := range iterations[0] {
+			gotAddrCount[addr]++
+		}
+		if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" {
+			continue
+		}
+		// Ensure all three iterations contain the same addresses.
+		if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) {
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs)
+}
+
+// TestOutlierDetectionWithOutlier tests the Outlier Detection Balancer e2e. It
+// spins up three backends, one which consistently errors, and configures the
+// ClientConn using xDS to connect to all three of those backends. The Outlier
+// Detection Balancer should eject the connection to the backend which
+// constantly errors, causing RPC's to not be routed to that upstream, and only
+// be Round Robined across the two healthy upstreams. Other than the intervals
+// the unhealthy upstream is ejected, RPC's should regularly round robin across
+// all three upstreams.
+func (s) TestOutlierDetectionWithOutlier(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Working backend 1.
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+
+	// Working backend 2.
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+
+	// Backend 3 that will always return an error and eventually ejected.
+	backend3 := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") },
+	})
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		SecLevel:   e2e.SecurityLevelNone,
+	}, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{
+		Interval:                       &durationpb.Duration{Nanos: 50000000}, // .5 seconds
+		BaseEjectionTime:               &durationpb.Duration{Seconds: 30},
+		MaxEjectionTime:                &durationpb.Duration{Seconds: 300},
+		MaxEjectionPercent:             &wrapperspb.UInt32Value{Value: 1},
+		FailurePercentageThreshold:     &wrapperspb.UInt32Value{Value: 50},
+		EnforcingFailurePercentage:     &wrapperspb.UInt32Value{Value: 100},
+		FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 8},
+		FailurePercentageMinimumHosts:  &wrapperspb.UInt32Value{Value: 3},
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	fullAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+		{Addr: backend3.Address},
+	}
+	// At first, due to no statistics on each of the backends, the 3
+	// upstreams should all be round robined across.
+	if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+	}
+	// After calling the three upstreams, one of them constantly error
+	// and should eventually be ejected for a period of time. This
+	// period of time should cause the RPC's to be round robined only
+	// across the two that are healthy.
+	if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}
+
+// TestOutlierDetectionXDSDefaultOn tests that Outlier Detection is by default
+// configured on in the xDS Flow. If the Outlier Detection proto message is
+// present with SuccessRateEjection unset, then Outlier Detection should be
+// turned on. The test setups and xDS system with xDS resources with Outlier
+// Detection present in the CDS update, but with SuccessRateEjection unset, and
+// asserts that Outlier Detection is turned on and ejects upstreams.
+func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Working backend 1.
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+
+	// Working backend 2.
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+
+	// Backend 3 that will always return an error and eventually ejected.
+	backend3 := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") },
+	})
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+
+	// Configure CDS resources with Outlier Detection set but
+	// EnforcingSuccessRate unset. This should cause Outlier Detection to be
+	// configured with SuccessRateEjection present in configuration, which will
+	// eventually be populated with its default values along with the knobs set
+	// as SuccessRate fields in the proto, and thus Outlier Detection should be
+	// on and actively eject upstreams.
+	const serviceName = "my-service-client-side-xds"
+	resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		SecLevel:   e2e.SecurityLevelNone,
+	}, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{
+		// Need to set knobs to trigger ejection within the test time frame.
+		Interval: &durationpb.Duration{Nanos: 50000000},
+		// EnforcingSuccessRateSet to nil, causes success rate algorithm to be
+		// turned on.
+		SuccessRateMinimumHosts:  &wrapperspb.UInt32Value{Value: 1},
+		SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 8},
+		SuccessRateStdevFactor:   &wrapperspb.UInt32Value{Value: 1},
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	fullAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+		{Addr: backend3.Address},
+	}
+	// At first, due to no statistics on each of the backends, the 3
+	// upstreams should all be round robined across.
+	if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+	}
+	// After calling the three upstreams, one of them constantly error
+	// and should eventually be ejected for a period of time. This
+	// period of time should cause the RPC's to be round robined only
+	// across the two that are healthy.
+	if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}

test/xds/xds_client_priority_locality_test.go

@@ -0,0 +1,211 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	rrutil "google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// backendAddressesAndPorts extracts the address and port of each of the
+// StubServers passed in and returns them. Fails the test if any of the
+// StubServers passed have an invalid address.
+func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) {
+	addrs := make([]resolver.Address, len(servers))
+	ports := make([]uint32, len(servers))
+	for i := 0; i < len(servers); i++ {
+		addrs[i] = resolver.Address{Addr: servers[i].Address}
+		ports[i] = testutils.ParsePort(t, servers[i].Address)
+	}
+	return addrs, ports
+}
+
+// Tests scenarios involving localities moving between priorities.
+//   - The test starts off with a cluster that contains two priorities, one
+//     locality in each, and one endpoint in each. Verifies that traffic reaches
+//     the endpoint in the higher priority.
+//   - The test then moves the locality in the lower priority over to the higher
+//     priority. At that point, we would have a cluster with a single priority,
+//     but two localities, and one endpoint in each. Verifies that traffic is
+//     split between the endpoints.
+//   - The test then deletes the locality that was originally in the higher
+//     priority.Verifies that all traffic is now reaching the only remaining
+//     endpoint.
+func (s) TestClientSideXDS_LocalityChangesPriority(t *testing.T) {
+	// Spin up a management server and two test service backends.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	backend0 := stubserver.StartTestService(t, nil)
+	defer backend0.Stop()
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	addrs, ports := backendAddressesAndPorts(t, []*stubserver.StubServer{backend0, backend1})
+
+	// Configure resources on the management server. We use default client side
+	// resources for listener, route configuration and cluster. For the
+	// endpoints resource though, we create one with two priorities, and one
+	// locality each, and one endpoint each.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	locality1 := e2e.LocalityID{Region: "my-region-1", Zone: "my-zone-1", SubZone: "my-subzone-1"}
+	locality2 := e2e.LocalityID{Region: "my-region-2", Zone: "my-zone-2", SubZone: "my-subzone-2"}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointsName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "my-locality-1",
+					Weight:   1000000,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}},
+					Locality: locality1,
+				},
+				{
+					Name:     "my-locality-2",
+					Weight:   1000000,
+					Priority: 1,
+					Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}},
+					Locality: locality2,
+				},
+			},
+		})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// // Ensure that RPCs get routed to the backend in the higher priority.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with two
+	// localities, and one endpoint each. The locality weights are equal at this
+	// point, and we expect RPCs to be round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the locality weights ever so slightly. We still expect RPCs to be
+	// round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   499884,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500115,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with one
+	// locality. The locality which was originally in the higher priority is now
+	// dropped.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-2",
+				Weight:   1000000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+}

test/xds/xds_client_retry_test.go

@@ -0,0 +1,192 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestClientSideRetry(t *testing.T) {
+	// serverMu guards access to ctr and errs.
+	serverMu := sync.Mutex{}
+	ctr := 0
+	errs := []codes.Code{codes.ResourceExhausted}
+
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			serverMu.Lock()
+			defer func() {
+				ctr++
+				serverMu.Unlock()
+			}()
+			if ctr < len(errs) {
+				return nil, status.Errorf(errs[ctr], "this should be retried")
+			}
+			return &testpb.Empty{}, nil
+		},
+	})
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	defer cancel()
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.ResourceExhausted {
+		t.Fatalf("rpc EmptyCall() = _, %v; want _, ResourceExhausted", err)
+	}
+
+	testCases := []struct {
+		name        string
+		vhPolicy    *v3routepb.RetryPolicy
+		routePolicy *v3routepb.RetryPolicy
+		errs        []codes.Code // the errors returned by the server for each RPC
+		tryAgainErr codes.Code   // the error that would be returned if we are still using the old retry policies.
+		errWant     codes.Code
+	}{{
+		name: "virtualHost only, fail",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted,unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 1},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		routePolicy: nil,
+		tryAgainErr: codes.ResourceExhausted,
+		errWant:     codes.Unavailable,
+	}, {
+		name: "virtualHost only",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted, unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		routePolicy: nil,
+		tryAgainErr: codes.Unavailable,
+		errWant:     codes.OK,
+	}, {
+		name: "virtualHost+route, fail",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted,unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		tryAgainErr: codes.OK,
+		errWant:     codes.Unavailable,
+	}, {
+		name: "virtualHost+route",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.Unavailable},
+		tryAgainErr: codes.Unavailable,
+		errWant:     codes.OK,
+	}, {
+		name: "virtualHost+route, not enough attempts",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 1},
+		},
+		errs:        []codes.Code{codes.Unavailable, codes.Unavailable},
+		tryAgainErr: codes.OK,
+		errWant:     codes.Unavailable,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			serverMu.Lock()
+			errs = tc.errs
+
+			// Confirm tryAgainErr is correct before updating resources.
+			ctr = 0
+			serverMu.Unlock()
+			_, err := client.EmptyCall(ctx, &testpb.Empty{})
+			if code := status.Code(err); code != tc.tryAgainErr {
+				t.Fatalf("with old retry policy: EmptyCall() = _, %v; want _, %v", err, tc.tryAgainErr)
+			}
+
+			resources.Routes[0].VirtualHosts[0].RetryPolicy = tc.vhPolicy
+			resources.Routes[0].VirtualHosts[0].Routes[0].GetRoute().RetryPolicy = tc.routePolicy
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			for {
+				serverMu.Lock()
+				ctr = 0
+				serverMu.Unlock()
+				_, err := client.EmptyCall(ctx, &testpb.Empty{})
+				if code := status.Code(err); code == tc.tryAgainErr {
+					continue
+				} else if code != tc.errWant {
+					t.Fatalf("rpc EmptyCall() = _, %v; want _, %v", err, tc.errWant)
+				}
+				break
+			}
+		})
+	}
+}

test/xds/xds_client_sni_test.go

@@ -0,0 +1,851 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// validSNI matches the wildcard SAN (*.test.example.com) in the test server
+// certificate.
+const validSNI = "x.test.example.com"
+
+// Tests the SNI and SAN validation logic by verifying that RPCs succeed when
+// AutoSNISANValidation is enabled and the SNI matches a server certificate DNS
+// SAN. Also verifies that RPCs fail with an 'Unavailable' status if the SNI is
+// present but does not match any DNS SAN in the certificate.
+func (s) TestClientSideXDS_SNISANValidation(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends for two clusters:
+	// - backend1 configured with TLS creds, represents cluster1 (valid SNI)
+	// - backend2 configured with TLS creds, represents cluster2 (invalid SNI)
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1 (valid SNI)
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2 (invalid SNI)
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// Configure cluster1 with valid SNI and AutoSniSanValidation set to true.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  validSNI,
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with invalid SNI and AutoSniSanValidation set to true.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  "wrong.sni.domain",
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC to cluster1 should succeed because auto_sni_san_validation is true
+	// and sni matches server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because even though auto_sni_san_validation
+	// is true, sni doesn't match server cert SAN.
+	const wantErr = "do not match the SNI"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when AutoHostSNI is enabled, the endpoint's hostname is used for
+// the ServerName even though SNI is specified. It verifies that the TLS
+// handshake and subsequent RPC succeed because the resolved SNI i.e. the
+// hostname matches the server's certificate SAN.
+func (s) TestClientSideXDS_AutoHostSNI(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backend
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointName = "endpoint-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	// Configure the cluster with AutoHostSni and AutoSniSanValidation set to
+	// true with a invalid SNI.
+	cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS)
+	cluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoHostSni:          true,
+				AutoSniSanValidation: true,
+				Sni:                  "wrong.sni.domain",
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Endpoints configuring Hostname to the defaultTestCertSAN to verify
+	// AutoHostSni usage.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{{
+				Weight: 1,
+				Backends: []e2e.BackendOptions{{
+					Ports:    []uint32{testutils.ParsePort(t, server.Address)},
+					Hostname: validSNI,
+				}},
+			}},
+		}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC should succeed because auto_host_sni sets SNI from the endpoint
+	// hostname and auto_sni_san_validation validates that the SNI matches
+	// server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+}
+
+// Tests the scenario where multiple endpoints are configured with distinct
+// hostnames. It verifies that the client uses the specific hostname associated
+// with each endpoint during the TLS handshake by simulating an EDS failover and
+// confirming that the SNI is dynamically updated to match the active endpoint.
+func (s) TestClientSideXDS_AutoHostSNI_MultipleEndpoints(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointName = "endpoint-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	clusters := []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS)}
+	clusters[0].TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoHostSni:          true,
+				AutoSniSanValidation: true,
+				Sni:                  "wrong.sni.domain",
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure only Priority 0 with an invalid hostname. RPC should fail due
+	// to SAN mismatch.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "locality-0",
+					Weight:   1,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server1.Address)},
+						Hostname: "wrong.sni.domain",
+					}},
+				},
+			},
+		}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC should fail because hostname for endpoints in Priority 0 is invalid.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatalf("EmptyCall() unexpectedly succeeded with invalid SNI and no failover")
+	} else if !strings.Contains(err.Error(), "authentication handshake failed") {
+		t.Fatalf("EmptyCall() failed with %v, want error containing 'authentication handshake failed'", err)
+	}
+
+	// Update the same EDS resource to add Priority 1 with an endpoint with
+	// valid hostname.
+	endpoints = []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "locality-0",
+					Weight:   1,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server1.Address)},
+						Hostname: "wrong.sni.domain",
+					}},
+				},
+				{
+					Name:     "locality-1",
+					Weight:   1,
+					Priority: 1,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server2.Address)},
+						Hostname: validSNI,
+					}},
+				},
+			},
+		}),
+	}
+	resources.Endpoints = endpoints
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// RPC should succeed because it automatically fails over to Priority 1. Id
+	// only one hostname would have been used per leaf cluster , this RPC should
+	// have failed due to invalid SNI in Priority 0.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo), grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed after adding Priority 1: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server2.Address; got != want {
+		t.Fatalf("EmptyCall() successes routed to %q, want %q", got, want)
+	}
+}
+
+// TestClientSideXDS_FallbackSANMatchers tests that when AutoSniSanValidation is
+// true, and no SNI is provided by the control plane and AutoHostSNI is not set,
+// the validation falls back to using the explicit SAN matchers specified in the
+// configuration. It verifies that RPCs succeed when the fallback matchers match
+// the server certificate SAN and fail when they do not.
+func (s) TestClientSideXDS_FallbackSANMatchers(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// Configure cluster1 with AutoSniSanValidation set to true and no SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which matches the server1's
+	// certificate SAN.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}},
+								},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with AutoSniSanValidation set to true and no SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which does not match the
+	// server2's certificate SAN.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC to cluster1 should succeed because fallback SAN mathchers are used
+	// and they match server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because fallback SAN matchers are used
+	// but they don't match server cert SAN.
+	const wantErr = "do not match any of the accepted SANs"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when the XDSSNIEnabled environment variable is set to false, SNI
+// is not used for validation even if AutoSniSanValidation is true and SNI is
+// set. It verifies that the system falls back to using explicit SAN matchers if
+// provided, and the TLS handshake succeeds when they match the server
+// certificate SAN.
+func (s) TestClientSideXDS_SNIEnvVarDisabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, false)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// cluster1 configuration with AutoSniSanValidation set to true and wrong SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which matches the server1's
+	// certificate SAN.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  "incorrect.sni",
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}},
+								},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with AutoSniSanValidation set to true and correct SNI
+	// provided for the handshake. The validation falls back to using the
+	// explicit SAN matchers specified in the configuration which does not match
+	// the server2's certificate SAN.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  validSNI,
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC to cluster1 should succeed because fallback SAN mathchers are used
+	// and they match server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because fallback SAN matchers are used but they
+	// don't match server cert SAN.
+	const wantErr = "do not match any of the accepted SANs"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when AutoHostSNI is enabled for a Logical DNS cluster, the SNI is
+// resolved from the DNSHostName in the cluster configuration. It verifies that
+// the TLS handshake succeeds when the DNSHostName matches the server's
+// certificate SAN.
+func (s) TestClientSideXDS_AutoHostSNI_LogicalDNS(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backend
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	// Replace DNS resolver with a manual resolver.
+	dnsR := manual.NewBuilderWithScheme("dns")
+	originalDNS := resolver.Get("dns")
+	resolver.Register(dnsR)
+	t.Cleanup(func() { resolver.Register(originalDNS) })
+
+	dnsR.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: server.Address}}}}})
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	// Cluster of Type LogicalDNS. with DNSHostName set to match the server's cert
+	// SAN.
+	cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		Type:          e2e.ClusterTypeLogicalDNS,
+		ClusterName:   clusterName,
+		DNSHostName:   validSNI,
+		DNSPort:       uint32(testutils.ParsePort(t, server.Address)),
+		SecurityLevel: e2e.SecurityLevelMTLS,
+	})
+	cluster.TransportSocket.ConfigType.(*v3corepb.TransportSocket_TypedConfig).TypedConfig = testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+		AutoHostSni:          true,
+		AutoSniSanValidation: true,
+		CommonTlsContext: &v3tlspb.CommonTlsContext{
+			ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+				ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+					InstanceName: e2e.ClientSideCertProviderInstance,
+				},
+			},
+			TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+				InstanceName: e2e.ClientSideCertProviderInstance,
+			},
+		},
+	})
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC should succeed because DNSHostName matches the server's certificate
+	// SAN.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_resolver_e2e_test.go

@@ -0,0 +1,754 @@
+/*
+*
+* Copyright 2026 gRPC authors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+
+	iresolver "google.golang.org/grpc/internal/resolver"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// wrappingBuilder wraps a resolver.Builder and intercepts the ClientConn
+// passed to Build by wrapping it with an interceptingClientConn.
+type wrappingBuilder struct {
+	resolver.Builder
+	jsonCh chan string
+}
+
+// Build wraps the provided resolver.ClientConn with an interceptingClientConn
+// before delegating to the underlying resolver.Builder.
+func (ib *wrappingBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+	icc := &interceptingClientConn{
+		ClientConn: cc,
+		jsonCh:     ib.jsonCh,
+	}
+	return ib.Builder.Build(target, icc, opts)
+}
+
+// interceptingClientConn wraps a resolver.ClientConn and intercepts calls to
+// ParseServiceConfig to send the raw JSON service config.
+type interceptingClientConn struct {
+	resolver.ClientConn
+	jsonCh chan string
+}
+
+func (icc *interceptingClientConn) ParseServiceConfig(js string) *serviceconfig.ParseResult {
+	select {
+	case icc.jsonCh <- js:
+	default:
+	}
+	return icc.ClientConn.ParseServiceConfig(js)
+}
+
+type dummyFilterCfg struct {
+	httpfilter.FilterConfig
+}
+
+type testFilterBuilder struct {
+	httpfilter.Builder
+	typeURL      string
+	blockChan    chan struct{}
+	enteredChan  chan struct{}
+	closeChan    chan struct{}
+	newStreamErr error
+}
+
+func (tb *testFilterBuilder) TypeURLs() []string { return []string{tb.typeURL} }
+
+func (*testFilterBuilder) ParseFilterConfig(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return dummyFilterCfg{}, nil
+}
+
+func (*testFilterBuilder) ParseFilterConfigOverride(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return dummyFilterCfg{}, nil
+}
+
+func (*testFilterBuilder) IsTerminal() bool { return false }
+
+func (tb *testFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter {
+	return tb
+}
+
+func (*testFilterBuilder) Close() {}
+
+// ClientFilterBuilder is an optional interface for filters to implement. This
+// compile time check ensures the test filter implements it.
+var _ httpfilter.ClientFilterBuilder = &testFilterBuilder{}
+
+func (tb *testFilterBuilder) BuildClientInterceptor(httpfilter.FilterConfig, httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) {
+	return &testInterceptor{
+		blockChan:    tb.blockChan,
+		enteredChan:  tb.enteredChan,
+		closeChan:    tb.closeChan,
+		newStreamErr: tb.newStreamErr,
+	}, nil
+}
+
+type testInterceptor struct {
+	blockChan    chan struct{}
+	enteredChan  chan struct{}
+	closeChan    chan struct{}
+	newStreamErr error
+}
+
+func (i *testInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	// Signal that we have entered the filter
+	i.enteredChan <- struct{}{}
+
+	select {
+	case <-i.blockChan:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	if i.newStreamErr != nil {
+		return nil, i.newStreamErr
+	}
+	return newStream(ctx, opts...)
+}
+
+func (i *testInterceptor) Close() {
+	i.closeChan <- struct{}{}
+}
+
+func newTestFilterBuilder(t testing.TB, typeURL string) *testFilterBuilder {
+	tb := &testFilterBuilder{
+		typeURL:     typeURL,
+		blockChan:   make(chan struct{}),
+		enteredChan: make(chan struct{}, 1),
+		closeChan:   make(chan struct{}, 3),
+	}
+	httpfilter.Register(tb)
+	t.Cleanup(func() {
+		httpfilter.UnregisterForTesting(typeURL)
+	})
+	return tb
+}
+
+// wantServiceConfig returns a JSON representation of a service config with
+// xds_cluster_manager_experimental LB policy with child policies of
+// cds_experimental for the provided cluster names.
+func wantServiceConfig(clusters ...string) string {
+	var children []string
+	for _, cluster := range clusters {
+		children = append(children, fmt.Sprintf(`"cluster:%s": {
+			"childPolicy": [{
+				"cds_experimental": {
+					"cluster": "%s"
+				}
+			}]
+		}`, cluster, cluster))
+	}
+	return fmt.Sprintf(`{
+		"loadBalancingConfig": [{
+			"xds_cluster_manager_experimental": {
+				"children": {
+					%s
+				}
+			}
+		}]
+	}`, strings.Join(children, ","))
+}
+
+// compareJSONConfigs parses gotJSON and wantJSON using the internal gRPC
+// Service Config parser and asserts that they represent the same configuration.
+func compareJSONConfigs(t *testing.T, gotJSON, wantJSON string) {
+	gotParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(gotJSON)
+	if gotParsed.Err != nil {
+		t.Fatalf("Failed to parse got service config %q: %v", gotJSON, gotParsed.Err)
+	}
+	wantParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(wantJSON)
+	if wantParsed.Err != nil {
+		t.Fatalf("Failed to parse want service config %q: %v", wantJSON, wantParsed.Err)
+	}
+	if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
+		t.Fatalf("Service config mismatch.\nGot:\n%s\nWant:\n%s", gotJSON, wantJSON)
+	}
+}
+
+func verifyServiceConfig(ctx context.Context, t *testing.T, jsonCh chan string, wantClusters ...string) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for resolver state update")
+	case js := <-jsonCh:
+		compareJSONConfigs(t, js, wantServiceConfig(wantClusters...))
+	}
+}
+
+// Test verifies that when RPCs are in flight holding references to an old
+// cluster, that cluster remains in the service config until all in-flight
+// RPCs finish.
+func (s) TestResolverDelayedClusterRemoval_MultipleInFlightRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	blockChan, enteredChan := tb.blockChan, tb.enteredChan
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create the xDS resolver builder.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Create an intercepting resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "delaying-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a gRPC client using the intercepting resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Trigger first RPC to cluster A. This RPC should reach our custom HTTP
+	// filter, and get blocked.
+	errCh := make(chan error, 2)
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	// Verify that first RPC has entered the HTTP filter's NewStream and is
+	// currently blocked.
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for first RPC to reach HTTP filter")
+	}
+
+	// Read the first state update from the intercepting resolver.
+	// This should contain cluster-A.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Trigger second RPC to cluster A. This RPC should reach our custom HTTP
+	// filter, and get blocked.
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	// Verify that second RPC has entered the HTTP filter's NewStream and is
+	// currently blocked.
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for second RPC to reach HTTP filter")
+	}
+
+	// Now update the route configuration on the management server to point to cluster B.
+	clusterSpec.Cluster = clusterB
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Unblock one of the RPCs and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for an RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Verify that because the other RPC to cluster-A is still in flight, the
+	// cluster reference count does not drop to 0 and no service config update
+	// is produced.
+	select {
+	case js := <-jsonCh:
+		t.Fatalf("Unexpected service config update received: %s", js)
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the remaining RPC and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for remaining RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("Remaining RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Once the second RPC finishes (refCount drops 1 -> 0), cluster-A should be
+	// removed from the service config.
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// Test verifies that if stream creation fails early inside an HTTP filter or
+// interceptor (before the stream is created), the OnFinish CallOption is still
+// executed. This decrements the cluster reference count to 0, allowing the
+// resolver to unsubscribe and prune the old cluster from the service config.
+func (s) TestResolverPrunesCluster_StreamCreationFailure(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	const wantErr = "blocking filter error"
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	tb.newStreamErr = status.Error(codes.Unavailable, wantErr)
+	close(tb.blockChan) // Close blockChan immediately so that the RPC fails without blocking
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create the underlying xDS resolver builder.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "blocking-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a gRPC client using the xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Read the first state update from the intercepting resolver.
+	// This should contain cluster-A.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Make an RPC. Since the filter immediately fails, this should fail with
+	// the expected error.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("RPC failed with error %v, want code %v with desc containing %q", err, codes.Unavailable, wantErr)
+	}
+
+	// Update the route configuration on the management server to point
+	// to cluster B.
+	clusterSpec.Cluster = clusterB
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read the second state update from the intercepting resolver. This should
+	// contain both cluster-A and cluster-B.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Read the third state update from the intercepting resolver. Cluster-A's
+	// reference count drops to 0, causing the resolver to unsubscribe and
+	// prune it.
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// TestResolverDelayedInterceptorClose_ActiveRPCs verifies that when a route
+// configuration is updated (causing the old config selector to stop), an HTTP
+// filter interceptor for a route/cluster with an active in-flight RPC is not
+// closed immediately. It remains active until all active RPCs to that cluster
+// in that route complete and its refcount drops to 0.
+func (s) TestResolverDelayedInterceptorClose_ActiveRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	blockChan, enteredChan, closeChan := tb.blockChan, tb.enteredChan, tb.closeChan
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "delaying-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for RPC to reach HTTP filter")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Update route configuration on the management server to point to cluster B.
+	hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}}
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Verify that because an RPC is still in flight on cluster-A, the interceptor
+	// is NOT closed prematurely when the old config selector is stopped.
+	select {
+	case <-closeChan:
+		t.Fatal("Unexpected interceptor Close() called while RPC is still active")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the active RPC and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Once the RPC finishes, its OnFinish CallOption decrements the refcount
+	// of cluster-A to 0. Verifies that interceptor.Close() is now called.
+	select {
+	case <-closeChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for interceptor Close() after RPC finished")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// Test verifies that when a route configuration is updated and there are no
+// active in-flight RPCs on a route/cluster, the HTTP filter interceptor is
+// closed immediately when the old config selector is stopped.
+func (s) TestResolverImmediateInterceptorClose_NoActiveRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	closeChan := tb.closeChan
+	close(tb.blockChan) // Do not block RPCs in the filter
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "closing-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Update route configuration on the management server to point to cluster B.
+	hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}}
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Because there are no active RPCs on cluster-A, interceptor.Close() should
+	// be invoked immediately when the old config selector is stopped.
+	select {
+	case <-closeChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for interceptor Close() when no active RPCs exist")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}

test/xds/xds_rls_clusterspecifier_plugin_test.go

@@ -0,0 +1,168 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/rls"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/balancer/rls" // Register the RLS Load Balancing policy.
+)
+
+// defaultClientResourcesWithRLSCSP returns a set of resources (LDS, RDS, CDS, EDS) for a
+// client to connect to a server with a RLS Load Balancer as a child of Cluster Manager.
+func defaultClientResourcesWithRLSCSP(t *testing.T, lb e2e.LoadBalancingPolicy, params e2e.ResourceParams, rlsProto *rlspb.RouteLookupConfig) e2e.UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
+	return e2e.UpdateOptions{
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{
+			RouteConfigName:            routeConfigName,
+			ListenerName:               params.DialTarget,
+			ClusterSpecifierType:       e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin,
+			ClusterSpecifierPluginName: "rls-csp",
+			ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &rlspb.RouteLookupClusterSpecifier{
+				RouteLookupConfig: rlsProto,
+			}),
+		})},
+		Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+			ClusterName:   clusterName,
+			ServiceName:   endpointsName,
+			Policy:        lb,
+			SecurityLevel: params.SecLevel,
+		})},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})},
+	}
+}
+
+// TestRLSinxDS tests an xDS configured system with an RLS Balancer present.
+//
+// This test sets up the RLS Balancer using the RLS Cluster Specifier Plugin,
+// spins up a test service and has a fake RLS Server correctly respond with a
+// target corresponding to this test service. This test asserts an RPC proceeds
+// as normal with the RLS Balancer as part of system.
+func (s) TestRLSinxDS(t *testing.T) {
+	tests := []struct {
+		name     string
+		lbPolicy e2e.LoadBalancingPolicy
+	}{
+		{
+			name:     "roundrobin",
+			lbPolicy: e2e.LoadBalancingPolicyRoundRobin,
+		},
+		{
+			name:     "ringhash",
+			lbPolicy: e2e.LoadBalancingPolicyRingHash,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testRLSinxDS(t, test.lbPolicy)
+		})
+	}
+}
+
+func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) {
+	// Set up all components and configuration necessary - management server,
+	// xDS resolver, fake RLS Server, and xDS configuration which specifies an
+	// RLS Balancer that communicates to this set up fake RLS Server.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	lis := testutils.NewListenerWrapper(t, nil)
+	rlsServer, rlsRequestCh := rls.SetupFakeRLSServer(t, lis)
+	rlsProto := &rlspb.RouteLookupConfig{
+		GrpcKeybuilders:      []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}},
+		LookupService:        rlsServer.Address,
+		LookupServiceTimeout: durationpb.New(defaultTestTimeout),
+		CacheSizeBytes:       1024,
+	}
+
+	const serviceName = "my-service-client-side-xds"
+	resources := defaultClientResourcesWithRLSCSP(t, lbPolicy, e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	}, rlsProto)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the fake RLS Server to set the RLS Balancers child CDS
+	// Cluster's name as the target for the RPC to use.
+	rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rls.RouteLookupResponse {
+		return &rls.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{"cluster-" + serviceName}}}
+	})
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// Successfully sending the RPC will require the RLS Load Balancer to
+	// communicate with the fake RLS Server for information about the target.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// These RLS Verifications makes sure the RLS Load Balancer is actually part
+	// of the xDS Configured system that correctly sends out RPC.
+
+	// Verify connection is established to RLS Server.
+	if _, err = lis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for RLS LB policy to create control channel")
+	}
+
+	// Verify an rls request is sent out to fake RLS Server.
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for an RLS request to be sent out")
+	case <-rlsRequestCh:
+	}
+}

test/xds/xds_security_config_nack_test.go

@@ -0,0 +1,377 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/google/uuid"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) {
+	const (
+		serviceName                     = "my-service-client-side-xds"
+		missingIdentityProviderInstance = "missing-identity-provider-instance"
+		missingRootProviderInstance     = "missing-root-provider-instance"
+	)
+
+	tests := []struct {
+		name           string
+		securityConfig *v3corepb.TransportSocket
+		wantErr        bool
+	}{
+		{
+			name: "both identity and root providers are not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only identity provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ServerSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only root provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ServerSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "both identity and root providers are present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ServerSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ServerSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelMTLS,
+			})
+
+			// Create an inbound xDS listener resource for the server side.
+			inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")
+			for _, fc := range inboundLis.GetFilterChains() {
+				fc.TransportSocket = test.securityConfig
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			// Make a context with a shorter timeout from the top level test
+			// context for cases where we expect failures.
+			timeout := defaultTestTimeout
+			if test.wantErr {
+				timeout = defaultTestShortTimeout
+			}
+			ctx, cancel = context.WithTimeout(ctx, timeout)
+			defer cancel()
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr {
+				t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) {
+	const (
+		serviceName                     = "my-service-client-side-xds"
+		missingIdentityProviderInstance = "missing-identity-provider-instance"
+		missingRootProviderInstance     = "missing-root-provider-instance"
+	)
+
+	tests := []struct {
+		name           string
+		securityConfig *v3corepb.TransportSocket
+		wantErr        bool
+	}{
+		{
+			name: "both identity and root providers are not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only identity provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only root provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "both identity and root providers are present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+			// Create bootstrap configuration pointing to the above management
+			// server with certificate provider configuration.
+			nodeID := uuid.New().String()
+			bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+			// Create an xDS resolver with the above bootstrap configuration.
+			xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+			if err != nil {
+				t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+			}
+
+			server := stubserver.StartTestService(t, nil)
+			defer server.Stop()
+
+			// This creates a `Cluster` resource with a security config which
+			// refers to `e2e.ClientSideCertProviderInstance` for both root and
+			// identity certs.
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       "localhost",
+				Port:       testutils.ParsePort(t, server.Address),
+				SecLevel:   e2e.SecurityLevelMTLS,
+			})
+			resources.Clusters[0].TransportSocket = test.securityConfig
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			// Make a context with a shorter timeout from the top level test
+			// context for cases where we expect failures.
+			timeout := defaultTestTimeout
+			if test.wantErr {
+				timeout = defaultTestShortTimeout
+			}
+			ctx2, cancel2 := context.WithTimeout(ctx, timeout)
+			defer cancel2()
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx2, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr {
+				t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}

test/xds/xds_server_filter_e2e_test.go

@@ -0,0 +1,402 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func makeUnaryRPC(ctx context.Context, client testgrpc.TestServiceClient) error {
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("EmptyCall() failed: %w", err)
+	}
+	return nil
+}
+
+func makeStreamingRPC(ctx context.Context, client testgrpc.TestServiceClient) error {
+	stream, err := client.FullDuplexCall(ctx)
+	if err != nil {
+		return fmt.Errorf("FullDuplexCall() failed: %w", err)
+	}
+	if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+		return fmt.Errorf("stream.Send() failed: %w", err)
+	}
+	if err := stream.CloseSend(); err != nil {
+		return fmt.Errorf("stream.CloseSend() failed: %w", err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		return err
+	}
+	return nil
+}
+
+// setupXDSListenerAndClient starts an xDS management server and a gRPC server,
+// configures the management server with an inbound listener containing the
+// specified httpFilters, dials the server using xDS, waits for the server to
+// enter SERVING mode, and returns a TestServiceClient.
+func setupXDSListenerAndClient(t *testing.T, httpFilters ...*v3httppb.HttpFilter) testgrpc.TestServiceClient {
+	t.Helper()
+	const serviceName = "my-service"
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	t.Cleanup(stopServer)
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("Failed to retrieve host and port of server: %v", err)
+	}
+
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+
+	filters := append([]*v3httppb.HttpFilter{}, httpFilters...)
+	filters = append(filters, e2e.HTTPFilter("router", &v3routerpb.Router{}))
+
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: filters,
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatalf("managementServer.Update() failed: %v", err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	return testgrpc.NewTestServiceClient(cc)
+}
+
+// Test verifies that the xDS server-side HTTP filter chain correctly executes
+// InterceptRPC and wraps grpc.ServerStream for both Unary and Streaming RPCs.
+func (s) TestServerSideXDSHTTPFilter_InterceptRPC(t *testing.T) {
+	dummyTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(dummyTypeURL)
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	dummyFilter := newHTTPFilter(t, "dummy-filter", dummyTypeURL, "")
+	client := setupXDSListenerAndClient(t, dummyFilter)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC execution.
+	if err := makeUnaryRPC(ctx, client); err != nil {
+		t.Fatalf("makeUnaryRPC() failed: %v", err)
+	}
+	if got := fb.interceptRPCCount.Load(); got != 1 {
+		t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got)
+	}
+	if got := fb.recvMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected recvMsgCount for unary RPC, got %d, want 1", got)
+	}
+	if got := fb.sendMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected sendMsgCount for unary RPC, got %d, want 1", got)
+	}
+
+	// Verify Streaming RPC execution.
+	if err := makeStreamingRPC(ctx, client); err != nil {
+		t.Fatalf("makeStreamingRPC() failed: %v", err)
+	}
+	if got := fb.interceptRPCCount.Load(); got != 2 {
+		t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got)
+	}
+	if got := fb.recvMsgCount.Load(); got != 2 {
+		t.Fatalf("Unexpected recvMsgCount for streaming RPC, got %d, want 2", got)
+	}
+	// Since the server handler did not call SendMsg for the streaming RPC, the
+	// sendMsg count for this RPC is zero, so the total count remains 1.
+	if got := fb.sendMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected sendMsgCount for streaming RPC, got %d, want 1", got)
+	}
+}
+
+// Test verifies that when a server-side xDS HTTP filter returns an error from
+// InterceptRPC, the RPC is rejected early with the expected error code and
+// message for both Unary and Streaming RPCs.
+func (s) TestServerSideXDSHTTPFilter_InterceptRPCEarlyRejection(t *testing.T) {
+	tests := []struct {
+		name       string
+		errToRet   error
+		wantCode   codes.Code
+		wantErrMsg string
+	}{
+		{
+			name:       "InterceptRPC_FailsWithStatusError",
+			errToRet:   status.Error(codes.PermissionDenied, "access denied by xDS server filter"),
+			wantCode:   codes.PermissionDenied,
+			wantErrMsg: "access denied by xDS server filter",
+		},
+		{
+			name:       "InterceptRPC_FailsWithNonStatusError",
+			errToRet:   errors.New("non-status error from xDS filter"),
+			wantCode:   codes.Unknown,
+			wantErrMsg: "non-status error from xDS filter",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			typeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(typeURL)
+			fb.interceptRPCFunc = func(grpc.ServerStream) (grpc.ServerStream, error) {
+				return nil, test.errToRet
+			}
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			rejectFilter := newHTTPFilter(t, "reject-filter", typeURL, "")
+			client := setupXDSListenerAndClient(t, rejectFilter)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			// Verify Unary RPC rejection.
+			if err := makeUnaryRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) {
+				t.Fatalf("makeUnaryRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg)
+			}
+			if got := fb.interceptRPCCount.Load(); got != 1 {
+				t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got)
+			}
+
+			// Verify Streaming RPC rejection.
+			if err := makeStreamingRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) {
+				t.Fatalf("makeStreamingRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg)
+			}
+			if got := fb.interceptRPCCount.Load(); got != 2 {
+				t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got)
+			}
+		})
+	}
+}
+
+// Test verifies that multiple server-side xDS HTTP filters in a chain execute
+// their InterceptRPC methods sequentially in the configured order for both
+// Unary and Streaming RPCs.
+func (s) TestServerSideXDS_InterceptRPCMultiFilterChaining(t *testing.T) {
+	typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name())
+	eventsCh := make(chan string, 2)
+	const filter1, filter2 = "intercepted-filter-1", "intercepted-filter-2"
+
+	fb1 := newTrackingHTTPFilterBuilder(typeURL1)
+	fb1.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		eventsCh <- filter1
+		return ss, nil
+	}
+	fb2 := newTrackingHTTPFilterBuilder(typeURL2)
+	fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		eventsCh <- filter2
+		return ss, nil
+	}
+
+	httpfilter.Register(fb1)
+	defer httpfilter.UnregisterForTesting(fb1.typeURL)
+	httpfilter.Register(fb2)
+	defer httpfilter.UnregisterForTesting(fb2.typeURL)
+
+	f1 := newHTTPFilter(t, "filter-1", typeURL1, "")
+	f2 := newHTTPFilter(t, "filter-2", typeURL2, "")
+
+	client := setupXDSListenerAndClient(t, f1, f2)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC filter chaining.
+	if err := makeUnaryRPC(ctx, client); err != nil {
+		t.Fatalf("makeUnaryRPC() failed: %v", err)
+	}
+	select {
+	case e := <-eventsCh:
+		if e != filter1 {
+			t.Fatalf("First intercept event = %q, want %q", e, filter1)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter1)
+	}
+
+	select {
+	case e := <-eventsCh:
+		if e != filter2 {
+			t.Fatalf("Second intercept event = %q, want %q", e, filter2)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter2)
+	}
+
+	// Verify Streaming RPC filter chaining.
+	if err := makeStreamingRPC(ctx, client); err != nil {
+		t.Fatalf("makeStreamingRPC() failed: %v", err)
+	}
+	select {
+	case e := <-eventsCh:
+		if e != filter1 {
+			t.Fatalf("First intercept event = %q, want %q", e, filter1)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter1)
+	}
+
+	select {
+	case e := <-eventsCh:
+		if e != filter2 {
+			t.Fatalf("Second intercept event = %q, want %q", e, filter2)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter2)
+	}
+}
+
+type errorWrappedStream struct {
+	grpc.ServerStream
+	recvErr string
+}
+
+func (w *errorWrappedStream) RecvMsg(m any) error {
+	if w.recvErr != "" {
+		return status.Error(codes.Internal, w.recvErr)
+	}
+	return w.ServerStream.RecvMsg(m)
+}
+
+// Test verifies that when multiple filters wrap the server stream and a
+// filter's overridden RecvMsg returns an error, that error is properly
+// propagated back to the client for both Unary and Streaming RPCs.
+func (s) TestServerSideXDS_InterceptRPCMultiFilterErrorPropagation(t *testing.T) {
+	typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name())
+
+	fb1 := newTrackingHTTPFilterBuilder(typeURL1)
+
+	const wantErr = "recvMsg failed in filter 2"
+	fb2 := newTrackingHTTPFilterBuilder(typeURL2)
+	fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		return &errorWrappedStream{ServerStream: ss, recvErr: wantErr}, nil
+	}
+
+	httpfilter.Register(fb1)
+	defer httpfilter.UnregisterForTesting(fb1.typeURL)
+	httpfilter.Register(fb2)
+	defer httpfilter.UnregisterForTesting(fb2.typeURL)
+
+	filter1 := newHTTPFilter(t, "filter-1", typeURL1, "")
+	filter2 := newHTTPFilter(t, "filter-2", typeURL2, "")
+
+	client := setupXDSListenerAndClient(t, filter1, filter2)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC error propagation.
+	if err := makeUnaryRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("makeUnaryRPC() returned error = %v, want error containing %q", err, wantErr)
+	}
+
+	// Verify Streaming RPC error propagation.
+	if err := makeStreamingRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("makeStreamingRPC() returned error = %v, want error containing %q", err, wantErr)
+	}
+}

test/xds/xds_server_filter_override_test.go

@@ -0,0 +1,393 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/xds"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+)
+
+type overrideInfo struct {
+	disabled bool
+	path     string
+}
+
+func makeOverrideConfig(t *testing.T, typeURL string, info *overrideInfo) *anypb.Any {
+	if info == nil {
+		return nil
+	}
+	cfg := &v3routepb.FilterConfig{Disabled: info.disabled}
+	if info.path != "" {
+		cfg.Config = testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+			TypeUrl: typeURL,
+			Value: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					"path": {Kind: &structpb.Value_StringValue{StringValue: info.path}},
+				},
+			},
+		})
+	}
+	return testutils.MarshalAny(t, cfg)
+}
+
+// TestServerSideXDS_FilterOverride_Disabled verifies that filters are NOT
+// created when disabled by either base config or overrides at either
+// VirtualHost or Route level.
+func (s) TestServerSideXDS_FilterOverride_Disabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true)
+
+	tests := []struct {
+		name         string
+		baseDisabled bool
+		vhost        *overrideInfo
+		route        *overrideInfo
+	}{
+		{
+			name:         "BaseDisabled_NoOverrides",
+			baseDisabled: true,
+		},
+		{
+			name:         "BaseEnabled_VHostDisables",
+			baseDisabled: false,
+			vhost:        &overrideInfo{disabled: true},
+		},
+		{
+			name:         "BaseEnabled_VHostEnabled_RouteDisables",
+			baseDisabled: false,
+			vhost:        &overrideInfo{disabled: false, path: "vhost-value"},
+			route:        &overrideInfo{disabled: true},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			testFilterTypeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			// Wait for the server to enter SERVING mode before making RPCs to avoid
+			// flakes due to the server closing connections.
+			servingCh := make(chan struct{})
+			opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+				if args.Mode == connectivity.ServingModeServing {
+					close(servingCh)
+				}
+			})
+			lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+			defer stopServer()
+
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("Failed to retrieve host and port of server: %v", err)
+			}
+			const serviceName = "my-service"
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelNone,
+			})
+
+			baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, "base-path")
+			baseFilter.Disabled = tc.baseDisabled
+
+			var vhOverrideConfig map[string]*anypb.Any
+			if tc.vhost != nil {
+				vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)}
+			}
+			var routeOverrideConfig map[string]*anypb.Any
+			if tc.route != nil {
+				routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)}
+			}
+
+			vhs := []*v3routepb.VirtualHost{{
+				Domains: []string{"*"},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action:               &v3routepb.Route_NonForwardingAction{},
+					TypedPerFilterConfig: routeOverrideConfig,
+				}},
+				TypedPerFilterConfig: vhOverrideConfig,
+			}}
+
+			networkFilters := []*v3listenerpb.Filter{{
+				Name: "hcm",
+				ConfigType: &v3listenerpb.Filter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+						HttpFilters: []*v3httppb.HttpFilter{
+							baseFilter,
+							e2e.HTTPFilter("router", &v3routerpb.Router{}),
+						},
+						RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+							RouteConfig: &v3routepb.RouteConfiguration{
+								Name:         "routeName",
+								VirtualHosts: vhs,
+							},
+						},
+					}),
+				},
+			}}
+
+			inboundLis := &v3listenerpb.Listener{
+				Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+				Address: &v3corepb.Address{
+					Address: &v3corepb.Address_SocketAddress{
+						SocketAddress: &v3corepb.SocketAddress{
+							Address: host,
+							PortSpecifier: &v3corepb.SocketAddress_PortValue{
+								PortValue: port,
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			select {
+			case <-servingCh:
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for server to enter SERVING mode")
+			}
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall() failed: %v", err)
+			}
+
+			// Verify no filters or interceptors were created.
+			if got := fb.filtersCreated.Load(); got != 0 {
+				t.Fatalf("filtersCreated = %d; want 0", got)
+			}
+			if got := fb.interceptorsCreated.Load(); got != 0 {
+				t.Fatalf("interceptorsCreated = %d; want 0", got)
+			}
+		})
+	}
+}
+
+// TestServerSideXDS_FilterOverride_Enabled verifies that filters ARE created
+// and receive the correct configuration when enabled by either base config or
+// overrides at either VirtualHost or Route level.
+func (s) TestServerSideXDS_FilterOverride_Enabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true)
+
+	const basePath = "base-path"
+
+	tests := []struct {
+		name         string
+		baseDisabled bool
+		vhost        *overrideInfo
+		route        *overrideInfo
+		wantPath     string
+	}{
+		{
+			name:         "BaseEnabled_NoOverrides",
+			baseDisabled: false,
+			wantPath:     basePath,
+		},
+		{
+			name:         "VHostEnables",
+			baseDisabled: true,
+			vhost:        &overrideInfo{disabled: false, path: "vhost-value"},
+			wantPath:     "vhost-value",
+		},
+		{
+			name:         "RouteEnables",
+			baseDisabled: true,
+			vhost:        &overrideInfo{disabled: true},
+			route:        &overrideInfo{disabled: false, path: "route-value"},
+			wantPath:     "route-value",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			pathCh := make(chan string, 1)
+			testFilterTypeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+			fb.pathCh = pathCh
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			servingCh := make(chan struct{})
+			opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+				if args.Mode == connectivity.ServingModeServing {
+					close(servingCh)
+				}
+			})
+			lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+			defer stopServer()
+
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("Failed to retrieve host and port of server: %v", err)
+			}
+			const serviceName = "my-service"
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelNone,
+			})
+
+			baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, basePath)
+			baseFilter.Disabled = tc.baseDisabled
+
+			var vhOverrideConfig map[string]*anypb.Any
+			if tc.vhost != nil {
+				vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)}
+			}
+			var routeOverrideConfig map[string]*anypb.Any
+			if tc.route != nil {
+				routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)}
+			}
+
+			vhs := []*v3routepb.VirtualHost{{
+				Domains: []string{"*"},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action:               &v3routepb.Route_NonForwardingAction{},
+					TypedPerFilterConfig: routeOverrideConfig,
+				}},
+				TypedPerFilterConfig: vhOverrideConfig,
+			}}
+
+			networkFilters := []*v3listenerpb.Filter{{
+				Name: "hcm",
+				ConfigType: &v3listenerpb.Filter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+						HttpFilters: []*v3httppb.HttpFilter{
+							baseFilter,
+							e2e.HTTPFilter("router", &v3routerpb.Router{}),
+						},
+						RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+							RouteConfig: &v3routepb.RouteConfiguration{
+								Name:         "routeName",
+								VirtualHosts: vhs,
+							},
+						},
+					}),
+				},
+			}}
+
+			inboundLis := &v3listenerpb.Listener{
+				Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+				Address: &v3corepb.Address{
+					Address: &v3corepb.Address_SocketAddress{
+						SocketAddress: &v3corepb.SocketAddress{
+							Address: host,
+							PortSpecifier: &v3corepb.SocketAddress_PortValue{
+								PortValue: port,
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			select {
+			case <-servingCh:
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for server to enter SERVING mode")
+			}
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall() failed: %v", err)
+			}
+
+			// Verify invoked with expected path.
+			select {
+			case p := <-pathCh:
+				if p != tc.wantPath {
+					t.Fatalf("Unexpected path received by filter: got %q, want %q", p, tc.wantPath)
+				}
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for filter to be invoked")
+			}
+			if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+				t.Fatalf("Created %d filter instances, want: %d", got, want)
+			}
+			if got, want := fb.interceptorsCreated.Load(), int32(1); got != want {
+				t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+			}
+		})
+	}
+}

test/xds/xds_server_filter_state_retention_test.go

@@ -0,0 +1,699 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/xds"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const filterCfgPathFieldName = "path"
+
+// testFilterCfg is the internal representation of the filter config proto. It
+// is returned by filter's config parsing methods.
+type testFilterCfg struct {
+	httpfilter.FilterConfig
+	path string
+}
+
+// filterConfigFromProto parses filter config specified as a v3.TypedStruct into
+// a testFilterCfg.
+func filterConfigFromProto(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	ts, ok := cfg.(*v3xdsxdstypepb.TypedStruct)
+	if !ok {
+		return nil, fmt.Errorf("unsupported filter config type: %T, want %T", cfg, &v3xdsxdstypepb.TypedStruct{})
+	}
+
+	if ts.GetValue() == nil {
+		return testFilterCfg{}, nil
+	}
+	ret := testFilterCfg{}
+	if v := ts.GetValue().GetFields()[filterCfgPathFieldName]; v != nil {
+		ret.path = v.GetStringValue()
+	}
+	return ret, nil
+}
+
+// trackingHTTPFilterBuilder is a test filter that allows counting and tracking
+// the lifecycle and execution of filter and interceptor instances.
+type trackingHTTPFilterBuilder struct {
+	httpfilter.Builder
+	filtersCreated        atomic.Int32
+	filtersDestroyed      atomic.Int32
+	interceptorsCreated   atomic.Int32
+	interceptorsDestroyed atomic.Int32
+	interceptRPCCount     atomic.Int32
+	recvMsgCount          atomic.Int32
+	sendMsgCount          atomic.Int32
+	closeSendCount        atomic.Int32
+	typeURL               string
+	pathCh                chan string
+	interceptRPCFunc      func(ss grpc.ServerStream) (grpc.ServerStream, error)
+}
+
+func newTrackingHTTPFilterBuilder(typeURL string) *trackingHTTPFilterBuilder {
+	return &trackingHTTPFilterBuilder{typeURL: typeURL}
+}
+
+func (t *trackingHTTPFilterBuilder) IsTerminal() bool { return false }
+
+func (t *trackingHTTPFilterBuilder) TypeURLs() []string { return []string{t.typeURL} }
+
+func (*trackingHTTPFilterBuilder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return filterConfigFromProto(cfg)
+}
+
+func (*trackingHTTPFilterBuilder) ParseFilterConfigOverride(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return filterConfigFromProto(cfg)
+}
+
+func (t *trackingHTTPFilterBuilder) BuildServerFilter() httpfilter.ServerFilter {
+	t.filtersCreated.Add(1)
+	return t
+}
+
+func (t *trackingHTTPFilterBuilder) Close() {
+	t.filtersDestroyed.Add(1)
+}
+
+var _ httpfilter.ClientFilterBuilder = &trackingHTTPFilterBuilder{}
+var _ httpfilter.ServerFilterBuilder = &trackingHTTPFilterBuilder{}
+
+func (t *trackingHTTPFilterBuilder) BuildServerInterceptor(config, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) {
+	t.interceptorsCreated.Add(1)
+
+	var effectiveCfg testFilterCfg
+	if override != nil {
+		effectiveCfg = override.(testFilterCfg)
+	} else if config != nil {
+		effectiveCfg = config.(testFilterCfg)
+	} else {
+		return nil, fmt.Errorf("unexpected missing config")
+	}
+
+	interceptor := &trackingServerInterceptor{
+		parent:   t,
+		pathCh:   t.pathCh,
+		basePath: effectiveCfg.path,
+	}
+	return interceptor, nil
+}
+
+type trackingServerInterceptor struct {
+	parent   *trackingHTTPFilterBuilder
+	pathCh   chan string
+	basePath string
+}
+
+func (i *trackingServerInterceptor) InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) {
+	i.parent.interceptRPCCount.Add(1)
+	if i.pathCh != nil {
+		i.pathCh <- i.basePath
+	}
+	if i.parent.interceptRPCFunc != nil {
+		return i.parent.interceptRPCFunc(ss)
+	}
+	return &wrappedServerStream{
+		ServerStream: ss,
+		parent:       i.parent,
+	}, nil
+}
+
+func (i *trackingServerInterceptor) Close() {
+	i.parent.interceptorsDestroyed.Add(1)
+}
+
+// wrappedServerStream wraps grpc.ServerStream to intercept and count
+// RecvMsg and SendMsg invocations for testing.
+type wrappedServerStream struct {
+	grpc.ServerStream
+	parent *trackingHTTPFilterBuilder
+}
+
+func (w *wrappedServerStream) RecvMsg(m any) error {
+	if err := w.ServerStream.RecvMsg(m); err != nil {
+		return err
+	}
+	w.parent.recvMsgCount.Add(1)
+	return nil
+}
+
+func (w *wrappedServerStream) SendMsg(m any) error {
+	if err := w.ServerStream.SendMsg(m); err != nil {
+		return err
+	}
+	w.parent.sendMsgCount.Add(1)
+	return nil
+}
+
+func newHTTPFilter(t *testing.T, name, typeURL, path string) *v3httppb.HttpFilter {
+	return &v3httppb.HttpFilter{
+		Name: name,
+		ConfigType: &v3httppb.HttpFilter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+				TypeUrl: typeURL,
+				Value: &structpb.Struct{
+					Fields: map[string]*structpb.Value{
+						filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: path}},
+					},
+				},
+			}),
+		},
+	}
+}
+
+func (t *trackingHTTPFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter {
+	t.filtersCreated.Add(1)
+	return t
+}
+
+func (t *trackingHTTPFilterBuilder) BuildClientInterceptor(_, _ httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) {
+	t.interceptorsCreated.Add(1)
+	return &trackingClientInterceptor{parent: t}, nil
+}
+
+type trackingClientInterceptor struct {
+	parent *trackingHTTPFilterBuilder
+}
+
+func (i *trackingClientInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	s, err := newStream(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &wrappedClientStream{
+		ClientStream: s,
+		parent:       i.parent,
+	}, nil
+}
+
+func (i *trackingClientInterceptor) Close() {
+	i.parent.interceptorsDestroyed.Add(1)
+}
+
+type wrappedClientStream struct {
+	grpc.ClientStream
+	parent *trackingHTTPFilterBuilder
+}
+
+func (w *wrappedClientStream) RecvMsg(m any) error {
+	w.parent.recvMsgCount.Add(1)
+	return w.ClientStream.RecvMsg(m)
+}
+
+func (w *wrappedClientStream) SendMsg(m any) error {
+	w.parent.sendMsgCount.Add(1)
+	return w.ClientStream.SendMsg(m)
+}
+
+func (w *wrappedClientStream) CloseSend() error {
+	w.parent.closeSendCount.Add(1)
+	return w.ClientStream.CloseSend()
+}
+
+// Tests the filter state retention behavior when filter configs in the existing
+// filter chain are updated, but there are no changes to the filter names. In
+// this case, the existing filter instance should be retained, while new
+// interceptor instances should be created with the updated config.
+func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterConfigChange(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "filter-1",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "v4-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+		},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Setup the management server with client and server-side resources.
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and two interceptors are created
+	// (one per filter chain).
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "initial-path"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Update the filter config in the listener resource.
+	networkFilters = []*v3listenerpb.Filter{{
+		Name: "filter-1",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "final-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis.FilterChains[0].Filters = networkFilters
+	inboundLis.FilterChains[1].Filters = networkFilters
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied on the gRPC server.
+WaitForUpdatedConfig:
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case cfg := <-pathCh:
+			if got, want := cfg, "final-path"; got == want {
+				break WaitForUpdatedConfig
+			}
+		case <-ctx.Done():
+			t.Fatalf("Timeout waiting for interceptor get updated config")
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the filter instance is retained, while the interceptor instances
+	// are replaced with the updated config.
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(4); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(2); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+
+	// Stop the server (which causes the listener to be closed) to trigger
+	// cleanup of filters and interceptors, and verify that all instances are
+	// cleaned up.
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}
+
+// Tests the filter state retention behavior when a new filter chain with a
+// filter of the existing type but different filter name is added to the
+// listener resource. In this case, a filter instance should be created because
+// the filter name is part of the key used to identify filter instances.
+func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterChainsChange(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Setup the management server with client and server-side resources.
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and one interceptor (for the
+	// default filter chain) is created.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "initial-path"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Add a new filter chain to the listener resource that contains HTTP
+	// filters with a different filter name. This should result in a new filter
+	// instance being created because the filter name is part of the key used to
+	// identify filter instances.
+	networkFilters = []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker-new", testFilterTypeURL, "final-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis.FilterChains = []*v3listenerpb.FilterChain{
+		{
+			Name: "v4-wildcard",
+			FilterChainMatch: &v3listenerpb.FilterChainMatch{
+				PrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "0.0.0.0",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+				SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				SourcePrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "0.0.0.0",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+			},
+			Filters: networkFilters,
+		},
+		{
+			Name: "v6-wildcard",
+			FilterChainMatch: &v3listenerpb.FilterChainMatch{
+				PrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "::",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+				SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				SourcePrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "::",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+			},
+			Filters: networkFilters,
+		},
+	}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied on the gRPC server.
+WaitForUpdatedConfig:
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case cfg := <-pathCh:
+			if got, want := cfg, "final-path"; got == want {
+				break WaitForUpdatedConfig
+			}
+		case <-ctx.Done():
+			t.Fatalf("Timeout waiting for interceptor get updated config")
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the a new filter instance is created because of the new filter
+	// name. Three new interceptor instances should also be created (one for the
+	// default filter chain, and two for the newly added filter chains).
+	if got, want := fb.filtersCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(4); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+
+	// Stop the server (which causes the listener to be closed) to trigger
+	// cleanup of filters and interceptors, and verify that all instances are
+	// cleaned up.
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(2); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}

test/xds/xds_server_integration_test.go

@@ -0,0 +1,548 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func testModeChangeServerOption(t *testing.T) grpc.ServerOption {
+	// Create a server option to get notified about serving mode changes. We don't
+	// do anything other than throwing a log entry here. But this is required,
+	// since the server code emits a log entry at the default level (which is
+	// ERROR) if no callback is registered for serving mode changes. Our
+	// testLogger fails the test if there is any log entry at ERROR level. It does
+	// provide an ExpectError()  method, but that takes a string and it would be
+	// painful to construct the exact error message expected here. Instead this
+	// works just fine.
+	return xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
+		t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
+	})
+}
+
+// acceptNotifyingListener wraps a listener and notifies users when a server
+// calls the Listener.Accept() method. This can be used to ensure that the
+// server is ready before requests are sent to it.
+type acceptNotifyingListener struct {
+	net.Listener
+	serverReady grpcsync.Event
+}
+
+func (l *acceptNotifyingListener) Accept() (net.Conn, error) {
+	l.serverReady.Fire()
+	return l.Listener.Accept()
+}
+
+// setupGRPCServer performs the following:
+//   - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
+//     register the test service on it
+//   - create a local TCP listener and start serving on it
+//
+// Returns the following:
+// - local listener on which the xDS-enabled gRPC server is serving on
+// - cleanup function to be invoked by the tests when done
+func setupGRPCServer(t *testing.T, bootstrapContents []byte, opts ...grpc.ServerOption) (net.Listener, func()) {
+	t.Helper()
+
+	// Configure xDS credentials to be used on the server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Initialize a test gRPC server, assign it to the stub server, and start
+	// the test service.
+	stub := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				_, err := stream.Recv() // hangs here forever if stream doesn't shut down...doesn't receive EOF without any errors
+				if err == io.EOF {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		},
+	}
+
+	opts = append([]grpc.ServerOption{
+		grpc.Creds(creds),
+		testModeChangeServerOption(t),
+		xds.BootstrapContentsForTesting(bootstrapContents),
+	}, opts...)
+	if stub.S, err = xds.NewGRPCServer(opts...); err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+
+	// Create a local listener and pass it to Serve().
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	readyLis := &acceptNotifyingListener{
+		Listener:    lis,
+		serverReady: *grpcsync.NewEvent(),
+	}
+
+	stub.Listener = readyLis
+	stubserver.StartTestService(t, stub)
+
+	// Wait for the server to start running.
+	select {
+	case <-readyLis.serverReady.Done():
+	case <-time.After(defaultTestTimeout):
+		t.Fatalf("Timed out while waiting for the backend server to start serving")
+	}
+
+	return lis, func() {
+		stub.S.Stop()
+	}
+}
+
+func hostPortFromListener(lis net.Listener) (string, uint32, error) {
+	host, p, err := net.SplitHostPort(lis.Addr().String())
+	if err != nil {
+		return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
+	}
+	port, err := strconv.ParseInt(p, 10, 32)
+	if err != nil {
+		return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
+	}
+	return host, uint32(port), nil
+}
+
+// TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials
+// fallback functionality.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to not send any security configuration to both
+//     the client and the server. This results in both of them using the
+//     configured fallback credentials (which is insecure creds in this case).
+func (s) TestServerSideXDS_Fallback(t *testing.T) {
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+	defer cleanup2()
+
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resources.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service-fallback"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the server-side
+	// xdsCredentials to use fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	// Setup the management server with client and server-side resources.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn with the xds scheme and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Errorf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS
+// credentials with file watcher certificate provider.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to send security configuration to both the
+//     client and the server, pointing to the file watcher certificate provider.
+//     We verify both TLS and mTLS scenarios.
+func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
+	tests := []struct {
+		name     string
+		secLevel e2e.SecurityLevel
+	}{
+		{
+			name:     "tls",
+			secLevel: e2e.SecurityLevelTLS,
+		},
+		{
+			name:     "mtls",
+			secLevel: e2e.SecurityLevelMTLS,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			serviceName := "my-service-file-watcher-certs-" + test.name
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   test.secLevel,
+			})
+
+			// Create an inbound xDS listener resource for the server side that
+			// contains security configuration pointing to the file watcher
+			// plugin.
+			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName")
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			// Setup the management server with client and server resources.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+				FallbackCreds: insecure.NewCredentials(),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("failed to create a client for server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+				t.Fatalf("rpc EmptyCall() failed: %v", err)
+			}
+		})
+	}
+}
+
+// TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on
+// the server-side and xdsCredentials are configured for security. The control
+// plane initially does not any security configuration. This forces the
+// xdsCredentials to use fallback creds, which is this case is insecure creds.
+// We verify that a client connecting with TLS creds is not able to successfully
+// make an RPC. The control plane then sends a listener resource with security
+// configuration pointing to the use of the file_watcher plugin and we verify
+// that the same client is now able to successfully make an RPC.
+func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+	defer cleanup2()
+
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resource. This should force the xDS
+	// credentials on the client to use its fallback.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service-security-config-change"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the xDS credentials
+	// on server to use its fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	// Setup the management server with client and server-side resources.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn with the xds scheme and make a successful RPC.
+	xdsCC, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer xdsCC.Close()
+
+	client := testgrpc.NewTestServiceClient(xdsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// Create a ClientConn with TLS creds. This should fail since the server is
+	// using fallback credentials which in this case in insecure creds.
+	tlsCreds := testutils.CreateClientTLSCredentials(t)
+	tlsCC, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer tlsCC.Close()
+
+	// We don't set 'waitForReady` here since we want this call to failfast.
+	client = testgrpc.NewTestServiceClient(tlsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+
+	// Switch server and client side resources with ones that contain required
+	// security configuration for mTLS with a file watcher certificate provider.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make another RPC with `waitForReady` set and expect this to succeed.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestServerSideXDS_FileWatcherCertsSPIFFE is an e2e test which verifies xDS
+// credentials with file watcher certificate provider that is configured with a
+// SPIFFE Bundle Map for it's roots.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to send security configuration to both the
+//     client and the server, pointing to the file watcher certificate provider.
+//     We verify both TLS and mTLS scenarios.
+func (s) TestServerSideXDS_FileWatcherCertsSPIFFE(t *testing.T) {
+	tests := []struct {
+		name     string
+		secLevel e2e.SecurityLevel
+	}{
+		{
+			name:     "tls",
+			secLevel: e2e.SecurityLevelTLS,
+		},
+		{
+			name:     "mtls",
+			secLevel: e2e.SecurityLevelMTLS,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t)
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			serviceName := "my-service-file-watcher-certs-" + test.name
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   test.secLevel,
+			})
+
+			// Create an inbound xDS listener resource for the server side that
+			// contains security configuration pointing to the file watcher
+			// plugin.
+			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName")
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			// Setup the management server with client and server resources.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+				FallbackCreds: insecure.NewCredentials(),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("failed to create a client for server: %v", err)
+			}
+			defer cc.Close()
+
+			peer := &peer.Peer{}
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+				t.Fatalf("rpc EmptyCall() failed: %v", err)
+			}
+			verifySecurityInformationFromPeerSPIFFE(t, peer, test.secLevel, 1)
+		})
+	}
+}
+
+// Checks the AuthInfo available in the peer if it matches the expected security
+// level of the connection.
+func verifySecurityInformationFromPeerSPIFFE(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel, wantPeerChainLen int) {
+	// This is not a true helper in the Go sense, because it does not perform
+	// setup or cleanup tasks. Marking it a helper is to ensure that when the
+	// test fails, the line information of the caller is outputted instead of
+	// from here.
+	//
+	// And this function directly calls t.Fatalf() instead of returning an error
+	// and letting the caller decide what to do with it. This is also OK since
+	// all callers will simply end up calling t.Fatalf() with the returned
+	// error, and can't add any contextual information of value to the error
+	// message.
+	t.Helper()
+
+	authType := pr.AuthInfo.AuthType()
+	switch wantSecLevel {
+	case e2e.SecurityLevelNone:
+		if authType != "insecure" {
+			t.Fatalf("AuthType() is %s, want insecure", authType)
+		}
+	case e2e.SecurityLevelMTLS:
+		if authType != "tls" {
+			t.Fatalf("AuthType() is %s, want tls", authType)
+		}
+		ai, ok := pr.AuthInfo.(credentials.TLSInfo)
+		if !ok {
+			t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{})
+		}
+		if len(ai.State.PeerCertificates) != wantPeerChainLen {
+			t.Fatalf("Number of peer certificates is %d, want %d", len(ai.State.PeerCertificates), wantPeerChainLen)
+		}
+	}
+}

test/xds/xds_server_interceptor_leak_test.go

@@ -0,0 +1,274 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/xds"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// Tests that interceptor instances are properly closed when a new RDS
+// configuration is received for the same route configuration name.
+func (s) TestServerSideXDS_InterceptorLeak_RDSUpdate(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	// Add xDS resources for the client.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Add xDS resources for the server.
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	const routeConfigName = "routeName"
+	rds1 := &v3routepb.RouteConfiguration{
+		Name:         routeConfigName,
+		VirtualHosts: vhs,
+	}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "path1"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+					Rds: &v3httppb.Rds{
+						ConfigSource: &v3corepb.ConfigSource{
+							ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+						},
+						RouteConfigName: routeConfigName,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "v4-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+		},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+	resources.Routes = append(resources.Routes, rds1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and two interceptors are created
+	// (one per filter chain).
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "path1"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Update the RDS configuration with a new route for the same route configuration
+	// name. This should result in a new interceptor instance being created for
+	// the new route.
+	rds2 := &v3routepb.RouteConfiguration{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{"*"},
+			Routes: []*v3routepb.Route{
+				{
+					Name: "EmptyCall",
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action: &v3routepb.Route_NonForwardingAction{},
+				},
+				{
+					Name: "Wildcard",
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+					},
+					Action: &v3routepb.Route_NonForwardingAction{},
+				},
+			},
+		}},
+	}
+	resources.Routes = []*v3routepb.RouteConfiguration{resources.Routes[0], rds2}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied. Since the new configuration has
+	// two routes, we expect two interceptor instances to be created for each
+	// filter chain, for a total of 6 interceptor instances. We also wait for
+	// the old interceptors (one per filter chain, two total) to be destroyed,
+	// since destruction happens after the new interceptors are created within
+	// the same RDS update callback. Breaking only on interceptorsCreated==6
+	// would race with the subsequent interceptorsDestroyed increment.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case <-pathCh:
+		default:
+		}
+		if fb.interceptorsCreated.Load() == 6 && fb.interceptorsDestroyed.Load() == 2 {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the filter instance is retained, while the interceptor instances
+	// are replaced with the updated config.
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(6); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}

test/xds/xds_telemetry_labels_test.go

@@ -0,0 +1,281 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/experimental/stats/telemetry"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const serviceNameKey = "service_name"
+const serviceNameKeyCSM = "csm.service_name"
+const serviceNamespaceKey = "service_namespace"
+const serviceNamespaceKeyCSM = "csm.service_namespace_name"
+const serviceNameValue = "grpc-service"
+const serviceNamespaceValue = "grpc-service-namespace"
+const backendServiceKey = "grpc.lb.backend_service"
+const backendServiceValue = "cluster-my-service-client-side-xds"
+const localityKey = "grpc.lb.locality"
+const localityValue = `{region="region-1", zone="zone-1", sub_zone="subzone-1"}`
+
+// TestTelemetryLabels tests that telemetry labels from CDS make their way to
+// the stats handler. The stats handler sets the mutable context value that the
+// cluster impl picker will write telemetry labels to, and then the stats
+// handler asserts that subsequent HandleRPC calls from the RPC lifecycle
+// contain telemetry labels that it can see.
+func (s) TestTelemetryLabels(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const xdsServiceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: xdsServiceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	resources.Clusters[0].Metadata = &v3corepb.Metadata{
+		FilterMetadata: map[string]*structpb.Struct{
+			"com.google.csm.telemetry_labels": {
+				Fields: map[string]*structpb.Value{
+					serviceNameKey:      structpb.NewStringValue(serviceNameValue),
+					serviceNamespaceKey: structpb.NewStringValue(serviceNamespaceValue),
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	fsh := &fakeStatsHandler{
+		t: t,
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), grpc.WithStatsHandler(fsh))
+	if err != nil {
+		t.Fatalf("failed to create a new client to local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests that telemetry labels for an aggregate cluster hierarchy reflect the
+// active leaf cluster receiving traffic, and correctly switch labels when
+// failing over to a secondary leaf cluster.
+func (s) TestTelemetryLabels_AggregateCluster(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	const (
+		xdsServiceName = "my-service-client-side-xds"
+		cluster1Name   = "cluster-1"
+		cluster2Name   = "cluster-2"
+
+		csmName1 = "service-1"
+		csmNs1   = "namespace-1"
+		csmName2 = "service-2"
+		csmNs2   = "namespace-2"
+	)
+
+	servers := make([]*stubserver.StubServer, 2)
+	for i := range 2 {
+		servers[i] = stubserver.StartTestService(t, nil)
+		defer servers[i].Stop()
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(xdsServiceName, "route-"+xdsServiceName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig("route-"+xdsServiceName, xdsServiceName, xdsServiceName)},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+				ClusterName: xdsServiceName,
+				Type:        e2e.ClusterTypeAggregate,
+				ChildNames:  []string{cluster1Name, cluster2Name},
+			}),
+			makeClusterResourceWithMetadata(cluster1Name, csmName1, csmNs1),
+			makeClusterResourceWithMetadata(cluster2Name, csmName2, csmNs2),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
+			e2e.DefaultEndpoint(cluster1Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[0].Address))}),
+			e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}),
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a new client to local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Make RPC to primary cluster and verify primary telemetry labels.
+	peer := &peer.Peer{}
+	var gotLabels map[string]string
+	callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+		gotLabels = l
+	})
+	if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), servers[0].Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want %q", got, want)
+	}
+
+	wantLabels := map[string]string{
+		localityKey:       localityValue,
+		backendServiceKey: cluster1Name,
+	}
+	if diff := cmp.Diff(gotLabels, wantLabels); diff != "" {
+		t.Fatalf("Telemetry labels for primary cluster (-got +want): %v", diff)
+	}
+
+	// Trigger failover to secondary cluster by clearing primary endpoints.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(cluster1Name, "localhost", nil),
+		e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}),
+	}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make RPCs until traffic switches to secondary cluster and capture
+	// secondary labels.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+			gotLabels = l
+		})
+		if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err == nil && peer.Addr.String() == servers[1].Address {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout waiting for RPCs to switch to secondary cluster %q", servers[1].Address)
+	}
+
+	wantLabels = map[string]string{
+		localityKey:       localityValue,
+		backendServiceKey: cluster2Name,
+	}
+	if diff := cmp.Diff(gotLabels, wantLabels); diff != "" {
+		t.Fatalf("Telemetry labels after failover (-got +want): %v", diff)
+	}
+}
+
+type fakeStatsHandler struct {
+	labels map[string]string
+
+	t *testing.T
+}
+
+func (fsh *fakeStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+func (fsh *fakeStatsHandler) HandleConn(context.Context, stats.ConnStats) {}
+
+func (fsh *fakeStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context {
+	fsh.labels = make(map[string]string)
+	ctx = telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+		for k, v := range l {
+			fsh.labels[k] = v
+		}
+	})
+	return ctx
+}
+
+func (fsh *fakeStatsHandler) HandleRPC(_ context.Context, rs stats.RPCStats) {
+	switch rs.(type) {
+	// stats.Begin is called before the picker runs, so it won't have telemetry
+	// labels.
+	// The following three stats callouts trigger OpenTelemetry metrics and are
+	// guaranteed to run after the picker has selected a subchannel. Therefore,
+	// they should have access to the desired telemetry labels.
+	case *stats.OutPayload, *stats.InPayload, *stats.End:
+		want := map[string]string{
+			serviceNameKeyCSM:      serviceNameValue,
+			serviceNamespaceKeyCSM: serviceNamespaceValue,
+			localityKey:            localityValue,
+			backendServiceKey:      backendServiceValue,
+		}
+		if diff := cmp.Diff(fsh.labels, want); diff != "" {
+			fsh.t.Fatalf("fsh.labels (-got +want): %v", diff)
+		}
+	default:
+		// Nothing to assert for the other stats.Handler callouts.
+
+	}
+}
+
+func makeClusterResourceWithMetadata(clusterName, serviceName, serviceNamespace string) *v3clusterpb.Cluster {
+	cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: clusterName,
+		Type:        e2e.ClusterTypeEDS,
+	})
+	cluster.Metadata = &v3corepb.Metadata{
+		FilterMetadata: map[string]*structpb.Struct{
+			"com.google.csm.telemetry_labels": {
+				Fields: map[string]*structpb.Value{
+					serviceNameKey:      structpb.NewStringValue(serviceName),
+					serviceNamespaceKey: structpb.NewStringValue(serviceNamespace),
+				},
+			},
+		},
+	}
+	return cluster
+}