snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

test/authority_test.go

@@ -0,0 +1,237 @@
+//go:build linux
+// +build linux
+
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func authorityChecker(ctx context.Context, expectedAuthority string) (*testpb.Empty, error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, "failed to parse metadata")
+	}
+	auths, ok := md[":authority"]
+	if !ok {
+		return nil, status.Error(codes.InvalidArgument, "no authority header")
+	}
+	if len(auths) != 1 {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("no authority header, auths = %v", auths))
+	}
+	if auths[0] != expectedAuthority {
+		return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority))
+	}
+	return &testpb.Empty{}, nil
+}
+
+func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) {
+	if !strings.HasPrefix(target, "unix-abstract:") {
+		if err := os.RemoveAll(address); err != nil {
+			t.Fatalf("Error removing socket file %v: %v\n", address, err)
+		}
+	}
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return authorityChecker(ctx, expectedAuthority)
+		},
+		Network: "unix",
+		Address: address,
+		Target:  target,
+	}
+	opts := []grpc.DialOption{}
+	if dialer != nil {
+		opts = append(opts, grpc.WithContextDialer(dialer))
+	}
+	if err := ss.Start(nil, opts...); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
+	if err != nil {
+		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}
+
+type authorityTest struct {
+	name           string
+	address        string
+	target         string
+	authority      string
+	dialTargetWant string
+}
+
+var authorityTests = []authorityTest{
+	{
+		name:           "UnixRelative",
+		address:        "sock.sock",
+		target:         "unix:sock.sock",
+		authority:      "localhost",
+		dialTargetWant: "unix:sock.sock",
+	},
+	{
+		name:           "UnixAbsolute",
+		address:        "/tmp/sock.sock",
+		target:         "unix:/tmp/sock.sock",
+		authority:      "localhost",
+		dialTargetWant: "unix:///tmp/sock.sock",
+	},
+	{
+		name:           "UnixAbsoluteAlternate",
+		address:        "/tmp/sock.sock",
+		target:         "unix:///tmp/sock.sock",
+		authority:      "localhost",
+		dialTargetWant: "unix:///tmp/sock.sock",
+	},
+	{
+		name:           "UnixPassthrough",
+		address:        "/tmp/sock.sock",
+		target:         "passthrough:///unix:///tmp/sock.sock",
+		authority:      "unix:%2F%2F%2Ftmp%2Fsock.sock",
+		dialTargetWant: "unix:///tmp/sock.sock",
+	},
+	{
+		name:           "UnixAbstract",
+		address:        "@abc efg",
+		target:         "unix-abstract:abc efg",
+		authority:      "localhost",
+		dialTargetWant: "unix:@abc efg",
+	},
+}
+
+// TestUnix does end to end tests with the various supported unix target
+// formats, ensuring that the authority is set as expected.
+func (s) TestUnix(t *testing.T) {
+	for _, test := range authorityTests {
+		t.Run(test.name, func(t *testing.T) {
+			runUnixTest(t, test.address, test.target, test.authority, nil)
+		})
+	}
+}
+
+// TestUnixCustomDialer does end to end tests with various supported unix target
+// formats, ensuring that the target sent to the dialer does NOT have the
+// "unix:" prefix stripped.
+func (s) TestUnixCustomDialer(t *testing.T) {
+	for _, test := range authorityTests {
+		t.Run(test.name+"WithDialer", func(t *testing.T) {
+			dialer := func(ctx context.Context, address string) (net.Conn, error) {
+				if address != test.dialTargetWant {
+					return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.dialTargetWant, address)
+				}
+				address = address[len("unix:"):]
+				return (&net.Dialer{}).DialContext(ctx, "unix", address)
+			}
+			runUnixTest(t, test.address, test.target, test.authority, dialer)
+		})
+	}
+}
+
+// TestColonPortAuthority does an end to end test with the target for grpc.NewClient
+// being ":[port]". Ensures authority is "localhost:[port]".
+func (s) TestColonPortAuthority(t *testing.T) {
+	expectedAuthority := ""
+	var authorityMu sync.Mutex
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			authorityMu.Lock()
+			defer authorityMu.Unlock()
+			return authorityChecker(ctx, expectedAuthority)
+		},
+		Network: "tcp",
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+	_, port, err := net.SplitHostPort(ss.Address)
+	if err != nil {
+		t.Fatalf("Failed splitting host from post: %v", err)
+	}
+	authorityMu.Lock()
+	expectedAuthority = "localhost:" + port
+	authorityMu.Unlock()
+	// ss.Start dials the server, but we explicitly test with ":[port]"
+	// as the target.
+	cc, err := grpc.NewClient(":"+port, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", ss.Target, err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{})
+	if err != nil {
+		t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}
+
+// TestAuthorityReplacedWithResolverAddress tests the scenario where the resolver
+// returned address contains a ServerName override. The test verifies that the
+// :authority header value sent to the server as part of the http/2 HEADERS frame
+// is set to the value specified in the resolver returned address.
+func (s) TestAuthorityReplacedWithResolverAddress(t *testing.T) {
+	const expectedAuthority = "test.server.name"
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return authorityChecker(ctx, expectedAuthority)
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address, ServerName: expectedAuthority}}})
+	cc, err := grpc.NewClient(r.Scheme()+":///whatever", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", ss.Address, err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() rpc failed: %v", err)
+	}
+}

test/balancer_switching_test.go

@@ -0,0 +1,530 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	grpclbstate "google.golang.org/grpc/balancer/grpclb/state"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils/fakegrpclb"
+	pfutil "google.golang.org/grpc/internal/testutils/pickfirst"
+	rrutil "google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const (
+	loadBalancedServiceName = "foo.bar.service"
+	loadBalancedServicePort = 443
+	wantGRPCLBTraceDesc     = `Channel switches to new LB policy "grpclb"`
+	wantRoundRobinTraceDesc = `Channel switches to new LB policy "round_robin"`
+	pickFirstServiceConfig  = `{"loadBalancingConfig": [{"pick_first":{}}]}`
+	grpclbServiceConfig     = `{"loadBalancingConfig": [{"grpclb":{}}]}`
+
+	// This is the number of stub backends set up at the start of each test. The
+	// first backend is used for the "grpclb" policy and the rest are used for
+	// other LB policies to test balancer switching.
+	backendCount = 3
+)
+
+// stubBackendsToResolverAddrs converts from a set of stub server backends to
+// resolver addresses. Useful when pushing addresses to the manual resolver.
+func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address {
+	addrs := make([]resolver.Address, len(backends))
+	for i, backend := range backends {
+		addrs[i] = resolver.Address{Addr: backend.Address}
+	}
+	return addrs
+}
+
+// setupBackendsAndFakeGRPCLB sets up backendCount number of stub server
+// backends and a fake grpclb server for tests which exercise balancer switch
+// scenarios involving grpclb.
+//
+// The fake grpclb server always returns the first of the configured stub
+// backends as backend addresses. So, the tests are free to use the other
+// backends with other LB policies to verify balancer switching scenarios.
+//
+// Returns a cleanup function to be invoked by the caller.
+func setupBackendsAndFakeGRPCLB(t *testing.T) ([]*stubserver.StubServer, *fakegrpclb.Server, func()) {
+	backends, backendsCleanup := startBackendsForBalancerSwitch(t)
+
+	lbServer, err := fakegrpclb.NewServer(fakegrpclb.ServerParams{
+		LoadBalancedServiceName: loadBalancedServiceName,
+		LoadBalancedServicePort: loadBalancedServicePort,
+		BackendAddresses:        []string{backends[0].Address},
+	})
+	if err != nil {
+		t.Fatalf("failed to create fake grpclb server: %v", err)
+	}
+	go func() {
+		if err := lbServer.Serve(); err != nil {
+			t.Errorf("fake grpclb Serve() failed: %v", err)
+		}
+	}()
+
+	return backends, lbServer, func() {
+		backendsCleanup()
+		lbServer.Stop()
+	}
+}
+
+// startBackendsForBalancerSwitch spins up a bunch of stub server backends
+// exposing the TestService. Returns a cleanup function to be invoked by the
+// caller.
+func startBackendsForBalancerSwitch(t *testing.T) ([]*stubserver.StubServer, func()) {
+	t.Helper()
+
+	backends := make([]*stubserver.StubServer, backendCount)
+	for i := 0; i < backendCount; i++ {
+		backend := &stubserver.StubServer{
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+		}
+		if err := backend.StartServer(); err != nil {
+			t.Fatalf("Failed to start backend: %v", err)
+		}
+		t.Logf("Started TestService backend at: %q", backend.Address)
+		backends[i] = backend
+	}
+	return backends, func() {
+		for _, b := range backends {
+			b.Stop()
+		}
+	}
+}
+
+// TestBalancerSwitch_Basic tests the basic scenario of switching from one LB
+// policy to another, as specified in the service config.
+func (s) TestBalancerSwitch_Basic(t *testing.T) {
+	backends, cleanup := startBackendsForBalancerSwitch(t)
+	defer cleanup()
+	addrs := stubBackendsToResolverAddrs(backends)
+
+	r := manual.NewBuilderWithScheme("whatever")
+
+	r.InitialState(resolver.State{Addresses: addrs})
+
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	// Push a resolver update without an LB policy in the service config. The
+	// channel should pick the default LB policy, which is pick_first.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update with a service config specifying "round_robin".
+	r.UpdateState(resolver.State{
+		Addresses:     addrs,
+		ServiceConfig: parseServiceConfig(t, r, rrServiceConfig),
+	})
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push another resolver update with a service config specifying "pick_first".
+	r.UpdateState(resolver.State{
+		Addresses:     addrs,
+		ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig),
+	})
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestBalancerSwitch_grpclbToPickFirst tests the scenario where the channel
+// starts off "grpclb", switches to "pick_first" and back.
+func (s) TestBalancerSwitch_grpclbToPickFirst(t *testing.T) {
+	backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t)
+	defer cleanup()
+
+	addrs := stubBackendsToResolverAddrs(backends)
+	r := manual.NewBuilderWithScheme("whatever")
+	target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Push a resolver update with a GRPCLB service config and a single address
+	// pointing to the grpclb server we created above. This will cause the
+	// channel to switch to the "grpclb" balancer, which returns a single
+	// backend address.
+	grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig)
+	state := resolver.State{ServiceConfig: grpclbConfig}
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}}))
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[0:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update containing a non-existent grpclb server address.
+	// This should not lead to a balancer switch.
+	const nonExistentServer = "non-existent-grpclb-server-address"
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: nonExistentServer}}}))
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update containing no grpclb server address. This should
+	// lead to the channel using the default LB policy which is pick_first. The
+	// list of addresses pushed as part of this update is different from the one
+	// returned by the "grpclb" balancer. So, we should see RPCs going to the
+	// newly configured backends, as part of the balancer switch.
+	emptyConfig := parseServiceConfig(t, r, `{}`)
+	r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig})
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestBalancerSwitch_pickFirstToGRPCLB tests the scenario where the channel
+// starts off with "pick_first", switches to "grpclb" and back.
+func (s) TestBalancerSwitch_pickFirstToGRPCLB(t *testing.T) {
+	backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t)
+	defer cleanup()
+
+	addrs := stubBackendsToResolverAddrs(backends)
+	r := manual.NewBuilderWithScheme("whatever")
+	target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	// Set an empty initial resolver state. This should lead to the channel
+	// using the default LB policy which is pick_first.
+	r.InitialState(resolver.State{Addresses: addrs[1:]})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update with no service config and a single address pointing
+	// to the grpclb server we created above. This will cause the channel to
+	// switch to the "grpclb" balancer, which returns a single backend address.
+	grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig)
+	state := resolver.State{ServiceConfig: grpclbConfig}
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}}))
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update containing a non-existent grpclb server address.
+	// This should not lead to a balancer switch.
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: "nonExistentServer"}}}))
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch to "pick_first" again by sending no grpclb server addresses.
+	emptyConfig := parseServiceConfig(t, r, `{}`)
+	r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig})
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestBalancerSwitch_RoundRobinToGRPCLB tests the scenario where the channel
+// starts off with "round_robin", switches to "grpclb" and back.
+//
+// Note that this test uses the deprecated `loadBalancingPolicy` field in the
+// service config.
+func (s) TestBalancerSwitch_RoundRobinToGRPCLB(t *testing.T) {
+	backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t)
+	defer cleanup()
+
+	addrs := stubBackendsToResolverAddrs(backends)
+	r := manual.NewBuilderWithScheme("whatever")
+	target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	// Note the use of the deprecated `loadBalancingPolicy` field here instead
+	// of the now recommended `loadBalancingConfig` field. The logic in the
+	// ClientConn which decides which balancer to switch to looks at the
+	// following places in the given order of preference:
+	// - `loadBalancingConfig` field
+	// - addresses of type grpclb
+	// - `loadBalancingPolicy` field
+	// If we use the `loadBalancingPolicy` field, the switch to "grpclb" later on
+	// in the test will not happen as the ClientConn will continue to use the LB
+	// policy received in the first update.
+	scpr := parseServiceConfig(t, r, rrServiceConfig)
+
+	// Push a resolver update with the service config specifying "round_robin".
+	r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Push a resolver update with grpclb and a single balancer address
+	// pointing to the grpclb server we created above. This will cause the
+	// channel to switch to the "grpclb" balancer, which returns a single
+	// backend address.
+	grpclbConfig := parseServiceConfig(t, r, grpclbServiceConfig)
+	state := resolver.State{ServiceConfig: grpclbConfig}
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}}))
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Switch back to "round_robin".
+	r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr})
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestBalancerSwitch_grpclbNotRegistered tests the scenario where the grpclb
+// balancer is not registered. Verifies that the ClientConn falls back to the
+// default LB policy or the LB policy specified in the service config, and that
+// addresses of type "grpclb" are filtered out.
+func (s) TestBalancerSwitch_grpclbNotRegistered(t *testing.T) {
+	// Unregister the grpclb balancer builder for the duration of this test.
+	grpclbBuilder := balancer.Get("grpclb")
+	internal.BalancerUnregister(grpclbBuilder.Name())
+	defer balancer.Register(grpclbBuilder)
+
+	backends, cleanup := startBackendsForBalancerSwitch(t)
+	defer cleanup()
+	addrs := stubBackendsToResolverAddrs(backends)
+
+	r := manual.NewBuilderWithScheme("whatever")
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Push a resolver update which contains a bunch of stub server backends and a
+	// grpclb server address. The latter should get the ClientConn to try and
+	// apply the grpclb policy. But since grpclb is not registered, it should
+	// fallback to the default LB policy which is pick_first. The ClientConn is
+	// also expected to filter out the grpclb address when sending the addresses
+	// list for pick_first.
+	grpclbAddr := []resolver.Address{{Addr: "non-existent-grpclb-server-address"}}
+	grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`)
+	state := resolver.State{ServiceConfig: grpclbConfig, Addresses: addrs}
+	r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: grpclbAddr}))
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
+		t.Fatalf("Pick_first backend readiness check failed: %v", err)
+	}
+
+	// Push a resolver update with the same addresses, but with a service config
+	// specifying "round_robin". The ClientConn is expected to filter out the
+	// grpclb address when sending the addresses list to round_robin.
+	r.UpdateState(resolver.State{
+		Addresses:     addrs,
+		ServiceConfig: parseServiceConfig(t, r, rrServiceConfig),
+	})
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatalf("Round robin RPCs failed: %v", err)
+	}
+}
+
+// TestBalancerSwitch_OldBalancerCallsShutdownInClose tests the scenario where
+// the balancer being switched out calls Shutdown() in its Close()
+// method. Verifies that this sequence of calls doesn't lead to a deadlock.
+func (s) TestBalancerSwitch_OldBalancerCallsShutdownInClose(t *testing.T) {
+	// Register a stub balancer which calls Shutdown() from its Close().
+	scChan := make(chan balancer.SubConn, 1)
+	uccsCalled := make(chan struct{}, 1)
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		UpdateClientConnState: func(data *stub.BalancerData, ccs balancer.ClientConnState) error {
+			sc, err := data.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{})
+			if err != nil {
+				t.Errorf("failed to create subConn: %v", err)
+			}
+			scChan <- sc
+			close(uccsCalled)
+			return nil
+		},
+		Close: func(*stub.BalancerData) {
+			(<-scChan).Shutdown()
+		},
+	})
+
+	r := manual.NewBuilderWithScheme("whatever")
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	defer cc.Close()
+
+	// Push a resolver update specifying our stub balancer as the LB policy.
+	scpr := parseServiceConfig(t, r, fmt.Sprintf(`{"loadBalancingPolicy": "%v"}`, t.Name()))
+	r.UpdateState(resolver.State{
+		Addresses:     []resolver.Address{{Addr: "dummy-address"}},
+		ServiceConfig: scpr,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for UpdateClientConnState to be called: %v", ctx.Err())
+	case <-uccsCalled:
+	}
+
+	// The following service config update will switch balancer from our stub
+	// balancer to pick_first. The former will be closed, which will call
+	// sc.Shutdown() inline.
+	//
+	// This is to make sure the sc.Shutdown() from Close() doesn't cause a
+	// deadlock (e.g. trying to grab a mutex while it's already locked).
+	//
+	// Do it in a goroutine so this test will fail with a helpful message
+	// (though the goroutine will still leak).
+	done := make(chan struct{})
+	go func() {
+		r.UpdateState(resolver.State{
+			Addresses:     []resolver.Address{{Addr: "dummy-address"}},
+			ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig),
+		})
+		close(done)
+	}()
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timeout waiting for resolver.UpdateState to finish: %v", ctx.Err())
+	case <-done:
+	}
+}
+
+// TestBalancerSwitch_Graceful tests the graceful switching of LB policies. It
+// starts off by configuring "round_robin" on the channel and ensures that RPCs
+// are successful. Then, it switches to a stub balancer which does not report a
+// picker until instructed by the test do to so. At this point, the test
+// verifies that RPCs are still successful using the old balancer. Then the test
+// asks the new balancer to report a healthy picker and the test verifies that
+// the RPCs get routed using the picker reported by the new balancer.
+func (s) TestBalancerSwitch_Graceful(t *testing.T) {
+	backends, cleanup := startBackendsForBalancerSwitch(t)
+	defer cleanup()
+	addrs := stubBackendsToResolverAddrs(backends)
+
+	r := manual.NewBuilderWithScheme("whatever")
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	// Push a resolver update with the service config specifying "round_robin".
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	r.UpdateState(resolver.State{
+		Addresses:     addrs[1:],
+		ServiceConfig: parseServiceConfig(t, r, rrServiceConfig),
+	})
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register a stub balancer which uses a "pick_first" balancer underneath and
+	// signals on a channel when it receives ClientConn updates. But it does not
+	// forward the ccUpdate to the underlying "pick_first" balancer until the test
+	// asks it to do so. This allows us to test the graceful switch functionality.
+	// Until the test asks the stub balancer to forward the ccUpdate, RPCs should
+	// get routed to the old balancer. And once the test gives the go ahead, RPCs
+	// should get routed to the new balancer.
+	ccUpdateCh := make(chan struct{})
+	waitToProceed := make(chan struct{})
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		Init: func(bd *stub.BalancerData) {
+			pf := balancer.Get(pickfirst.Name)
+			bd.ChildBalancer = pf.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			close(ccUpdateCh)
+			go func() {
+				<-waitToProceed
+				bd.ChildBalancer.UpdateClientConnState(ccs)
+			}()
+			return nil
+		},
+	})
+
+	// Push a resolver update with the service config specifying our stub
+	// balancer. We should see a trace event for this balancer switch. But RPCs
+	// should still be routed to the old balancer since our stub balancer does not
+	// report a ready picker until we ask it to do so.
+	r.UpdateState(resolver.State{
+		Addresses:     addrs[:1],
+		ServiceConfig: r.CC().ParseServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%v": {}}]}`, t.Name())),
+	})
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for a ClientConnState update on the new balancer")
+	case <-ccUpdateCh:
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatalf("RPCs routed to old balancer failed: %v", err)
+	}
+
+	// Ask our stub balancer to forward the earlier received ccUpdate to the
+	// underlying "pick_first" balancer which will result in a healthy picker
+	// being reported to the channel. RPCs should start using the new balancer.
+	close(waitToProceed)
+	if err := pfutil.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil {
+		t.Fatalf("RPCs routed to new balancer failed: %v", err)
+	}
+}

test/bufconn/bufconn.go

@@ -0,0 +1,317 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package bufconn provides a net.Conn implemented by a buffer and related
+// dialing and listening functionality.
+package bufconn
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"time"
+)
+
+// Listener implements a net.Listener that creates local, buffered net.Conns
+// via its Accept and Dial method.
+type Listener struct {
+	mu   sync.Mutex
+	sz   int
+	ch   chan net.Conn
+	done chan struct{}
+}
+
+// Implementation of net.Error providing timeout
+type netErrorTimeout struct {
+	error
+}
+
+func (e netErrorTimeout) Timeout() bool   { return true }
+func (e netErrorTimeout) Temporary() bool { return false }
+
+var errClosed = fmt.Errorf("closed")
+var errTimeout net.Error = netErrorTimeout{error: fmt.Errorf("i/o timeout")}
+
+// Listen returns a Listener that can only be contacted by its own Dialers and
+// creates buffered connections between the two.
+func Listen(sz int) *Listener {
+	return &Listener{sz: sz, ch: make(chan net.Conn), done: make(chan struct{})}
+}
+
+// Accept blocks until Dial is called, then returns a net.Conn for the server
+// half of the connection.
+func (l *Listener) Accept() (net.Conn, error) {
+	select {
+	case <-l.done:
+		return nil, errClosed
+	case c := <-l.ch:
+		return c, nil
+	}
+}
+
+// Close stops the listener.
+func (l *Listener) Close() error {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	select {
+	case <-l.done:
+		// Already closed.
+	default:
+		close(l.done)
+	}
+	return nil
+}
+
+// Addr reports the address of the listener.
+func (l *Listener) Addr() net.Addr { return addr{} }
+
+// Dial creates an in-memory full-duplex network connection, unblocks Accept by
+// providing it the server half of the connection, and returns the client half
+// of the connection.
+func (l *Listener) Dial() (net.Conn, error) {
+	return l.DialContext(context.Background())
+}
+
+// DialContext creates an in-memory full-duplex network connection, unblocks Accept by
+// providing it the server half of the connection, and returns the client half
+// of the connection.  If ctx is Done, returns ctx.Err()
+func (l *Listener) DialContext(ctx context.Context) (net.Conn, error) {
+	p1, p2 := newPipe(l.sz), newPipe(l.sz)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case <-l.done:
+		return nil, errClosed
+	case l.ch <- &conn{p1, p2}:
+		return &conn{p2, p1}, nil
+	}
+}
+
+type pipe struct {
+	mu sync.Mutex
+
+	// buf contains the data in the pipe.  It is a ring buffer of fixed capacity,
+	// with r and w pointing to the offset to read and write, respectively.
+	//
+	// Data is read between [r, w) and written to [w, r), wrapping around the end
+	// of the slice if necessary.
+	//
+	// The buffer is empty if r == len(buf), otherwise if r == w, it is full.
+	//
+	// w and r are always in the range [0, cap(buf)) and [0, len(buf)].
+	buf  []byte
+	w, r int
+
+	wwait sync.Cond
+	rwait sync.Cond
+
+	// Indicate that a write/read timeout has occurred
+	wtimedout bool
+	rtimedout bool
+
+	wtimer *time.Timer
+	rtimer *time.Timer
+
+	closed      bool
+	writeClosed bool
+}
+
+func newPipe(sz int) *pipe {
+	p := &pipe{buf: make([]byte, 0, sz)}
+	p.wwait.L = &p.mu
+	p.rwait.L = &p.mu
+
+	p.wtimer = time.AfterFunc(0, func() {})
+	p.rtimer = time.AfterFunc(0, func() {})
+	return p
+}
+
+func (p *pipe) empty() bool {
+	return p.r == len(p.buf)
+}
+
+func (p *pipe) full() bool {
+	return p.r < len(p.buf) && p.r == p.w
+}
+
+func (p *pipe) Read(b []byte) (n int, err error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	// Block until p has data.
+	for {
+		if p.closed {
+			return 0, io.ErrClosedPipe
+		}
+		if !p.empty() {
+			break
+		}
+		if p.writeClosed {
+			return 0, io.EOF
+		}
+		if p.rtimedout {
+			return 0, errTimeout
+		}
+
+		p.rwait.Wait()
+	}
+	wasFull := p.full()
+
+	n = copy(b, p.buf[p.r:len(p.buf)])
+	p.r += n
+	if p.r == cap(p.buf) {
+		p.r = 0
+		p.buf = p.buf[:p.w]
+	}
+
+	// Signal a blocked writer, if any
+	if wasFull {
+		p.wwait.Signal()
+	}
+
+	return n, nil
+}
+
+func (p *pipe) Write(b []byte) (n int, err error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.closed {
+		return 0, io.ErrClosedPipe
+	}
+	for len(b) > 0 {
+		// Block until p is not full.
+		for {
+			if p.closed || p.writeClosed {
+				return 0, io.ErrClosedPipe
+			}
+			if !p.full() {
+				break
+			}
+			if p.wtimedout {
+				return 0, errTimeout
+			}
+
+			p.wwait.Wait()
+		}
+		wasEmpty := p.empty()
+
+		end := cap(p.buf)
+		if p.w < p.r {
+			end = p.r
+		}
+		x := copy(p.buf[p.w:end], b)
+		b = b[x:]
+		n += x
+		p.w += x
+		if p.w > len(p.buf) {
+			p.buf = p.buf[:p.w]
+		}
+		if p.w == cap(p.buf) {
+			p.w = 0
+		}
+
+		// Signal a blocked reader, if any.
+		if wasEmpty {
+			p.rwait.Signal()
+		}
+	}
+	return n, nil
+}
+
+func (p *pipe) Close() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.closed = true
+	// Signal all blocked readers and writers to return an error.
+	p.rwait.Broadcast()
+	p.wwait.Broadcast()
+	return nil
+}
+
+func (p *pipe) closeWrite() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.writeClosed = true
+	// Signal all blocked readers and writers to return an error.
+	p.rwait.Broadcast()
+	p.wwait.Broadcast()
+	return nil
+}
+
+type conn struct {
+	io.Reader
+	io.Writer
+}
+
+func (c *conn) Close() error {
+	err1 := c.Reader.(*pipe).Close()
+	err2 := c.Writer.(*pipe).closeWrite()
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (c *conn) SetDeadline(t time.Time) error {
+	c.SetReadDeadline(t)
+	c.SetWriteDeadline(t)
+	return nil
+}
+
+func (c *conn) SetReadDeadline(t time.Time) error {
+	p := c.Reader.(*pipe)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.rtimer.Stop()
+	p.rtimedout = false
+	if !t.IsZero() {
+		p.rtimer = time.AfterFunc(time.Until(t), func() {
+			p.mu.Lock()
+			defer p.mu.Unlock()
+			p.rtimedout = true
+			p.rwait.Broadcast()
+		})
+	}
+	return nil
+}
+
+func (c *conn) SetWriteDeadline(t time.Time) error {
+	p := c.Writer.(*pipe)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.wtimer.Stop()
+	p.wtimedout = false
+	if !t.IsZero() {
+		p.wtimer = time.AfterFunc(time.Until(t), func() {
+			p.mu.Lock()
+			defer p.mu.Unlock()
+			p.wtimedout = true
+			p.wwait.Broadcast()
+		})
+	}
+	return nil
+}
+
+func (*conn) LocalAddr() net.Addr  { return addr{} }
+func (*conn) RemoteAddr() net.Addr { return addr{} }
+
+type addr struct{}
+
+func (addr) Network() string { return "bufconn" }
+func (addr) String() string  { return "bufconn" }

test/bufconn/bufconn_test.go

@@ -0,0 +1,328 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package bufconn
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"reflect"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func testRW(r io.Reader, w io.Writer) error {
+	for i := 0; i < 20; i++ {
+		d := make([]byte, i)
+		for j := 0; j < i; j++ {
+			d[j] = byte(i - j)
+		}
+		var rn int
+		var rerr error
+		b := make([]byte, i)
+		done := make(chan struct{})
+		go func() {
+			for rn < len(b) && rerr == nil {
+				var x int
+				x, rerr = r.Read(b[rn:])
+				rn += x
+			}
+			close(done)
+		}()
+		wn, werr := w.Write(d)
+		if wn != i || werr != nil {
+			return fmt.Errorf("%v: w.Write(%v) = %v, %v; want %v, nil", i, d, wn, werr, i)
+		}
+		select {
+		case <-done:
+		case <-time.After(500 * time.Millisecond):
+			return fmt.Errorf("%v: r.Read never returned", i)
+		}
+		if rn != i || rerr != nil {
+			return fmt.Errorf("%v: r.Read = %v, %v; want %v, nil", i, rn, rerr, i)
+		}
+		if !reflect.DeepEqual(b, d) {
+			return fmt.Errorf("%v: r.Read read %v; want %v", i, b, d)
+		}
+	}
+	return nil
+}
+
+func (s) TestPipe(t *testing.T) {
+	p := newPipe(10)
+	if err := testRW(p, p); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestPipeClose(t *testing.T) {
+	p := newPipe(10)
+	p.Close()
+	if _, err := p.Write(nil); err != io.ErrClosedPipe {
+		t.Fatalf("p.Write = _, %v; want _, %v", err, io.ErrClosedPipe)
+	}
+	if _, err := p.Read(nil); err != io.ErrClosedPipe {
+		t.Fatalf("p.Read = _, %v; want _, %v", err, io.ErrClosedPipe)
+	}
+}
+
+func (s) TestConn(t *testing.T) {
+	p1, p2 := newPipe(10), newPipe(10)
+	c1, c2 := &conn{p1, p2}, &conn{p2, p1}
+
+	if err := testRW(c1, c2); err != nil {
+		t.Fatal(err)
+	}
+	if err := testRW(c2, c1); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestConnCloseWithData(t *testing.T) {
+	lis := Listen(7)
+	errChan := make(chan error, 1)
+	var lisConn net.Conn
+	go func() {
+		var err error
+		if lisConn, err = lis.Accept(); err != nil {
+			errChan <- err
+		}
+		close(errChan)
+	}()
+	dialConn, err := lis.Dial()
+	if err != nil {
+		t.Fatalf("Dial error: %v", err)
+	}
+	if err := <-errChan; err != nil {
+		t.Fatalf("Listen error: %v", err)
+	}
+
+	// Write some data on both sides of the connection.
+	n, err := dialConn.Write([]byte("hello"))
+	if n != 5 || err != nil {
+		t.Fatalf("dialConn.Write([]byte{\"hello\"}) = %v, %v; want 5, <nil>", n, err)
+	}
+	n, err = lisConn.Write([]byte("hello"))
+	if n != 5 || err != nil {
+		t.Fatalf("lisConn.Write([]byte{\"hello\"}) = %v, %v; want 5, <nil>", n, err)
+	}
+
+	// Close dial-side; writes from either side should fail.
+	dialConn.Close()
+	if _, err := lisConn.Write([]byte("hello")); err != io.ErrClosedPipe {
+		t.Fatalf("lisConn.Write() = _, <nil>; want _, <non-nil>")
+	}
+	if _, err := dialConn.Write([]byte("hello")); err != io.ErrClosedPipe {
+		t.Fatalf("dialConn.Write() = _, <nil>; want _, <non-nil>")
+	}
+
+	// Read from both sides; reads on lisConn should work, but dialConn should
+	// fail.
+	buf := make([]byte, 6)
+	if _, err := dialConn.Read(buf); err != io.ErrClosedPipe {
+		t.Fatalf("dialConn.Read(buf) = %v, %v; want _, io.ErrClosedPipe", n, err)
+	}
+	n, err = lisConn.Read(buf)
+	if n != 5 || err != nil {
+		t.Fatalf("lisConn.Read(buf) = %v, %v; want 5, <nil>", n, err)
+	}
+}
+
+func (s) TestListener(t *testing.T) {
+	l := Listen(7)
+	var s net.Conn
+	var serr error
+	done := make(chan struct{})
+	go func() {
+		s, serr = l.Accept()
+		close(done)
+	}()
+	c, cerr := l.Dial()
+	<-done
+	if cerr != nil || serr != nil {
+		t.Fatalf("cerr = %v, serr = %v; want nil, nil", cerr, serr)
+	}
+	if err := testRW(c, s); err != nil {
+		t.Fatal(err)
+	}
+	if err := testRW(s, c); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func (s) TestCloseWhileDialing(t *testing.T) {
+	l := Listen(7)
+	var c net.Conn
+	var err error
+	done := make(chan struct{})
+	go func() {
+		c, err = l.Dial()
+		close(done)
+	}()
+	l.Close()
+	<-done
+	if c != nil || err != errClosed {
+		t.Fatalf("c, err = %v, %v; want nil, %v", c, err, errClosed)
+	}
+}
+
+func (s) TestCloseWhileAccepting(t *testing.T) {
+	l := Listen(7)
+	var c net.Conn
+	var err error
+	done := make(chan struct{})
+	go func() {
+		c, err = l.Accept()
+		close(done)
+	}()
+	l.Close()
+	<-done
+	if c != nil || err != errClosed {
+		t.Fatalf("c, err = %v, %v; want nil, %v", c, err, errClosed)
+	}
+}
+
+func (s) TestDeadline(t *testing.T) {
+	sig := make(chan error, 2)
+	blockingWrite := func(conn net.Conn) {
+		_, err := conn.Write([]byte("0123456789"))
+		sig <- err
+	}
+
+	blockingRead := func(conn net.Conn) {
+		_, err := conn.Read(make([]byte, 10))
+		sig <- err
+	}
+
+	p1, p2 := newPipe(5), newPipe(5)
+	c1, c2 := &conn{p1, p1}, &conn{p2, p2}
+	defer c1.Close()
+	defer c2.Close()
+
+	// Test with deadline
+	c1.SetWriteDeadline(time.Now())
+
+	go blockingWrite(c1)
+	select {
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Write timeout timed out, c = %v", c1)
+	case err := <-sig:
+		if netErr, ok := err.(net.Error); ok {
+			if !netErr.Timeout() {
+				t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, netErr)
+			}
+		} else {
+			t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, err)
+		}
+	}
+
+	c2.SetReadDeadline(time.Now())
+
+	go blockingRead(c2)
+	select {
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Read timeout timed out, c = %v", c2)
+	case err := <-sig:
+		if netErr, ok := err.(net.Error); ok {
+			if !netErr.Timeout() {
+				t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, netErr)
+			}
+		} else {
+			t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, err)
+		}
+	}
+
+	// Test timing out pending reads/writes
+	c1.SetWriteDeadline(time.Time{})
+	c2.SetReadDeadline(time.Time{})
+
+	go blockingWrite(c1)
+	select {
+	case <-time.After(100 * time.Millisecond):
+	case err := <-sig:
+		t.Fatalf("Write returned before timeout, err = %v", err)
+	}
+
+	c1.SetWriteDeadline(time.Now())
+	select {
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Write timeout timed out, c = %v", c1)
+	case err := <-sig:
+		if netErr, ok := err.(net.Error); ok {
+			if !netErr.Timeout() {
+				t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, netErr)
+			}
+		} else {
+			t.Fatalf("Write returned unexpected error, c = %v, err = %v", c1, err)
+		}
+	}
+
+	go blockingRead(c2)
+	select {
+	case <-time.After(100 * time.Millisecond):
+	case err := <-sig:
+		t.Fatalf("Read returned before timeout, err = %v", err)
+	}
+
+	c2.SetReadDeadline(time.Now())
+	select {
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Read timeout timed out, c = %v", c2)
+	case err := <-sig:
+		if netErr, ok := err.(net.Error); ok {
+			if !netErr.Timeout() {
+				t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, netErr)
+			}
+		} else {
+			t.Fatalf("Read returned unexpected error, c = %v, err = %v", c2, err)
+		}
+	}
+
+	// Test non-blocking read/write
+	c1, c2 = &conn{p1, p2}, &conn{p2, p1}
+
+	c1.SetWriteDeadline(time.Now().Add(10 * time.Second))
+	c2.SetReadDeadline(time.Now().Add(10 * time.Second))
+
+	// Not blocking here
+	go blockingWrite(c1)
+	go blockingRead(c2)
+
+	// Read response from both routines
+	for i := 0; i < 2; i++ {
+		select {
+		case <-time.After(100 * time.Millisecond):
+			t.Fatalf("Read/Write timed out, c1 = %v, c2 = %v", c1, c2)
+		case err := <-sig:
+			if err != nil {
+				t.Fatalf("Read/Write failed to complete, c1 = %v, c2 = %v, err = %v", c1, c2, err)
+			}
+		}
+	}
+}

test/channelz_linux_test.go

@@ -0,0 +1,95 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/channelz"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestCZSocketMetricsSocketOption(t *testing.T) {
+	envs := []env{tcpClearRREnv, tcpTLSRREnv}
+	for _, e := range envs {
+		testCZSocketMetricsSocketOption(t, e)
+	}
+}
+
+func testCZSocketMetricsSocketOption(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	doSuccessfulUnaryCall(tc, t)
+
+	time.Sleep(10 * time.Millisecond)
+	ss, _ := channelz.GetServers(0, 0)
+	if len(ss) != 1 {
+		t.Fatalf("There should be one server, not %d", len(ss))
+	}
+	skts := ss[0].ListenSockets()
+	if len(skts) != 1 {
+		t.Fatalf("There should be one listen socket, not %d", len(skts))
+	}
+	for id := range skts {
+		sm := channelz.GetSocket(id)
+		if sm == nil || sm.SocketOptions == nil {
+			t.Fatalf("Unable to get server listen socket options")
+		}
+	}
+	ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0)
+	if len(ns) != 1 {
+		t.Fatalf("There should be one server normal socket, not %d", len(ns))
+	}
+	if ns[0] == nil || ns[0].SocketOptions == nil {
+		t.Fatalf("Unable to get server normal socket options")
+	}
+
+	tchan, _ := channelz.GetTopChannels(0, 0)
+	if len(tchan) != 1 {
+		t.Fatalf("There should only be one top channel, not %d", len(tchan))
+	}
+	subChans := tchan[0].SubChans()
+	if len(subChans) != 1 {
+		t.Fatalf("There should only be one subchannel under top channel %d, not %d", tchan[0].ID, len(subChans))
+	}
+	var id int64
+	for id = range subChans {
+		break
+	}
+	sc := channelz.GetSubChannel(id)
+	if sc == nil {
+		t.Fatalf("There should only be one socket under subchannel %d, not 0", id)
+	}
+	skts = sc.Sockets()
+	if len(skts) != 1 {
+		t.Fatalf("There should only be one socket under subchannel %d, not %d", sc.ID, len(skts))
+	}
+	for id = range skts {
+		break
+	}
+	skt := channelz.GetSocket(id)
+	if skt == nil || skt.SocketOptions == nil {
+		t.Fatalf("Unable to get client normal socket options")
+	}
+}

test/clientconn_state_transition_test.go

@@ -0,0 +1,832 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/backoff"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Keep reading until something causes the connection to die (EOF, server
+// closed, etc). Useful as a tool for mindlessly keeping the connection
+// healthy, since the client will error if things like client prefaces are not
+// accepted in a timely fashion.
+func keepReading(conn net.Conn) {
+	io.Copy(io.Discard, conn)
+}
+
+type funcConnectivityStateSubscriber struct {
+	onMsg func(connectivity.State)
+}
+
+func (f *funcConnectivityStateSubscriber) OnMessage(msg any) {
+	f.onMsg(msg.(connectivity.State))
+}
+
+func waitForState(ctx context.Context, t *testing.T, stateCh <-chan connectivity.State, want connectivity.State) {
+	t.Helper()
+	select {
+	case gotState := <-stateCh:
+		if gotState != want {
+			t.Fatalf("State is %s; want %s", gotState, want)
+		}
+		t.Logf("State is %s as expected", gotState)
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for state update: %s", want)
+	}
+}
+
+// Tests for state transitions in various scenarios with a single address.
+func (s) TestStateTransitions_SingleAddress(t *testing.T) {
+	for _, test := range []struct {
+		desc       string
+		wantStates []connectivity.State
+		server     func(net.Listener) net.Conn
+	}{
+		{
+			desc: "ServerSendsPreface",
+			wantStates: []connectivity.State{
+				connectivity.Connecting,
+				connectivity.Ready,
+			},
+			server: func(lis net.Listener) net.Conn {
+				conn, err := lis.Accept()
+				if err != nil {
+					t.Error(err)
+					return nil
+				}
+
+				go keepReading(conn)
+
+				framer := http2.NewFramer(conn, conn)
+				if err := framer.WriteSettings(http2.Setting{}); err != nil {
+					t.Errorf("Error while writing settings frame. %v", err)
+					return nil
+				}
+
+				return conn
+			},
+		},
+		{
+			desc: "ConnectionClosesBeforeServerPreface",
+			wantStates: []connectivity.State{
+				connectivity.Connecting,
+				connectivity.TransientFailure,
+			},
+			server: func(lis net.Listener) net.Conn {
+				conn, err := lis.Accept()
+				if err != nil {
+					t.Error(err)
+					return nil
+				}
+
+				conn.Close()
+				return nil
+			},
+		},
+		{
+			desc: "ConnectionClosesBeforeClientPreface",
+			wantStates: []connectivity.State{
+				connectivity.Connecting,
+				connectivity.TransientFailure,
+			},
+			server: func(lis net.Listener) net.Conn {
+				conn, err := lis.Accept()
+				if err != nil {
+					t.Error(err)
+					return nil
+				}
+
+				framer := http2.NewFramer(conn, conn)
+				if err := framer.WriteSettings(http2.Setting{}); err != nil {
+					t.Errorf("Error while writing settings frame. %v", err)
+					return nil
+				}
+
+				conn.Close()
+				return nil
+			},
+		},
+		{
+			desc: "ServerNeverSendsPreface",
+			wantStates: []connectivity.State{
+				connectivity.Connecting,
+				connectivity.TransientFailure,
+			},
+			server: func(lis net.Listener) net.Conn {
+				conn, err := lis.Accept()
+				if err != nil {
+					t.Error(err)
+					return nil
+				}
+
+				go keepReading(conn)
+
+				return conn
+			},
+		},
+	} {
+		t.Run(test.desc, func(t *testing.T) {
+			testStateTransitionSingleAddress(t, test.wantStates, test.server)
+		})
+	}
+}
+
+func testStateTransitionSingleAddress(t *testing.T, wantStates []connectivity.State, server func(net.Listener) net.Conn) {
+	pl := testutils.NewPipeListener()
+	defer pl.Close()
+
+	// Launch the server.
+	var conn net.Conn
+	var connMu sync.Mutex
+	go func() {
+		connMu.Lock()
+		conn = server(pl)
+		connMu.Unlock()
+	}()
+
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDialer(pl.Dialer()),
+		grpc.WithConnectParams(grpc.ConnectParams{
+			Backoff:           backoff.Config{},
+			MinConnectTimeout: 100 * time.Millisecond,
+		}),
+	}
+	cc, err := grpc.NewClient("passthrough:///", dopts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+
+	// Ensure that the client is in IDLE before connecting.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+
+	// Subscribe to state updates.
+	stateCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case stateCh <- s:
+			case <-ctx.Done():
+			}
+		},
+	}
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	cc.Connect()
+	for _, wantState := range wantStates {
+		waitForState(ctx, t, stateCh, wantState)
+	}
+
+	connMu.Lock()
+	defer connMu.Unlock()
+	if conn != nil {
+		err = conn.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// Tests for state transitions when the READY connection is closed.
+func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	defer lis.Close()
+
+	sawReady := make(chan struct{}, 1)
+	defer close(sawReady)
+
+	// Launch the server.
+	go func() {
+		conn, err := lis.Accept()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		go keepReading(conn)
+
+		framer := http2.NewFramer(conn, conn)
+		if err := framer.WriteSettings(http2.Setting{}); err != nil {
+			t.Errorf("Error while writing settings frame. %v", err)
+			return
+		}
+
+		// Prevents race between onPrefaceReceipt and onClose.
+		<-sawReady
+
+		conn.Close()
+	}()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+
+	// Ensure that the client is in IDLE before connecting.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+
+	// Subscribe to state updates.
+	stateCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case stateCh <- s:
+			case <-ctx.Done():
+			}
+		},
+	}
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	cc.Connect()
+	wantStates := []connectivity.State{
+		connectivity.Connecting,
+		connectivity.Ready,
+		connectivity.Idle,
+		connectivity.Connecting,
+	}
+	for _, wantState := range wantStates {
+		waitForState(ctx, t, stateCh, wantState)
+		if wantState == connectivity.Ready {
+			sawReady <- struct{}{}
+		}
+		if wantState == connectivity.Idle {
+			cc.Connect()
+		}
+	}
+}
+
+// Tests for state transitions when there are multiple addresses and all the
+// addresses fail.
+func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) {
+	lis1, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	defer lis1.Close()
+
+	lis2, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	defer lis2.Close()
+
+	server1Done := make(chan struct{})
+	server2Done := make(chan struct{})
+
+	// Launch server 1.
+	go func() {
+		conn, err := lis1.Accept()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		conn.Close()
+		close(server1Done)
+	}()
+	// Launch server 2.
+	go func() {
+		conn, err := lis2.Accept()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		conn.Close()
+		close(server2Done)
+	}()
+
+	rb := manual.NewBuilderWithScheme("whatever")
+	rb.InitialState(resolver.State{Addresses: []resolver.Address{
+		{Addr: lis1.Addr().String()},
+		{Addr: lis2.Addr().String()},
+	}})
+
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithResolvers(rb),
+		grpc.WithConnectParams(grpc.ConnectParams{
+			// Set a really long back-off delay to ensure the subchannels stay
+			// in TRANSIENT_FAILURE and not enter IDLE.
+			Backoff: backoff.Config{BaseDelay: 1 * time.Hour},
+		}),
+	}
+	cc, err := grpc.NewClient("whatever:///this-gets-overwritten", dopts...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+
+	// Ensure that the client is in IDLE before connecting.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+
+	// Subscribe to state updates.
+	stateCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case stateCh <- s:
+			case <-ctx.Done():
+			}
+		},
+	}
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	cc.Connect()
+	wantStates := []connectivity.State{
+		connectivity.Connecting,
+		connectivity.TransientFailure,
+	}
+	for _, wantState := range wantStates {
+		waitForState(ctx, t, stateCh, wantState)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1")
+	case <-server1Done:
+	}
+	select {
+	case <-ctx.Done():
+		t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 2")
+	case <-server2Done:
+	}
+}
+
+// Tests for state transitions with multiple addresses when the READY connection
+// is closed.
+func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) {
+	lis1, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	defer lis1.Close()
+
+	// Never actually gets used; we just want it to be alive so that the
+	// resolver has two addresses to target.
+	lis2, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	defer lis2.Close()
+
+	server1Done := make(chan struct{})
+	sawReady := make(chan struct{}, 1)
+	defer close(sawReady)
+
+	// Launch server 1.
+	go func() {
+		conn, err := lis1.Accept()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		go keepReading(conn)
+
+		framer := http2.NewFramer(conn, conn)
+		if err := framer.WriteSettings(http2.Setting{}); err != nil {
+			t.Errorf("Error while writing settings frame. %v", err)
+			return
+		}
+
+		<-sawReady
+
+		conn.Close()
+
+		close(server1Done)
+	}()
+
+	rb := manual.NewBuilderWithScheme("whatever")
+	rb.InitialState(resolver.State{Addresses: []resolver.Address{
+		{Addr: lis1.Addr().String()},
+		{Addr: lis2.Addr().String()},
+	}})
+	cc, err := grpc.NewClient("whatever:///this-gets-overwritten", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(rb))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+
+	// Ensure that the client is in IDLE before connecting.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+
+	// Subscribe to state updates.
+	stateCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case stateCh <- s:
+			case <-ctx.Done():
+			}
+		},
+	}
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	cc.Connect()
+	wantStates := []connectivity.State{
+		connectivity.Connecting,
+		connectivity.Ready,
+		connectivity.Idle,
+		connectivity.Connecting,
+	}
+	for _, wantState := range wantStates {
+		waitForState(ctx, t, stateCh, wantState)
+		if wantState == connectivity.Ready {
+			sawReady <- struct{}{}
+		}
+		if wantState == connectivity.Idle {
+			cc.Connect()
+		}
+	}
+	select {
+	case <-ctx.Done():
+		t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1")
+	case <-server1Done:
+	}
+}
+
+// TestConnectivityStateSubscriber confirms updates sent by the balancer in
+// rapid succession are not missed by the subscriber.
+func (s) TestConnectivityStateSubscriber(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	sendStates := []connectivity.State{
+		connectivity.Connecting,
+		connectivity.Ready,
+		connectivity.Idle,
+		connectivity.Connecting,
+		connectivity.Idle,
+		connectivity.Connecting,
+		connectivity.Ready,
+	}
+	wantStates := append(sendStates, connectivity.Shutdown)
+
+	const testBalName = "any"
+	bf := stub.BalancerFuncs{
+		UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
+			// Send the expected states in rapid succession.
+			for _, s := range sendStates {
+				t.Logf("Sending state update %s", s)
+				bd.ClientConn.UpdateState(balancer.State{ConnectivityState: s})
+			}
+			return nil
+		},
+	}
+	stub.Register(testBalName, bf)
+
+	// Create the ClientConn.
+	const testResName = "any"
+	rb := manual.NewBuilderWithScheme(testResName)
+	cc, err := grpc.NewClient(testResName+":///",
+		grpc.WithResolvers(rb),
+		grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalName)),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	)
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	// Subscribe to state updates.  Use a buffer size of 1 to allow the
+	// Shutdown state to go into the channel when Close()ing.
+	connCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case connCh <- s:
+			case <-ctx.Done():
+			}
+			if s == connectivity.Shutdown {
+				close(connCh)
+			}
+		},
+	}
+
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	// Send an update from the resolver that will trigger the LB policy's UpdateClientConnState.
+	go rb.UpdateState(resolver.State{})
+
+	// Verify the resulting states.
+	for i, want := range wantStates {
+		if i == len(sendStates) {
+			// Trigger Shutdown to be sent by the channel.  Use a goroutine to
+			// ensure the operation does not block.
+			cc.Close()
+		}
+		select {
+		case got := <-connCh:
+			if got != want {
+				t.Errorf("Update %v was %s; want %s", i, got, want)
+			} else {
+				t.Logf("Update %v was %s as expected", i, got)
+			}
+		case <-ctx.Done():
+			t.Fatalf("Timed out waiting for state update %v: %s", i, want)
+		}
+	}
+}
+
+// Test verifies that a channel starts off in IDLE and transitions to CONNECTING
+// when Connect() is called, and stays there when there are no resolver updates.
+func (s) TestStateTransitions_WithConnect_NoResolverUpdate(t *testing.T) {
+	backend := stubserver.StartTestService(t, nil)
+	defer backend.Stop()
+
+	mr := manual.NewBuilderWithScheme("e2e-test")
+	defer mr.Close()
+
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("Failed to create new client: %v", err)
+	}
+	defer cc.Close()
+
+	if state := cc.GetState(); state != connectivity.Idle {
+		t.Fatalf("Expected initial state to be IDLE, got %v", state)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// The channel should transition to CONNECTING automatically when Connect()
+	// is called.
+	cc.Connect()
+	testutils.AwaitState(ctx, t, cc, connectivity.Connecting)
+
+	// Verify that the channel remains in CONNECTING state for a short time.
+	shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer shortCancel()
+	testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Connecting)
+}
+
+// Test verifies that a channel starts off in IDLE and transitions to CONNECTING
+// when Connect() is called, and stays there when there are no resolver updates.
+func (s) TestStateTransitions_WithRPC_NoResolverUpdate(t *testing.T) {
+	backend := stubserver.StartTestService(t, nil)
+	defer backend.Stop()
+
+	mr := manual.NewBuilderWithScheme("e2e-test")
+	defer mr.Close()
+
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("Failed to create new client: %v", err)
+	}
+	defer cc.Close()
+
+	if state := cc.GetState(); state != connectivity.Idle {
+		t.Fatalf("Expected initial state to be IDLE, got %v", state)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Make an RPC call to transition the channel to CONNECTING.
+	go func() {
+		if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err == nil {
+			t.Errorf("Expected RPC to fail, but it succeeded")
+		}
+	}()
+
+	// The channel should transition to CONNECTING automatically when an RPC
+	// is made.
+	testutils.AwaitState(ctx, t, cc, connectivity.Connecting)
+
+	// The channel remains in CONNECTING state for a short time.
+	shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer shortCancel()
+	testutils.AwaitNoStateChange(shortCtx, t, cc, connectivity.Connecting)
+}
+
+const testResolverBuildFailureScheme = "test-resolver-build-failure"
+
+// testResolverBuilder is a resolver builder that fails the first time its
+// Build method is called, and succeeds thereafter.
+type testResolverBuilder struct {
+	logger interface {
+		Logf(format string, args ...any)
+	}
+	buildCalled bool
+	manualR     *manual.Resolver
+}
+
+func (b *testResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+	b.logger.Logf("testResolverBuilder: Build called with target: %v", target)
+	if !b.buildCalled {
+		b.buildCalled = true
+		b.logger.Logf("testResolverBuilder: returning build failure")
+		return nil, fmt.Errorf("simulated resolver build failure")
+	}
+	return b.manualR.Build(target, cc, opts)
+}
+
+func (b *testResolverBuilder) Scheme() string {
+	return testResolverBuildFailureScheme
+}
+
+// Tests for state transitions when the resolver initially fails to build.
+func (s) TestStateTransitions_ResolverBuildFailure(t *testing.T) {
+	tests := []struct {
+		name            string
+		exitIdleWithRPC bool
+	}{
+		{
+			name:            "exitIdleByConnecting",
+			exitIdleWithRPC: false,
+		},
+		{
+			name:            "exitIdleByRPC",
+			exitIdleWithRPC: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			mr := manual.NewBuilderWithScheme("whatever" + tt.name)
+			backend := stubserver.StartTestService(t, nil)
+			defer backend.Stop()
+			mr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}})
+
+			dopts := []grpc.DialOption{
+				grpc.WithTransportCredentials(insecure.NewCredentials()),
+				grpc.WithResolvers(&testResolverBuilder{logger: t, manualR: mr}),
+			}
+
+			cc, err := grpc.NewClient(testResolverBuildFailureScheme+":///", dopts...)
+			if err != nil {
+				t.Fatalf("Failed to create new client: %v", err)
+			}
+			defer cc.Close()
+
+			// Ensure that the client is in IDLE before connecting.
+			if state := cc.GetState(); state != connectivity.Idle {
+				t.Fatalf("Expected initial state to be IDLE, got %v", state)
+			}
+
+			// Subscribe to state updates.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			stateCh := make(chan connectivity.State, 1)
+			s := &funcConnectivityStateSubscriber{
+				onMsg: func(s connectivity.State) {
+					select {
+					case stateCh <- s:
+					case <-ctx.Done():
+					}
+				},
+			}
+			internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+			if tt.exitIdleWithRPC {
+				// The first attempt to kick the channel is expected to return
+				// the resolver build error to the RPC.
+				const wantErr = "simulated resolver build failure"
+				for range 2 {
+					_, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{})
+					if code := status.Code(err); code != codes.Unavailable {
+						t.Fatalf("EmptyCall RPC failed with code %v, want %v", err, codes.Unavailable)
+					}
+					if err == nil || !strings.Contains(err.Error(), wantErr) {
+						t.Fatalf("EmptyCall RPC failed with error: %q, want %q", err, wantErr)
+					}
+				}
+			} else {
+				cc.Connect()
+			}
+
+			wantStates := []connectivity.State{
+				connectivity.Connecting,       // When channel exits IDLE for the first time.
+				connectivity.TransientFailure, // Resolver build failure.
+				connectivity.Idle,             // After idle timeout.
+				connectivity.Connecting,       // When channel exits IDLE again.
+				connectivity.Ready,            // Successful resolver build and connection to backend.
+			}
+			for _, wantState := range wantStates {
+				waitForState(ctx, t, stateCh, wantState)
+				switch wantState {
+				case connectivity.TransientFailure:
+					internal.EnterIdleModeForTesting.(func(*grpc.ClientConn))(cc)
+				case connectivity.Idle:
+					if tt.exitIdleWithRPC {
+						if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil {
+							t.Fatalf("EmptyCall RPC failed: %v", err)
+						}
+					} else {
+						cc.Connect()
+					}
+				}
+			}
+		})
+	}
+}
+
+// Tests for state transitions when the resolver reports no addresses.
+func (s) TestStateTransitions_WithRPC_ResolverUpdateContainsNoAddresses(t *testing.T) {
+	mr := manual.NewBuilderWithScheme("e2e-test")
+	mr.InitialState(resolver.State{})
+	defer mr.Close()
+
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("Failed to create new client: %v", err)
+	}
+	defer cc.Close()
+
+	if state := cc.GetState(); state != connectivity.Idle {
+		t.Fatalf("Expected initial state to be IDLE, got %v", state)
+	}
+
+	// Subscribe to state updates.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stateCh := make(chan connectivity.State, 1)
+	s := &funcConnectivityStateSubscriber{
+		onMsg: func(s connectivity.State) {
+			select {
+			case stateCh <- s:
+			case <-ctx.Done():
+			}
+		},
+	}
+	internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s)
+
+	// Make an RPC call to transition the channel to CONNECTING.
+	const wantErr = "name resolver error: produced zero addresses"
+	for range 2 {
+		_, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{})
+		if code := status.Code(err); code != codes.Unavailable {
+			t.Errorf("EmptyCall RPC failed with code %v, want %v", err, codes.Unavailable)
+		}
+		if err == nil || !strings.Contains(err.Error(), wantErr) {
+			t.Errorf("EmptyCall RPC failed with error: %q, want %q", err, wantErr)
+		}
+	}
+
+	wantStates := []connectivity.State{
+		connectivity.Connecting,       // When channel exits IDLE for the first time.
+		connectivity.TransientFailure, // No endpoints from the resolver
+		connectivity.Idle,             // After idle timeout.
+	}
+	for _, wantState := range wantStates {
+		waitForState(ctx, t, stateCh, wantState)
+		if wantState == connectivity.TransientFailure {
+			internal.EnterIdleModeForTesting.(func(*grpc.ClientConn))(cc)
+		}
+	}
+}

test/clientconn_test.go

@@ -0,0 +1,197 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/channelz"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+// TestClientConnClose_WithPendingRPC tests the scenario where the channel has
+// not yet received any update from the name resolver and hence RPCs are
+// blocking. The test verifies that closing the ClientConn unblocks the RPC with
+// the expected error code.
+func (s) TestClientConnClose_WithPendingRPC(t *testing.T) {
+	r := manual.NewBuilderWithScheme("whatever")
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	client := testgrpc.NewTestServiceClient(cc)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	doneErrCh := make(chan error, 1)
+	go func() {
+		// This RPC would block until the ClientConn is closed, because the
+		// resolver has not provided its first update yet.
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		if status.Code(err) != codes.Canceled || !strings.Contains(err.Error(), "client connection is closing") {
+			doneErrCh <- fmt.Errorf("EmptyCall() = %v, want %s", err, codes.Canceled)
+		}
+		doneErrCh <- nil
+	}()
+
+	// Make sure that there is one pending RPC on the ClientConn before attempting
+	// to close it. If we don't do this, cc.Close() can happen before the above
+	// goroutine gets to make the RPC.
+	for {
+		if err := ctx.Err(); err != nil {
+			t.Fatal(err)
+		}
+		tcs, _ := channelz.GetTopChannels(0, 0)
+		if len(tcs) != 1 {
+			t.Fatalf("there should only be one top channel, not %d", len(tcs))
+		}
+		started := tcs[0].ChannelMetrics.CallsStarted.Load()
+		completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load()
+		if (started - completed) == 1 {
+			break
+		}
+		time.Sleep(defaultTestShortTimeout)
+	}
+	cc.Close()
+	if err := <-doneErrCh; err != nil {
+		t.Fatal(err)
+	}
+}
+
+type testStatsHandler struct {
+	nameResolutionDelayed bool
+}
+
+// TagRPC is called when an RPC is initiated and allows adding metadata to the
+// context. It checks if the RPC experienced a name resolution delay and
+// updates the handler's state.
+func (h *testStatsHandler) TagRPC(ctx context.Context, rpcInfo *stats.RPCTagInfo) context.Context {
+	h.nameResolutionDelayed = rpcInfo.NameResolutionDelay
+	return ctx
+}
+
+// This method is required to satisfy the stats.Handler interface.
+func (h *testStatsHandler) HandleRPC(_ context.Context, _ stats.RPCStats) {}
+
+// TagConn exists to satisfy stats.Handler.
+func (h *testStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+// HandleConn exists to satisfy stats.Handler.
+func (h *testStatsHandler) HandleConn(_ context.Context, _ stats.ConnStats) {}
+
+// TestClientConnRPC_WithoutNameResolutionDelay verify that if the resolution
+// has already happened once before at the time of making RPC, the name
+// resolution flag is not set indicating there was no delay in name resolution.
+func (s) TestClientConnRPC_WithoutNameResolutionDelay(t *testing.T) {
+	statsHandler := &testStatsHandler{}
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil, grpc.WithStatsHandler(statsHandler)); err != nil {
+		t.Fatalf("Failed to start StubServer: %v", err)
+	}
+	defer ss.Stop()
+
+	rb := manual.NewBuilderWithScheme("instant")
+	rb.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}})
+	cc := ss.CC
+	defer cc.Close()
+
+	cc.Connect()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+	client := testgrpc.NewTestServiceClient(cc)
+	// Verify that the RPC succeeds.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("First RPC failed unexpectedly: %v", err)
+	}
+	// verifying that RPC was not blocked on resolver indicating there was no
+	// delay in name resolution.
+	if statsHandler.nameResolutionDelayed {
+		t.Fatalf("statsHandler.nameResolutionDelayed = %v; want false", statsHandler.nameResolutionDelayed)
+	}
+}
+
+// TestStatsHandlerDetectsResolutionDelay verifies that if this is the
+// first time resolution is happening at the time of making RPC,
+// nameResolutionDelayed flag is set indicating there was a delay in name
+// resolution waiting for resolver to return addresses.
+func (s) TestClientConnRPC_WithNameResolutionDelay(t *testing.T) {
+	resolutionWait := grpcsync.NewEvent()
+	prevHook := internal.NewStreamWaitingForResolver
+	internal.NewStreamWaitingForResolver = func() { resolutionWait.Fire() }
+	defer func() { internal.NewStreamWaitingForResolver = prevHook }()
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start StubServer: %v", err)
+	}
+	defer ss.Stop()
+
+	statsHandler := &testStatsHandler{}
+	rb := manual.NewBuilderWithScheme("delayed")
+	cc, err := grpc.NewClient(rb.Scheme()+":///test.server",
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithResolvers(rb),
+		grpc.WithStatsHandler(statsHandler),
+	)
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	go func() {
+		<-resolutionWait.Done()
+		rb.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}})
+	}()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall RPC failed: %v", err)
+	}
+	if !statsHandler.nameResolutionDelayed {
+		t.Fatalf("statsHandler.nameResolutionDelayed = %v; want true", statsHandler.nameResolutionDelayed)
+	}
+}

test/clienttester.go

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"testing"
+
+	"golang.org/x/net/http2"
+)
+
+var (
+	clientPreface = []byte(http2.ClientPreface)
+)
+
+func newClientTester(t *testing.T, conn net.Conn) *clientTester {
+	ct := &clientTester{
+		t:    t,
+		conn: conn,
+	}
+	ct.fr = http2.NewFramer(conn, conn)
+	ct.greet()
+	return ct
+}
+
+type clientTester struct {
+	t    *testing.T
+	conn net.Conn
+	fr   *http2.Framer
+}
+
+// greet() performs the necessary steps for http2 connection establishment on
+// the server side.
+func (ct *clientTester) greet() {
+	ct.wantClientPreface()
+	ct.wantSettingsFrame()
+	ct.writeSettingsFrame()
+	ct.writeSettingsAck()
+
+	for {
+		f, err := ct.fr.ReadFrame()
+		if err != nil {
+			ct.t.Errorf("error reading frame from client side: %v", err)
+		}
+		switch f := f.(type) {
+		case *http2.SettingsFrame:
+			if f.IsAck() { // HTTP/2 handshake completed.
+				return
+			}
+		default:
+			ct.t.Errorf("during greet, unexpected frame type %T", f)
+		}
+	}
+}
+
+func (ct *clientTester) wantClientPreface() {
+	preface := make([]byte, len(clientPreface))
+	if _, err := io.ReadFull(ct.conn, preface); err != nil {
+		ct.t.Errorf("Error at server-side while reading preface from client. Err: %v", err)
+	}
+	if !bytes.Equal(preface, clientPreface) {
+		ct.t.Errorf("received bogus greeting from client %q", preface)
+	}
+}
+
+func (ct *clientTester) wantSettingsFrame() {
+	frame, err := ct.fr.ReadFrame()
+	if err != nil {
+		ct.t.Errorf("error reading initial settings frame from client: %v", err)
+	}
+	_, ok := frame.(*http2.SettingsFrame)
+	if !ok {
+		ct.t.Errorf("initial frame sent from client is not a settings frame, type %T", frame)
+	}
+}
+
+func (ct *clientTester) writeSettingsFrame() {
+	if err := ct.fr.WriteSettings(); err != nil {
+		ct.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err)
+	}
+}
+
+func (ct *clientTester) writeSettingsAck() {
+	if err := ct.fr.WriteSettingsAck(); err != nil {
+		ct.t.Fatalf("Error writing ACK of client's SETTINGS: %v", err)
+	}
+}
+
+func (ct *clientTester) writeGoAway(maxStreamID uint32, code http2.ErrCode, debugData []byte) {
+	if err := ct.fr.WriteGoAway(maxStreamID, code, debugData); err != nil {
+		ct.t.Fatalf("Error writing GOAWAY: %v", err)
+	}
+}

test/codec_perf/perf.pb.go

@@ -0,0 +1,142 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Messages used for performance tests that may not reference grpc directly for
+// reasons of import cycles.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.36.12
+// 	protoc        v5.27.1
+// source: test/codec_perf/perf.proto
+
+package codec_perf
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+	unsafe "unsafe"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// Buffer is a message that contains a body of bytes that is used to exercise
+// encoding and decoding overheads.
+type Buffer struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Body          []byte                 `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Buffer) Reset() {
+	*x = Buffer{}
+	mi := &file_test_codec_perf_perf_proto_msgTypes[0]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Buffer) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Buffer) ProtoMessage() {}
+
+func (x *Buffer) ProtoReflect() protoreflect.Message {
+	mi := &file_test_codec_perf_perf_proto_msgTypes[0]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Buffer.ProtoReflect.Descriptor instead.
+func (*Buffer) Descriptor() ([]byte, []int) {
+	return file_test_codec_perf_perf_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *Buffer) GetBody() []byte {
+	if x != nil {
+		return x.Body
+	}
+	return nil
+}
+
+var File_test_codec_perf_perf_proto protoreflect.FileDescriptor
+
+const file_test_codec_perf_perf_proto_rawDesc = "" +
+	"\n" +
+	"\x1atest/codec_perf/perf.proto\x12\n" +
+	"codec.perf\"\x1c\n" +
+	"\x06Buffer\x12\x12\n" +
+	"\x04body\x18\x01 \x01(\fR\x04bodyB(Z&google.golang.org/grpc/test/codec_perfb\x06proto3"
+
+var (
+	file_test_codec_perf_perf_proto_rawDescOnce sync.Once
+	file_test_codec_perf_perf_proto_rawDescData []byte
+)
+
+func file_test_codec_perf_perf_proto_rawDescGZIP() []byte {
+	file_test_codec_perf_perf_proto_rawDescOnce.Do(func() {
+		file_test_codec_perf_perf_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_test_codec_perf_perf_proto_rawDesc), len(file_test_codec_perf_perf_proto_rawDesc)))
+	})
+	return file_test_codec_perf_perf_proto_rawDescData
+}
+
+var file_test_codec_perf_perf_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
+var file_test_codec_perf_perf_proto_goTypes = []any{
+	(*Buffer)(nil), // 0: codec.perf.Buffer
+}
+var file_test_codec_perf_perf_proto_depIdxs = []int32{
+	0, // [0:0] is the sub-list for method output_type
+	0, // [0:0] is the sub-list for method input_type
+	0, // [0:0] is the sub-list for extension type_name
+	0, // [0:0] is the sub-list for extension extendee
+	0, // [0:0] is the sub-list for field type_name
+}
+
+func init() { file_test_codec_perf_perf_proto_init() }
+func file_test_codec_perf_perf_proto_init() {
+	if File_test_codec_perf_perf_proto != nil {
+		return
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: unsafe.Slice(unsafe.StringData(file_test_codec_perf_perf_proto_rawDesc), len(file_test_codec_perf_perf_proto_rawDesc)),
+			NumEnums:      0,
+			NumMessages:   1,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_test_codec_perf_perf_proto_goTypes,
+		DependencyIndexes: file_test_codec_perf_perf_proto_depIdxs,
+		MessageInfos:      file_test_codec_perf_perf_proto_msgTypes,
+	}.Build()
+	File_test_codec_perf_perf_proto = out.File
+	file_test_codec_perf_perf_proto_goTypes = nil
+	file_test_codec_perf_perf_proto_depIdxs = nil
+}

test/codec_perf/perf.proto

@@ -0,0 +1,27 @@
+// Copyright 2017 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Messages used for performance tests that may not reference grpc directly for
+// reasons of import cycles.
+syntax = "proto3";
+
+option go_package = "google.golang.org/grpc/test/codec_perf";
+
+package codec.perf;
+
+// Buffer is a message that contains a body of bytes that is used to exercise
+// encoding and decoding overheads.
+message Buffer {
+  bytes body = 1;
+}

test/compressor_test.go

@@ -0,0 +1,694 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/experimental"
+	"google.golang.org/grpc/internal/grpcutil"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestUnsupportedEncodingResponse validates gRPC status codes
+// for different client-server compression setups
+// ensuring the correct behavior when compression is enabled or disabled on either side.
+func (s) TestUnsupportedEncodingResponse(t *testing.T) {
+	tests := []struct {
+		name           string
+		clientCompress bool
+		serverCompress bool
+		wantStatus     codes.Code
+	}{
+		{
+			name:           "client_server_compression",
+			clientCompress: true,
+			serverCompress: true,
+			wantStatus:     codes.OK,
+		},
+		{
+			name:           "client_compression",
+			clientCompress: true,
+			serverCompress: false,
+			wantStatus:     codes.Unimplemented,
+		},
+		{
+			name:           "server_compression",
+			clientCompress: false,
+			serverCompress: true,
+			wantStatus:     codes.Internal,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+					return &testpb.SimpleResponse{Payload: in.Payload}, nil
+				},
+			}
+			sopts := []grpc.ServerOption{}
+			if test.serverCompress {
+				// Using deprecated methods to selectively apply compression
+				// only on the server side. With encoding.registerCompressor(),
+				// the compressor is applied globally, affecting client and server
+				sopts = append(sopts, grpc.RPCCompressor(newNopCompressor()), grpc.RPCDecompressor(newNopDecompressor()))
+			}
+			if err := ss.StartServer(sopts...); err != nil {
+				t.Fatalf("Error starting server: %v", err)
+			}
+			defer ss.Stop()
+
+			dopts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+			if test.clientCompress {
+				// UseCompressor() requires the compressor to be registered
+				// using encoding.RegisterCompressor() which applies compressor globally,
+				// Hence, using deprecated WithCompressor() and WithDecompressor()
+				// to apply compression only on client.
+				dopts = append(dopts, grpc.WithCompressor(newNopCompressor()), grpc.WithDecompressor(newNopDecompressor()))
+			}
+			if err := ss.StartClient(dopts...); err != nil {
+				t.Fatalf("Error starting client: %v", err)
+			}
+
+			payload := &testpb.SimpleRequest{
+				Payload: &testpb.Payload{
+					Body: []byte("test message"),
+				},
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			_, err := ss.Client.UnaryCall(ctx, payload)
+			if got, want := status.Code(err), test.wantStatus; got != want {
+				t.Errorf("Client.UnaryCall() = %v, want %v", got, want)
+			}
+		})
+	}
+}
+
+func (s) TestCompressServerHasNoSupport(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testCompressServerHasNoSupport(t, e)
+	}
+}
+
+func testCompressServerHasNoSupport(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.serverCompression = false
+	te.clientCompression = false
+	te.clientNopCompression = true
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
+
+	const argSize = 271828
+	const respSize = 314159
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.SimpleRequest{
+		ResponseType: testpb.PayloadType_COMPRESSABLE,
+		ResponseSize: respSize,
+		Payload:      payload,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.Unimplemented {
+		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code %s", err, codes.Unimplemented)
+	}
+	// Streaming RPC
+	stream, err := tc.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	if _, err := stream.Recv(); err == nil || status.Code(err) != codes.Unimplemented {
+		t.Fatalf("%v.Recv() = %v, want error code %s", stream, err, codes.Unimplemented)
+	}
+}
+
+func (s) TestCompressOK(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testCompressOK(t, e)
+	}
+}
+
+func testCompressOK(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.serverCompression = true
+	te.clientCompression = true
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
+
+	// Unary call
+	const argSize = 271828
+	const respSize = 314159
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.SimpleRequest{
+		ResponseType: testpb.PayloadType_COMPRESSABLE,
+		ResponseSize: respSize,
+		Payload:      payload,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something"))
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
+		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, <nil>", err)
+	}
+	// Streaming RPC
+	stream, err := tc.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	respParam := []*testpb.ResponseParameters{
+		{
+			Size: 31415,
+		},
+	}
+	payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sreq := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: respParam,
+		Payload:            payload,
+	}
+	if err := stream.Send(sreq); err != nil {
+		t.Fatalf("%v.Send(%v) = %v, want <nil>", stream, sreq, err)
+	}
+	stream.CloseSend()
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("%v.Recv() = %v, want <nil>", stream, err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err)
+	}
+}
+
+func (s) TestIdentityEncoding(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testIdentityEncoding(t, e)
+	}
+}
+
+func testIdentityEncoding(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
+
+	// Unary call
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.SimpleRequest{
+		ResponseType: testpb.PayloadType_COMPRESSABLE,
+		ResponseSize: 10,
+		Payload:      payload,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something"))
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
+		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, <nil>", err)
+	}
+	// Streaming RPC
+	stream, err := tc.FullDuplexCall(ctx, grpc.UseCompressor("identity"))
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sreq := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: []*testpb.ResponseParameters{{Size: 10}},
+		Payload:            payload,
+	}
+	if err := stream.Send(sreq); err != nil {
+		t.Fatalf("%v.Send(%v) = %v, want <nil>", stream, sreq, err)
+	}
+	stream.CloseSend()
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("%v.Recv() = %v, want <nil>", stream, err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err)
+	}
+}
+
+// renameCompressor is a grpc.Compressor wrapper that allows customizing the
+// Type() of another compressor.
+type renameCompressor struct {
+	grpc.Compressor
+	name string
+}
+
+func (r *renameCompressor) Type() string { return r.name }
+
+// renameDecompressor is a grpc.Decompressor wrapper that allows customizing the
+// Type() of another Decompressor.
+type renameDecompressor struct {
+	grpc.Decompressor
+	name string
+}
+
+func (r *renameDecompressor) Type() string { return r.name }
+
+func (s) TestClientForwardsGrpcAcceptEncodingHeader(t *testing.T) {
+	wantGrpcAcceptEncodingCh := make(chan []string, 1)
+	defer close(wantGrpcAcceptEncodingCh)
+
+	compressor := renameCompressor{Compressor: grpc.NewGZIPCompressor(), name: "testgzip"}
+	decompressor := renameDecompressor{Decompressor: grpc.NewGZIPDecompressor(), name: "testgzip"}
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			md, ok := metadata.FromIncomingContext(ctx)
+			if !ok {
+				return nil, status.Errorf(codes.Internal, "no metadata in context")
+			}
+			if got, want := md["grpc-accept-encoding"], <-wantGrpcAcceptEncodingCh; !reflect.DeepEqual(got, want) {
+				return nil, status.Errorf(codes.Internal, "got grpc-accept-encoding=%q; want [%q]", got, want)
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start([]grpc.ServerOption{grpc.RPCDecompressor(&decompressor)}); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	wantGrpcAcceptEncodingCh <- []string{"gzip"}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+
+	wantGrpcAcceptEncodingCh <- []string{"gzip"}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.UseCompressor("gzip")); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+
+	// Use compressor directly which is not registered via
+	// encoding.RegisterCompressor.
+	if err := ss.StartClient(grpc.WithCompressor(&compressor)); err != nil {
+		t.Fatalf("Error starting client: %v", err)
+	}
+	wantGrpcAcceptEncodingCh <- []string{"gzip,testgzip"}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
+	}
+}
+
+func (s) TestUnregisteredSetSendCompressorFailure(t *testing.T) {
+	resCompressor := "snappy2"
+	wantErr := status.Error(codes.Unknown, "unable to set send compressor: compressor not registered \"snappy2\"")
+
+	t.Run("unary", func(t *testing.T) {
+		testUnarySetSendCompressorFailure(t, resCompressor, wantErr)
+	})
+
+	t.Run("stream", func(t *testing.T) {
+		testStreamSetSendCompressorFailure(t, resCompressor, wantErr)
+	})
+}
+
+func testUnarySetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil {
+				return nil, err
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) {
+		t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr)
+	}
+}
+
+func testStreamSetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			if _, err := stream.Recv(); err != nil {
+				return err
+			}
+
+			if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil {
+				return err
+			}
+
+			return stream.Send(&testpb.StreamingOutputCallResponse{})
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v, want: nil", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	s, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err)
+	}
+
+	if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+		t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err)
+	}
+
+	if _, err := s.Recv(); !equalError(err, wantErr) {
+		t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err)
+	}
+}
+
+func (s) TestUnarySetSendCompressorAfterHeaderSendFailure(t *testing.T) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			// Send headers early and then set send compressor.
+			grpc.SendHeader(ctx, metadata.MD{})
+			err := grpc.SetSendCompressor(ctx, "gzip")
+			if err == nil {
+				t.Error("Wanted set send compressor error")
+				return &testpb.Empty{}, nil
+			}
+			return nil, err
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done")
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) {
+		t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr)
+	}
+}
+
+func (s) TestStreamSetSendCompressorAfterHeaderSendFailure(t *testing.T) {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			// Send headers early and then set send compressor.
+			grpc.SendHeader(stream.Context(), metadata.MD{})
+			err := grpc.SetSendCompressor(stream.Context(), "gzip")
+			if err == nil {
+				t.Error("Wanted set send compressor error")
+			}
+			return err
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done")
+	s, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err)
+	}
+
+	if _, err := s.Recv(); !equalError(err, wantErr) {
+		t.Fatalf("Unexpected full duplex recv error, got: %v, want: %v", err, wantErr)
+	}
+}
+
+func (s) TestClientSupportedCompressors(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	for _, tt := range []struct {
+		desc string
+		ctx  context.Context
+		want []string
+	}{
+		{
+			desc: "No additional grpc-accept-encoding header",
+			ctx:  ctx,
+			want: []string{"gzip"},
+		},
+		{
+			desc: "With additional grpc-accept-encoding header",
+			ctx: metadata.AppendToOutgoingContext(ctx,
+				"grpc-accept-encoding", "test-compressor-1",
+				"grpc-accept-encoding", "test-compressor-2",
+			),
+			want: []string{"gzip", "test-compressor-1", "test-compressor-2"},
+		},
+		{
+			desc: "With additional empty grpc-accept-encoding header",
+			ctx: metadata.AppendToOutgoingContext(ctx,
+				"grpc-accept-encoding", "",
+			),
+			want: []string{"gzip"},
+		},
+		{
+			desc: "With additional grpc-accept-encoding header with spaces between values",
+			ctx: metadata.AppendToOutgoingContext(ctx,
+				"grpc-accept-encoding", "identity, deflate",
+			),
+			want: []string{"gzip", "identity", "deflate"},
+		},
+	} {
+		t.Run(tt.desc, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+					got, err := grpc.ClientSupportedCompressors(ctx)
+					if err != nil {
+						return nil, err
+					}
+
+					if !reflect.DeepEqual(got, tt.want) {
+						t.Errorf("unexpected client compressors got: %v, want: %v", got, tt.want)
+					}
+
+					return &testpb.Empty{}, nil
+				},
+			}
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("Error starting endpoint server: %v, want: nil", err)
+			}
+			defer ss.Stop()
+
+			_, err := ss.Client.EmptyCall(tt.ctx, &testpb.Empty{})
+			if err != nil {
+				t.Fatalf("Unexpected unary call error, got: %v, want: nil", err)
+			}
+		})
+	}
+}
+
+func (s) TestAcceptCompressorsCallOption(t *testing.T) {
+	tests := []struct {
+		name       string
+		callOption grpc.CallOption
+		wantHeader string
+	}{
+		{
+			name:       "with AcceptCompressors",
+			callOption: experimental.AcceptCompressors("gzip"),
+			wantHeader: "gzip",
+		},
+		{
+			name:       "without AcceptCompressors uses default",
+			callOption: nil,
+			wantHeader: grpcutil.RegisteredCompressors(),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+					md, _ := metadata.FromIncomingContext(ctx)
+					header := md.Get("grpc-accept-encoding")
+
+					if len(header) != 1 || header[0] != tt.wantHeader {
+						t.Errorf("unexpected grpc-accept-encoding header: got %v, want %v", header, tt.wantHeader)
+					}
+					return &testpb.Empty{}, nil
+				},
+			}
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("failed to start server: %v", err)
+			}
+			defer ss.Stop()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			opts := []grpc.CallOption{}
+			if tt.callOption != nil {
+				opts = append(opts, tt.callOption)
+			}
+
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, opts...); err != nil {
+				t.Fatalf("EmptyCall failed: %v", err)
+			}
+		})
+	}
+}
+
+func (s) TestCompressorRegister(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testCompressorRegister(t, e)
+	}
+}
+
+func testCompressorRegister(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.clientCompression = false
+	te.serverCompression = false
+	te.clientUseCompression = true
+
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	tc := testgrpc.NewTestServiceClient(te.clientConn())
+
+	// Unary call
+	const argSize = 271828
+	const respSize = 314159
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.SimpleRequest{
+		ResponseType: testpb.PayloadType_COMPRESSABLE,
+		ResponseSize: respSize,
+		Payload:      payload,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something"))
+	if _, err := tc.UnaryCall(ctx, req); err != nil {
+		t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, <nil>", err)
+	}
+	// Streaming RPC
+	stream, err := tc.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	respParam := []*testpb.ResponseParameters{
+		{
+			Size: 31415,
+		},
+	}
+	payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sreq := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: respParam,
+		Payload:            payload,
+	}
+	if err := stream.Send(sreq); err != nil {
+		t.Fatalf("%v.Send(%v) = %v, want <nil>", stream, sreq, err)
+	}
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("%v.Recv() = %v, want <nil>", stream, err)
+	}
+}
+
+type badGzipCompressor struct{}
+
+func (badGzipCompressor) Do(w io.Writer, p []byte) error {
+	buf := &bytes.Buffer{}
+	gzw := gzip.NewWriter(buf)
+	if _, err := gzw.Write(p); err != nil {
+		return err
+	}
+	err := gzw.Close()
+	bs := buf.Bytes()
+	if len(bs) >= 6 {
+		bs[len(bs)-6] ^= 1 // modify checksum at end by 1 byte
+	}
+	w.Write(bs)
+	return err
+}
+
+func (badGzipCompressor) Type() string {
+	return "gzip"
+}
+
+func (s) TestGzipBadChecksum(t *testing.T) {
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+	}
+	if err := ss.Start(nil, grpc.WithCompressor(badGzipCompressor{})); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	p, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1024))
+	if err != nil {
+		t.Fatalf("Unexpected error from newPayload: %v", err)
+	}
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil ||
+		status.Code(err) != codes.Internal ||
+		!strings.Contains(status.Convert(err).Message(), gzip.ErrChecksum.Error()) {
+		t.Errorf("ss.Client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum)
+	}
+}

test/config_selector_test.go

@@ -0,0 +1,216 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/codes"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/serviceconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+)
+
+type funcConfigSelector struct {
+	f func(iresolver.RPCInfo) (*iresolver.RPCConfig, error)
+}
+
+func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
+	return f.f(i)
+}
+
+func (s) TestConfigSelector(t *testing.T) {
+	gotContextChan := testutils.NewChannelWithSize(1)
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			gotContextChan.SendContext(ctx, ctx)
+			return &testpb.Empty{}, nil
+		},
+	}
+	ss.R = manual.NewBuilderWithScheme("confSel")
+
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	const normalTimeout = 10 * time.Second
+	ctxDeadline := time.Now().Add(normalTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), normalTimeout)
+	defer cancel()
+
+	const longTimeout = 30 * time.Second
+	longCtxDeadline := time.Now().Add(longTimeout)
+	longdeadlineCtx, cancel := context.WithTimeout(context.Background(), longTimeout)
+	defer cancel()
+	shorterTimeout := 3 * time.Second
+
+	testMD := metadata.MD{"footest": []string{"bazbar"}}
+	mdOut := metadata.MD{"handler": []string{"value"}}
+
+	var onCommittedCalled bool
+
+	testCases := []struct {
+		name   string
+		md     metadata.MD          // MD sent with RPC
+		config *iresolver.RPCConfig // config returned by config selector
+		csErr  error                // error returned by config selector
+
+		wantMD       metadata.MD
+		wantDeadline time.Time
+		wantTimeout  time.Duration
+		wantErr      error
+	}{{
+		name:         "basic",
+		md:           testMD,
+		config:       &iresolver.RPCConfig{},
+		wantMD:       testMD,
+		wantDeadline: ctxDeadline,
+	}, {
+		name: "alter MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			Context: metadata.NewOutgoingContext(ctx, mdOut),
+		},
+		wantMD:       mdOut,
+		wantDeadline: ctxDeadline,
+	}, {
+		name:    "erroring SelectConfig",
+		csErr:   status.Errorf(codes.Unavailable, "cannot send RPC"),
+		wantErr: status.Errorf(codes.Unavailable, "cannot send RPC"),
+	}, {
+		name: "alter timeout; remove MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			Context: longdeadlineCtx, // no metadata
+		},
+		wantMD:       nil,
+		wantDeadline: longCtxDeadline,
+	}, {
+		name:         "nil config",
+		md:           metadata.MD{},
+		config:       nil,
+		wantMD:       nil,
+		wantDeadline: ctxDeadline,
+	}, {
+		name: "alter timeout via method config; remove MD",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			MethodConfig: serviceconfig.MethodConfig{
+				Timeout: &shorterTimeout,
+			},
+		},
+		wantMD:      nil,
+		wantTimeout: shorterTimeout,
+	}, {
+		name: "onCommitted callback",
+		md:   testMD,
+		config: &iresolver.RPCConfig{
+			OnCommitted: func() {
+				onCommittedCalled = true
+			},
+		},
+		wantMD:       testMD,
+		wantDeadline: ctxDeadline,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var gotInfo *iresolver.RPCInfo
+			state := iresolver.SetConfigSelector(resolver.State{
+				Addresses:     []resolver.Address{{Addr: ss.Address}},
+				ServiceConfig: parseServiceConfig(t, ss.R, "{}"),
+			}, funcConfigSelector{
+				f: func(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
+					gotInfo = &i
+					cfg := tc.config
+					if cfg != nil && cfg.Context == nil {
+						cfg.Context = i.Context
+					}
+					return cfg, tc.csErr
+				},
+			})
+			ss.R.UpdateState(state) // Blocks until config selector is applied
+
+			onCommittedCalled = false
+			ctx := metadata.NewOutgoingContext(ctx, tc.md)
+			startTime := time.Now()
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); fmt.Sprint(err) != fmt.Sprint(tc.wantErr) {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.wantErr)
+			} else if err != nil {
+				return // remaining checks are invalid
+			}
+
+			if gotInfo == nil {
+				t.Fatalf("no config selector data")
+			}
+
+			if want := "/grpc.testing.TestService/EmptyCall"; gotInfo.Method != want {
+				t.Errorf("gotInfo.Method = %q; want %q", gotInfo.Method, want)
+			}
+
+			gotContextI, ok := gotContextChan.ReceiveOrFail()
+			if !ok {
+				t.Fatalf("no context received")
+			}
+			gotContext := gotContextI.(context.Context)
+
+			gotMD, _ := metadata.FromOutgoingContext(gotInfo.Context)
+			if diff := cmp.Diff(tc.md, gotMD); diff != "" {
+				t.Errorf("gotInfo.Context contains MD %v; want %v\nDiffs: %v", gotMD, tc.md, diff)
+			}
+
+			gotMD, _ = metadata.FromIncomingContext(gotContext)
+			// Remove entries from gotMD not in tc.wantMD (e.g. authority header).
+			for k := range gotMD {
+				if _, ok := tc.wantMD[k]; !ok {
+					delete(gotMD, k)
+				}
+			}
+			if diff := cmp.Diff(tc.wantMD, gotMD, cmpopts.EquateEmpty()); diff != "" {
+				t.Errorf("received md = %v; want %v\nDiffs: %v", gotMD, tc.wantMD, diff)
+			}
+
+			wantDeadline := tc.wantDeadline
+			if wantDeadline.Equal(time.Time{}) {
+				wantDeadline = startTime.Add(tc.wantTimeout)
+			}
+			deadlineGot, _ := gotContext.Deadline()
+			if diff := deadlineGot.Sub(wantDeadline); diff > time.Second || diff < -time.Second {
+				t.Errorf("received deadline = %v; want ~%v", deadlineGot, wantDeadline)
+			}
+
+			if tc.config != nil && tc.config.OnCommitted != nil && !onCommittedCalled {
+				t.Errorf("OnCommitted callback not called")
+			}
+		})
+	}
+}

test/context_canceled_test.go

@@ -0,0 +1,164 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/encoding/gzip"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestContextCanceled(t *testing.T) {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			stream.SetTrailer(metadata.New(map[string]string{"a": "b"}))
+			return status.Error(codes.PermissionDenied, "perm denied")
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	// Runs 10 rounds of tests with the given delay and returns counts of status codes.
+	// Fails in case of trailer/status code inconsistency.
+	const cntRetry uint = 10
+	runTest := func(delay time.Duration) (cntCanceled, cntPermDenied uint) {
+		for i := uint(0); i < cntRetry; i++ {
+			ctx, cancel := context.WithTimeout(context.Background(), delay)
+			defer cancel()
+
+			str, err := ss.Client.FullDuplexCall(ctx)
+			if err != nil {
+				continue
+			}
+
+			_, err = str.Recv()
+			if err == nil {
+				t.Fatalf("non-nil error expected from Recv()")
+			}
+
+			_, trlOk := str.Trailer()["a"]
+			switch status.Code(err) {
+			case codes.PermissionDenied:
+				if !trlOk {
+					t.Fatalf(`status err: %v; wanted key "a" in trailer but didn't get it`, err)
+				}
+				cntPermDenied++
+			case codes.DeadlineExceeded:
+				if trlOk {
+					t.Fatalf(`status err: %v; didn't want key "a" in trailer but got it`, err)
+				}
+				cntCanceled++
+			default:
+				t.Fatalf(`unexpected status err: %v`, err)
+			}
+		}
+		return cntCanceled, cntPermDenied
+	}
+
+	// Tries to find the delay that causes canceled/perm denied race.
+	canceledOk, permDeniedOk := false, false
+	for lower, upper := time.Duration(0), 2*time.Millisecond; lower <= upper; {
+		delay := lower + (upper-lower)/2
+		cntCanceled, cntPermDenied := runTest(delay)
+		if cntPermDenied > 0 && cntCanceled > 0 {
+			// Delay that causes the race is found.
+			return
+		}
+
+		// Set OK flags.
+		if cntCanceled > 0 {
+			canceledOk = true
+		}
+		if cntPermDenied > 0 {
+			permDeniedOk = true
+		}
+
+		if cntPermDenied == 0 {
+			// No perm denied, increase the delay.
+			lower += (upper-lower)/10 + 1
+		} else {
+			// All perm denied, decrease the delay.
+			upper -= (upper-lower)/10 + 1
+		}
+	}
+
+	if !canceledOk || !permDeniedOk {
+		t.Fatalf(`couldn't find the delay that causes canceled/perm denied race.`)
+	}
+}
+
+// To make sure that canceling a stream with compression enabled won't result in
+// internal error, compressed flag set with identity or empty encoding.
+//
+// The root cause is a select race on stream headerChan and ctx. Stream gets
+// whether compression is enabled and the compression type from two separate
+// functions, both include select with context. If the `case non-ctx:` wins the
+// first one, but `case ctx.Done()` wins the second one, the compression info
+// will be inconsistent, and it causes internal error.
+func (s) TestCancelWhileRecvingWithCompression(t *testing.T) {
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				if err := stream.Send(&testpb.StreamingOutputCallResponse{
+					Payload: nil,
+				}); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	for i := 0; i < 10; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		s, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name))
+		if err != nil {
+			t.Fatalf("failed to start bidi streaming RPC: %v", err)
+		}
+		// Cancel the stream while receiving to trigger the internal error.
+		time.AfterFunc(time.Millisecond, cancel)
+		for {
+			_, err := s.Recv()
+			if err != nil {
+				if status.Code(err) != codes.Canceled {
+					t.Fatalf("recv failed with %v, want Canceled", err)
+				}
+				break
+			}
+		}
+	}
+	if err := ss.CC.Close(); err != nil {
+		t.Fatalf("Close failed with %v, want nil", err)
+	}
+}

test/control_plane_status_test.go

@@ -0,0 +1,234 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/stubserver"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+)
+
+func (s) TestConfigSelectorStatusCodes(t *testing.T) {
+	testCases := []struct {
+		name  string
+		csErr error
+		want  error
+	}{{
+		name:  "legal status code",
+		csErr: status.Errorf(codes.Unavailable, "this error is fine"),
+		want:  status.Errorf(codes.Unavailable, "this error is fine"),
+	}, {
+		name:  "illegal status code",
+		csErr: status.Errorf(codes.NotFound, "this error is bad"),
+		want:  status.Errorf(codes.Internal, "this error is bad"),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+					return &testpb.Empty{}, nil
+				},
+			}
+			ss.R = manual.NewBuilderWithScheme("confSel")
+
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			state := iresolver.SetConfigSelector(resolver.State{
+				Addresses:     []resolver.Address{{Addr: ss.Address}},
+				ServiceConfig: parseServiceConfig(t, ss.R, "{}"),
+			}, funcConfigSelector{
+				f: func(iresolver.RPCInfo) (*iresolver.RPCConfig, error) {
+					return nil, tc.csErr
+				},
+			})
+			ss.R.UpdateState(state) // Blocks until config selector is applied
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want)
+			}
+		})
+	}
+}
+
+func (s) TestPickerStatusCodes(t *testing.T) {
+	testCases := []struct {
+		name      string
+		pickerErr error
+		want      error
+	}{{
+		name:      "legal status code",
+		pickerErr: status.Errorf(codes.Unavailable, "this error is fine"),
+		want:      status.Errorf(codes.Unavailable, "this error is fine"),
+	}, {
+		name:      "illegal status code",
+		pickerErr: status.Errorf(codes.NotFound, "this error is bad"),
+		want:      status.Errorf(codes.Internal, "this error is bad"),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			// Create a stub balancer that creates a picker that always returns
+			// an error.
+			sbf := stub.BalancerFuncs{
+				UpdateClientConnState: func(d *stub.BalancerData, _ balancer.ClientConnState) error {
+					d.ClientConn.UpdateState(balancer.State{
+						ConnectivityState: connectivity.TransientFailure,
+						Picker:            base.NewErrPicker(tc.pickerErr),
+					})
+					return nil
+				},
+			}
+			stub.Register("testPickerStatusCodesBalancer", sbf)
+
+			ss.NewServiceConfig(`{"loadBalancingConfig": [{"testPickerStatusCodesBalancer":{}}] }`)
+
+			// Make calls until pickerErr is received.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			var lastErr error
+			for ctx.Err() == nil {
+				if _, lastErr = ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(lastErr) == status.Code(tc.want) && strings.Contains(lastErr.Error(), status.Convert(tc.want).Message()) {
+					// Success!
+					return
+				}
+				time.Sleep(time.Millisecond)
+			}
+
+			t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", lastErr, tc.want)
+		})
+	}
+}
+
+func (s) TestCallCredsFromDialOptionsStatusCodes(t *testing.T) {
+	testCases := []struct {
+		name     string
+		credsErr error
+		want     error
+	}{{
+		name:     "legal status code",
+		credsErr: status.Errorf(codes.Unavailable, "this error is fine"),
+		want:     status.Errorf(codes.Unavailable, "this error is fine"),
+	}, {
+		name:     "illegal status code",
+		credsErr: status.Errorf(codes.NotFound, "this error is bad"),
+		want:     status.Errorf(codes.Internal, "this error is bad"),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			errChan := make(chan error, 1)
+			creds := &testPerRPCCredentials{errChan: errChan}
+
+			if err := ss.Start(nil, grpc.WithPerRPCCredentials(creds)); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			errChan <- tc.credsErr
+
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want)
+			}
+		})
+	}
+}
+
+func (s) TestCallCredsFromCallOptionsStatusCodes(t *testing.T) {
+	testCases := []struct {
+		name     string
+		credsErr error
+		want     error
+	}{{
+		name:     "legal status code",
+		credsErr: status.Errorf(codes.Unavailable, "this error is fine"),
+		want:     status.Errorf(codes.Unavailable, "this error is fine"),
+	}, {
+		name:     "illegal status code",
+		credsErr: status.Errorf(codes.NotFound, "this error is bad"),
+		want:     status.Errorf(codes.Internal, "this error is bad"),
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			ss := &stubserver.StubServer{
+				EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			errChan := make(chan error, 1)
+			creds := &testPerRPCCredentials{errChan: errChan}
+
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			errChan <- tc.credsErr
+
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(creds)); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) {
+				t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want)
+			}
+		})
+	}
+}

test/creds_test.go

@@ -0,0 +1,626 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/tap"
+	"google.golang.org/grpc/testdata"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const (
+	bundlePerRPCOnly = "perRPCOnly"
+	bundleTLSOnly    = "tlsOnly"
+)
+
+type testCredsBundle struct {
+	t    *testing.T
+	mode string
+}
+
+func (c *testCredsBundle) TransportCredentials() credentials.TransportCredentials {
+	if c.mode == bundlePerRPCOnly {
+		return insecure.NewCredentials()
+	}
+
+	creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com")
+	if err != nil {
+		c.t.Logf("Failed to load credentials: %v", err)
+		return nil
+	}
+	return creds
+}
+
+func (c *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials {
+	if c.mode == bundleTLSOnly {
+		return nil
+	}
+	return testPerRPCCredentials{authdata: authdata}
+}
+
+func (c *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) {
+	return &testCredsBundle{mode: mode}, nil
+}
+
+func (s) TestCredsBundleBoth(t *testing.T) {
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
+	te.tapHandle = authHandle
+	te.customDialOptions = []grpc.DialOption{
+		grpc.WithCredentialsBundle(&testCredsBundle{t: t}),
+		grpc.WithAuthority("x.test.example.com"),
+	}
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("Failed to generate credentials %v", err)
+	}
+	te.customServerOptions = []grpc.ServerOption{
+		grpc.Creds(creds),
+	}
+	te.startServer(&testServer{})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestCredsBundleTransportCredentials(t *testing.T) {
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
+	te.customDialOptions = []grpc.DialOption{
+		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundleTLSOnly}),
+		grpc.WithAuthority("x.test.example.com"),
+	}
+	creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem"))
+	if err != nil {
+		t.Fatalf("Failed to generate credentials %v", err)
+	}
+	te.customServerOptions = []grpc.ServerOption{
+		grpc.Creds(creds),
+	}
+	te.startServer(&testServer{})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestCredsBundlePerRPCCredentials(t *testing.T) {
+	te := newTest(t, env{name: "creds-bundle", network: "tcp", security: "empty"})
+	te.tapHandle = authHandle
+	te.customDialOptions = []grpc.DialOption{
+		grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}),
+	}
+	te.startServer(&testServer{})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+type clientTimeoutCreds struct {
+	credentials.TransportCredentials
+	timeoutReturned bool
+}
+
+func (c *clientTimeoutCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	if !c.timeoutReturned {
+		c.timeoutReturned = true
+		return nil, nil, context.DeadlineExceeded
+	}
+	return rawConn, nil, nil
+}
+
+func (c *clientTimeoutCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+
+func (c *clientTimeoutCreds) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) {
+	te := newTest(t, env{name: "timeout-cred", network: "tcp", security: "empty"})
+	te.userAgent = testAppUA
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{}))
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// This unary call should succeed, because ClientHandshake will succeed for the second time.
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want <nil>", err)
+	}
+}
+
+type methodTestCreds struct{}
+
+func (m *methodTestCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) {
+	ri, _ := credentials.RequestInfoFromContext(ctx)
+	return nil, status.Error(codes.Unknown, ri.Method)
+}
+
+func (m *methodTestCreds) RequireTransportSecurity() bool { return false }
+
+func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) {
+	const wantMethod = "/grpc.testing.TestService/EmptyCall"
+	te := newTest(t, env{name: "context-request-info", network: "tcp"})
+	te.userAgent = testAppUA
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{}))
+	tc := testgrpc.NewTestServiceClient(cc)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod {
+		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
+	}
+
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Convert(err).Message() != wantMethod {
+		t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod)
+	}
+}
+
+const clientAlwaysFailCredErrorMsg = "clientAlwaysFailCred always fails"
+
+type clientAlwaysFailCred struct {
+	credentials.TransportCredentials
+}
+
+func (c clientAlwaysFailCred) ClientHandshake(context.Context, string, net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return nil, nil, errors.New(clientAlwaysFailCredErrorMsg)
+}
+func (c clientAlwaysFailCred) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c clientAlwaysFailCred) Clone() credentials.TransportCredentials {
+	return nil
+}
+
+func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) {
+	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, err := grpc.NewClient(te.srvAddr, opts...)
+	if err != nil {
+		t.Fatalf("NewClient(_) = %v, want %v", err, nil)
+	}
+	defer cc.Close()
+
+	tc := testgrpc.NewTestServiceClient(cc)
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
+		return
+	}
+	te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
+}
+
+func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) {
+	te := newTest(t, env{name: "bad-cred", network: "tcp", security: "empty", balancer: "round_robin"})
+	te.startServer(&testServer{security: te.e.security})
+	defer te.tearDown()
+
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})}
+	cc, err := grpc.NewClient(te.srvAddr, opts...)
+	if err != nil {
+		t.Fatalf("NewClient(_) = %v, want %v", err, nil)
+	}
+	defer cc.Close()
+
+	// The DNS resolver may take more than defaultTestShortTimeout, we let the
+	// channel enter TransientFailure signalling that the first resolver state
+	// has been produced.
+	cc.Connect()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	tc := testgrpc.NewTestServiceClient(cc)
+	// Use a short context as WaitForReady waits for context expiration before
+	// failing the RPC.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer cancel()
+	if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); !strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) {
+		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg)
+	}
+}
+
+var (
+	// test authdata
+	authdata = map[string]string{
+		"test-key":      "test-value",
+		"test-key2-bin": string([]byte{1, 2, 3}),
+	}
+)
+
+type testPerRPCCredentials struct {
+	authdata map[string]string
+	errChan  chan error
+}
+
+func (cr testPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
+	var err error
+	if cr.errChan != nil {
+		err = <-cr.errChan
+	}
+	return cr.authdata, err
+}
+
+func (cr testPerRPCCredentials) RequireTransportSecurity() bool {
+	return false
+}
+
+func authHandle(ctx context.Context, _ *tap.Info) (context.Context, error) {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return ctx, fmt.Errorf("didn't find metadata in context")
+	}
+	for k, vwant := range authdata {
+		vgot, ok := md[k]
+		if !ok {
+			return ctx, fmt.Errorf("didn't find authdata key %v in context", k)
+		}
+		if vgot[0] != vwant {
+			return ctx, fmt.Errorf("for key %v, got value %v, want %v", k, vgot, vwant)
+		}
+	}
+	return ctx, nil
+}
+
+func (s) TestPerRPCCredentialsViaDialOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaDialOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.tapHandle = authHandle
+	te.perRPCCreds = testPerRPCCredentials{authdata: authdata}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestPerRPCCredentialsViaCallOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaCallOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.tapHandle = authHandle
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		testPerRPCCredentialsViaDialOptionsAndCallOptions(t, e)
+	}
+}
+
+func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.perRPCCreds = testPerRPCCredentials{authdata: authdata}
+	// When credentials are provided via both dial options and call options,
+	// we apply both sets.
+	te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) {
+		md, ok := metadata.FromIncomingContext(ctx)
+		if !ok {
+			return ctx, fmt.Errorf("couldn't find metadata in context")
+		}
+		for k, vwant := range authdata {
+			vgot, ok := md[k]
+			if !ok {
+				return ctx, fmt.Errorf("couldn't find metadata for key %v", k)
+			}
+			if len(vgot) != 2 {
+				return ctx, fmt.Errorf("len of value for key %v was %v, want 2", k, len(vgot))
+			}
+			if vgot[0] != vwant || vgot[1] != vwant {
+				return ctx, fmt.Errorf("value for %v was %v, want [%v, %v]", k, vgot, vwant, vwant)
+			}
+		}
+		return ctx, nil
+	}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil {
+		t.Fatalf("Test failed. Reason: %v", err)
+	}
+}
+
+func (s) TestPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		for _, test := range []struct {
+			name     string
+			authdata map[string]string
+			wantErr  string
+		}{
+			{
+				name:     "invalid key",
+				authdata: map[string]string{"bad key": "value"},
+				wantErr:  fmt.Sprintf("header key %q contains illegal characters not in [0-9a-z-_.]", "bad key"),
+			},
+			{
+				name:     "invalid value",
+				authdata: map[string]string{"valid-key": "bad\x01value"},
+				wantErr:  fmt.Sprintf("header key %q contains value with non-printable ASCII characters", "valid-key"),
+			},
+		} {
+			t.Run(e.name+"/"+test.name, func(t *testing.T) {
+				testPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t, e, test.authdata, test.wantErr)
+			})
+		}
+	}
+}
+
+func testPerRPCCredentialMetadataValidationRejectsInvalidDialOptions(t *testing.T, e env, authdata map[string]string, wantErr string) {
+	te := newTest(t, e)
+	te.perRPCCreds = testPerRPCCredentials{authdata: authdata}
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr)
+	}
+}
+
+func (s) TestPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t *testing.T) {
+	for _, e := range listTestEnv() {
+		for _, test := range []struct {
+			name     string
+			authdata map[string]string
+			wantErr  string
+		}{
+			{
+				name:     "invalid key",
+				authdata: map[string]string{"bad key": "value"},
+				wantErr:  fmt.Sprintf("header key %q contains illegal characters not in [0-9a-z-_.]", "bad key"),
+			},
+			{
+				name:     "invalid value",
+				authdata: map[string]string{"valid-key": "bad\x01value"},
+				wantErr:  fmt.Sprintf("header key %q contains value with non-printable ASCII characters", "valid-key"),
+			},
+		} {
+			t.Run(e.name+"/"+test.name, func(t *testing.T) {
+				testPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t, e, test.authdata, test.wantErr)
+			})
+		}
+	}
+}
+
+func testPerRPCCredentialMetadataValidationRejectsInvalidCallOptions(t *testing.T, e env, authdata map[string]string, wantErr string) {
+	te := newTest(t, e)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); status.Code(err) != codes.Internal || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() error = %v, want code %v and message containing %q", err, codes.Internal, wantErr)
+	}
+}
+
+const testAuthority = "test.auth.ori.ty"
+
+type authorityCheckCreds struct {
+	credentials.TransportCredentials
+	got string
+}
+
+func (c *authorityCheckCreds) ClientHandshake(_ context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	c.got = authority
+	return rawConn, nil, nil
+}
+func (c *authorityCheckCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *authorityCheckCreds) Clone() credentials.TransportCredentials {
+	return c
+}
+
+// This test makes sure that the authority client handshake gets is the endpoint
+// in dial target, not the resolved ip address.
+func (s) TestCredsHandshakeAuthority(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &authorityCheckCreds{}
+	s := grpc.NewServer()
+	go s.Serve(lis)
+	defer s.Stop()
+
+	r := manual.NewBuilderWithScheme("whatever")
+
+	cc, err := grpc.NewClient(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	if cred.got != testAuthority {
+		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
+	}
+}
+
+// This test makes sure that the authority client handshake gets is the endpoint
+// of the ServerName of the address when it is set.
+func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) {
+	const testServerName = "test.server.name"
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &authorityCheckCreds{}
+	s := grpc.NewServer()
+	go s.Serve(lis)
+	defer s.Stop()
+
+	r := manual.NewBuilderWithScheme("whatever")
+
+	cc, err := grpc.NewClient(r.Scheme()+":///"+testAuthority, grpc.WithTransportCredentials(cred), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String(), ServerName: testServerName}}})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+
+	if cred.got != testServerName {
+		t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority)
+	}
+}
+
+type serverDispatchCred struct {
+	rawConnCh chan net.Conn
+}
+
+func (c *serverDispatchCred) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return rawConn, nil, nil
+}
+func (c *serverDispatchCred) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	select {
+	case c.rawConnCh <- rawConn:
+	default:
+	}
+	return nil, nil, credentials.ErrConnDispatched
+}
+func (c *serverDispatchCred) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *serverDispatchCred) Clone() credentials.TransportCredentials {
+	return nil
+}
+func (c *serverDispatchCred) OverrideServerName(string) error {
+	return nil
+}
+func (c *serverDispatchCred) getRawConn() net.Conn {
+	return <-c.rawConnCh
+}
+
+func (s) TestServerCredsDispatch(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cred := &serverDispatchCred{
+		rawConnCh: make(chan net.Conn, 1),
+	}
+	s := grpc.NewServer(grpc.Creds(cred))
+	go s.Serve(lis)
+	defer s.Stop()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(cred))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	rawConn := cred.getRawConn()
+	// Give grpc a chance to see the error and potentially close the connection.
+	// And check that connection is not closed after that.
+	time.Sleep(100 * time.Millisecond)
+	// Check rawConn is not closed.
+	if n, err := rawConn.Write([]byte{0}); n <= 0 || err != nil {
+		t.Errorf("Read() = %v, %v; want n>0, <nil>", n, err)
+	}
+}

test/goaway_test.go

@@ -0,0 +1,823 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/keepalive"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestGracefulClientOnGoAway attempts to ensure that when the server sends a
+// GOAWAY (in this test, by configuring max connection age on the server), a
+// client will never see an error.  This requires that the client is appraised
+// of the GOAWAY and updates its state accordingly before the transport stops
+// accepting new streams.  If a subconn is chosen by a picker and receives the
+// goaway before creating the stream, an error will occur, but upon transparent
+// retry, the clientconn will ensure a ready subconn is chosen.
+func (s) TestGracefulClientOnGoAway(t *testing.T) {
+	const maxConnAge = 100 * time.Millisecond
+	const testTime = maxConnAge * 10
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create listener: %v", err)
+	}
+	ss := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge})),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("Failed to dial server: %v", err)
+	}
+	defer cc.Close()
+	c := testgrpc.NewTestServiceClient(cc)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	endTime := time.Now().Add(testTime)
+	for time.Now().Before(endTime) {
+		if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall(_, _) = _, %v; want _, <nil>", err)
+		}
+	}
+}
+
+func (s) TestDetailedGoAwayErrorOnGracefulClosePropagatesToRPCError(t *testing.T) {
+	rpcDoneOnClient := make(chan struct{})
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			<-rpcDoneOnClient
+			return status.Error(codes.Internal, "arbitrary status")
+		},
+	}
+	sopts := []grpc.ServerOption{
+		grpc.KeepaliveParams(keepalive.ServerParameters{
+			MaxConnectionAge:      time.Millisecond * 100,
+			MaxConnectionAgeGrace: time.Nanosecond, // ~instantaneously, but non-zero to avoid default
+		}),
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall = _, %v, want _, <nil>", ss.Client, err)
+	}
+	const expectedErrorMessageSubstring = "received prior goaway: code: NO_ERROR"
+	_, err = stream.Recv()
+	close(rpcDoneOnClient)
+	if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) {
+		t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q", stream, err, expectedErrorMessageSubstring)
+	}
+}
+
+func (s) TestDetailedGoAwayErrorOnAbruptClosePropagatesToRPCError(t *testing.T) {
+	grpctest.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"")
+	// set the min keepalive time very low so that this test can take
+	// a reasonable amount of time
+	prev := internal.KeepaliveMinPingTime
+	internal.KeepaliveMinPingTime = time.Millisecond
+	defer func() { internal.KeepaliveMinPingTime = prev }()
+
+	rpcDoneOnClient := make(chan struct{})
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			<-rpcDoneOnClient
+			return status.Error(codes.Internal, "arbitrary status")
+		},
+	}
+	sopts := []grpc.ServerOption{
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime: time.Second * 1000, /* arbitrary, large value */
+		}),
+	}
+	dopts := []grpc.DialOption{
+		grpc.WithKeepaliveParams(keepalive.ClientParameters{
+			Time:                time.Millisecond,   /* should trigger "too many pings" error quickly */
+			Timeout:             time.Second * 1000, /* arbitrary, large value */
+			PermitWithoutStream: false,
+		}),
+	}
+	if err := ss.Start(sopts, dopts...); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall = _, %v, want _, <nil>", ss.Client, err)
+	}
+	const expectedErrorMessageSubstring = `received prior goaway: code: ENHANCE_YOUR_CALM, debug data: "too_many_pings"`
+	_, err = stream.Recv()
+	close(rpcDoneOnClient)
+	if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) {
+		t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: |%v|", stream, err, expectedErrorMessageSubstring)
+	}
+}
+
+func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testClientConnCloseAfterGoAwayWithActiveStream(t, e)
+	}
+}
+
+func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.FullDuplexCall(ctx); err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want _, <nil>", tc, err)
+	}
+	done := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(done)
+	}()
+	time.Sleep(50 * time.Millisecond)
+	cc.Close()
+	timeout := time.NewTimer(time.Second)
+	select {
+	case <-done:
+	case <-timeout.C:
+		t.Fatalf("Test timed-out.")
+	}
+}
+
+func (s) TestServerGoAway(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testServerGoAway(t, e)
+	}
+}
+
+func testServerGoAway(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.userAgent = testAppUA
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	// Finish an RPC to make sure the connection is good.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, <nil>", err)
+	}
+	ch := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(ch)
+	}()
+	// Loop until the server side GoAway signal is propagated to the client.
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil && status.Code(err) != codes.DeadlineExceeded {
+			cancel()
+			break
+		}
+		cancel()
+	}
+	// A new RPC should fail.
+	ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable && status.Code(err) != codes.Internal {
+		t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s or %s", err, codes.Unavailable, codes.Internal)
+	}
+	<-ch
+	awaitNewConnLogOutput()
+}
+
+func (s) TestServerGoAwayPendingRPC(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testServerGoAwayPendingRPC(t, e)
+	}
+}
+
+func testServerGoAwayPendingRPC(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.userAgent = testAppUA
+	te.declareLogNoise(
+		"transport: http2Client.notifyError got notified that the client transport was broken EOF",
+		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
+		"grpc: addrConn.resetTransport failed to create client transport: connection error",
+	)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	// Finish an RPC to make sure the connection is good.
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
+	}
+	ch := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(ch)
+	}()
+	// Loop until the server side GoAway signal is propagated to the client.
+	start := time.Now()
+	errored := false
+	for time.Since(start) < time.Second {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		_, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true))
+		cancel()
+		if err != nil {
+			errored = true
+			break
+		}
+	}
+	if !errored {
+		t.Fatalf("GoAway never received by client")
+	}
+	respParam := []*testpb.ResponseParameters{{Size: 1}}
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: respParam,
+		Payload:            payload,
+	}
+	// The existing RPC should be still good to proceed.
+	if err := stream.Send(req); err != nil {
+		t.Fatalf("%v.Send(_) = %v, want <nil>", stream, err)
+	}
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("%v.Recv() = _, %v, want _, <nil>", stream, err)
+	}
+	// The RPC will run until canceled.
+	cancel()
+	<-ch
+	awaitNewConnLogOutput()
+}
+
+func (s) TestServerMultipleGoAwayPendingRPC(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testServerMultipleGoAwayPendingRPC(t, e)
+	}
+}
+
+func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.userAgent = testAppUA
+	te.declareLogNoise(
+		"transport: http2Client.notifyError got notified that the client transport was broken EOF",
+		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
+		"grpc: addrConn.resetTransport failed to create client transport: connection error",
+	)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+	// Finish an RPC to make sure the connection is good.
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
+	}
+	ch1 := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(ch1)
+	}()
+	ch2 := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(ch2)
+	}()
+	// Loop until the server side GoAway signal is propagated to the client.
+
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+			cancel()
+			break
+		}
+		cancel()
+	}
+	select {
+	case <-ch1:
+		t.Fatal("GracefulStop() terminated early")
+	case <-ch2:
+		t.Fatal("GracefulStop() terminated early")
+	default:
+	}
+	respParam := []*testpb.ResponseParameters{
+		{
+			Size: 1,
+		},
+	}
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: respParam,
+		Payload:            payload,
+	}
+	// The existing RPC should be still good to proceed.
+	if err := stream.Send(req); err != nil {
+		t.Fatalf("%v.Send(%v) = %v, want <nil>", stream, req, err)
+	}
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("%v.Recv() = _, %v, want _, <nil>", stream, err)
+	}
+	if err := stream.CloseSend(); err != nil {
+		t.Fatalf("%v.CloseSend() = %v, want <nil>", stream, err)
+	}
+
+	<-ch1
+	<-ch2
+	cancel()
+	awaitNewConnLogOutput()
+}
+
+func (s) TestConcurrentClientConnCloseAndServerGoAway(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testConcurrentClientConnCloseAndServerGoAway(t, e)
+	}
+}
+
+func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.userAgent = testAppUA
+	te.declareLogNoise(
+		"transport: http2Client.notifyError got notified that the client transport was broken EOF",
+		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
+		"grpc: addrConn.resetTransport failed to create client transport: connection error",
+	)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
+	}
+	ch := make(chan struct{})
+	// Close ClientConn and Server concurrently.
+	go func() {
+		te.srv.GracefulStop()
+		close(ch)
+	}()
+	go func() {
+		cc.Close()
+	}()
+	<-ch
+}
+
+func (s) TestConcurrentServerStopAndGoAway(t *testing.T) {
+	for _, e := range listTestEnv() {
+		if e.name == "handler-tls" {
+			continue
+		}
+		testConcurrentServerStopAndGoAway(t, e)
+	}
+}
+
+func testConcurrentServerStopAndGoAway(t *testing.T, e env) {
+	te := newTest(t, e)
+	te.userAgent = testAppUA
+	te.declareLogNoise(
+		"transport: http2Client.notifyError got notified that the client transport was broken EOF",
+		"grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing",
+		"grpc: addrConn.resetTransport failed to create client transport: connection error",
+	)
+	te.startServer(&testServer{security: e.security})
+	defer te.tearDown()
+
+	cc := te.clientConn()
+	tc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true))
+	if err != nil {
+		t.Fatalf("%v.FullDuplexCall(_) = _, %v, want <nil>", tc, err)
+	}
+
+	// Finish an RPC to make sure the connection is good.
+	if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, <nil>", tc, err)
+	}
+
+	ch := make(chan struct{})
+	go func() {
+		te.srv.GracefulStop()
+		close(ch)
+	}()
+	// Loop until the server side GoAway signal is propagated to the client.
+	for {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+			cancel()
+			break
+		}
+		cancel()
+	}
+	// Stop the server and close all the connections.
+	te.srv.Stop()
+	respParam := []*testpb.ResponseParameters{
+		{
+			Size: 1,
+		},
+	}
+	payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100))
+	if err != nil {
+		t.Fatal(err)
+	}
+	req := &testpb.StreamingOutputCallRequest{
+		ResponseType:       testpb.PayloadType_COMPRESSABLE,
+		ResponseParameters: respParam,
+		Payload:            payload,
+	}
+	sendStart := time.Now()
+	for {
+		if err := stream.Send(req); err == io.EOF {
+			// stream.Send should eventually send io.EOF
+			break
+		} else if err != nil {
+			// Send should never return a transport-level error.
+			t.Fatalf("stream.Send(%v) = %v; want <nil or io.EOF>", req, err)
+		}
+		if time.Since(sendStart) > 2*time.Second {
+			t.Fatalf("stream.Send(_) did not return io.EOF after 2s")
+		}
+		time.Sleep(time.Millisecond)
+	}
+	if _, err := stream.Recv(); err == nil || err == io.EOF {
+		t.Fatalf("%v.Recv() = _, %v, want _, <non-nil, non-EOF>", stream, err)
+	}
+	<-ch
+	awaitNewConnLogOutput()
+}
+
+// Proxies typically send GO_AWAY followed by connection closure a minute or so later. This
+// test ensures that the connection is re-created after GO_AWAY and not affected by the
+// subsequent (old) connection closure.
+func (s) TestGoAwayThenClose(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	lis1, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+
+	unaryCallF := func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+		return &testpb.SimpleResponse{}, nil
+	}
+	fullDuplexCallF := func(stream testgrpc.TestService_FullDuplexCallServer) error {
+		if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+			t.Errorf("Unexpected error from send: %v", err)
+			return err
+		}
+		// Wait until a message is received from client
+		_, err := stream.Recv()
+		if err == nil {
+			t.Error("Expected to never receive any message")
+		}
+		return err
+	}
+	ss1 := &stubserver.StubServer{
+		Listener:        lis1,
+		UnaryCallF:      unaryCallF,
+		FullDuplexCallF: fullDuplexCallF,
+		S:               grpc.NewServer(),
+	}
+	stubserver.StartTestService(t, ss1)
+	defer ss1.S.Stop()
+
+	conn2Established := grpcsync.NewEvent()
+	lis2, err := listenWithNotifyingListener("tcp", "localhost:0", conn2Established)
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	ss2 := &stubserver.StubServer{
+		Listener:        lis2,
+		UnaryCallF:      unaryCallF,
+		FullDuplexCallF: fullDuplexCallF,
+		S:               grpc.NewServer(),
+	}
+	stubserver.StartTestService(t, ss2)
+	defer ss2.S.Stop()
+
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{Addresses: []resolver.Address{
+		{Addr: lis1.Addr().String()},
+		{Addr: lis2.Addr().String()},
+	}})
+	cc, err := grpc.NewClient(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("Error creating client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// We make a streaming RPC and do an one-message-round-trip to make sure
+	// it's created on connection 1.
+	//
+	// We use a long-lived RPC because it will cause GracefulStop to send
+	// GO_AWAY, but the connection won't get closed until the server stops and
+	// the client receives the error.
+	t.Log("Creating first streaming RPC to server 1.")
+	stream, err := client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err)
+	}
+	if _, err = stream.Recv(); err != nil {
+		t.Fatalf("unexpected error from first recv: %v", err)
+	}
+
+	t.Log("Gracefully stopping server 1.")
+	go ss1.S.GracefulStop()
+
+	t.Log("Waiting for the ClientConn to enter IDLE state.")
+	testutils.AwaitState(ctx, t, cc, connectivity.Idle)
+
+	t.Log("Performing another RPC to create a connection to server 2.")
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+		t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err)
+	}
+
+	t.Log("Waiting for a connection to server 2.")
+	select {
+	case <-conn2Established.Done():
+	case <-ctx.Done():
+		t.Fatalf("timed out waiting for connection 2 to be established")
+	}
+
+	// Close the listener for server2 to prevent it from allowing new connections.
+	lis2.Close()
+
+	t.Log("Hard closing connection 1.")
+	ss1.S.Stop()
+
+	t.Log("Waiting for the first stream to error.")
+	if _, err = stream.Recv(); err == nil {
+		t.Fatal("expected the stream to die, but got a successful Recv")
+	}
+
+	t.Log("Ensuring connection 2 is stable.")
+	for i := 0; i < 10; i++ {
+		if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+			t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err)
+		}
+	}
+}
+
+// TestGoAwayStreamIDSmallerThanCreatedStreams tests the scenario where a server
+// sends a goaway with a stream id that is smaller than some created streams on
+// the client, while the client is simultaneously creating new streams. This
+// should not induce a deadlock.
+func (s) TestGoAwayStreamIDSmallerThanCreatedStreams(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("error listening: %v", err)
+	}
+
+	ctCh := testutils.NewChannel()
+	go func() {
+		conn, err := lis.Accept()
+		if err != nil {
+			t.Errorf("error in lis.Accept(): %v", err)
+		}
+		ct := newClientTester(t, conn)
+		ctCh.Send(ct)
+	}()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	val, err := ctCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout waiting for client transport (should be given after http2 creation)")
+	}
+	ct := val.(*clientTester)
+
+	tc := testgrpc.NewTestServiceClient(cc)
+	someStreamsCreated := grpcsync.NewEvent()
+	goAwayWritten := grpcsync.NewEvent()
+	go func() {
+		for i := 0; i < 20; i++ {
+			if i == 10 {
+				<-goAwayWritten.Done()
+			}
+			tc.FullDuplexCall(ctx)
+			if i == 4 {
+				someStreamsCreated.Fire()
+			}
+		}
+	}()
+
+	<-someStreamsCreated.Done()
+	ct.writeGoAway(1, http2.ErrCodeNo, []byte{})
+	goAwayWritten.Fire()
+}
+
+// TestTwoGoAwayPingFrames tests the scenario where you get two go away ping
+// frames from the client during graceful shutdown. This should not crash the
+// server.
+func (s) TestTwoGoAwayPingFrames(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Failed to listen: %v", err)
+	}
+	defer lis.Close()
+	s := grpc.NewServer()
+	defer s.Stop()
+	go s.Serve(lis)
+
+	conn, err := net.DialTimeout("tcp", lis.Addr().String(), defaultTestTimeout)
+	if err != nil {
+		t.Fatalf("Failed to dial: %v", err)
+	}
+
+	st := newServerTesterFromConn(t, conn)
+	st.greet()
+	pingReceivedClientSide := testutils.NewChannel()
+	go func() {
+		for {
+			f, err := st.readFrame()
+			if err != nil {
+				return
+			}
+			switch f.(type) {
+			case *http2.GoAwayFrame:
+			case *http2.PingFrame:
+				pingReceivedClientSide.Send(nil)
+			default:
+				t.Errorf("server tester received unexpected frame type %T", f)
+			}
+		}
+	}()
+	gsDone := testutils.NewChannel()
+	go func() {
+		s.GracefulStop()
+		gsDone.Send(nil)
+	}()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := pingReceivedClientSide.Receive(ctx); err != nil {
+		t.Fatalf("Error waiting for ping frame client side from graceful shutdown: %v", err)
+	}
+	// Write two goaway pings here.
+	st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9})
+	st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9})
+	// Close the conn to finish up the Graceful Shutdown process.
+	conn.Close()
+	if _, err := gsDone.Receive(ctx); err != nil {
+		t.Fatalf("Error waiting for graceful shutdown of the server: %v", err)
+	}
+}
+
+// TestClientSendsAGoAway tests the scenario where you get a go away ping
+// frames from the client during graceful shutdown.
+func (s) TestClientSendsAGoAway(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("error listening: %v", err)
+	}
+	defer lis.Close()
+	goAwayReceived := make(chan struct{})
+	errCh := make(chan error)
+	go func() {
+		conn, err := lis.Accept()
+		if err != nil {
+			t.Errorf("error in lis.Accept(): %v", err)
+		}
+		ct := newClientTester(t, conn)
+		defer ct.conn.Close()
+		for {
+			f, err := ct.fr.ReadFrame()
+			if err != nil {
+				errCh <- fmt.Errorf("error reading frame: %v", err)
+				return
+			}
+			switch fr := f.(type) {
+			case *http2.GoAwayFrame:
+				fr = f.(*http2.GoAwayFrame)
+				if fr.ErrCode == http2.ErrCodeNo {
+					t.Logf("GoAway received from client")
+					close(goAwayReceived)
+					return
+				}
+			default:
+				t.Errorf("server tester received unexpected frame type %T", f)
+				errCh <- fmt.Errorf("server tester received unexpected frame type %T", f)
+				close(errCh)
+			}
+		}
+	}()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("error dialing: %v", err)
+	}
+	cc.Connect()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	testutils.AwaitState(ctx, t, cc, connectivity.Ready)
+	cc.Close()
+	select {
+	case <-goAwayReceived:
+	case err := <-errCh:
+		t.Errorf("Error receiving the goAway: %v", err)
+	case <-ctx.Done():
+		t.Errorf("Context timed out")
+	}
+}

test/gracefulstop_test.go

@@ -0,0 +1,364 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+type delayListener struct {
+	net.Listener
+	closeCalled  chan struct{}
+	acceptCalled chan struct{}
+	allowCloseCh chan struct{}
+	dialed       bool
+}
+
+func (d *delayListener) Accept() (net.Conn, error) {
+	select {
+	case <-d.acceptCalled:
+		// On the second call, block until closed, then return an error.
+		<-d.closeCalled
+		<-d.allowCloseCh
+		return nil, fmt.Errorf("listener is closed")
+	default:
+		close(d.acceptCalled)
+		conn, err := d.Listener.Accept()
+		if err != nil {
+			return nil, err
+		}
+		// Allow closing of listener only after accept.
+		// Note: Dial can return successfully, yet Accept
+		// might now have finished.
+		d.allowClose()
+		return conn, nil
+	}
+}
+
+func (d *delayListener) allowClose() {
+	close(d.allowCloseCh)
+}
+func (d *delayListener) Close() error {
+	close(d.closeCalled)
+	go func() {
+		<-d.allowCloseCh
+		d.Listener.Close()
+	}()
+	return nil
+}
+
+func (d *delayListener) Dial(ctx context.Context) (net.Conn, error) {
+	if d.dialed {
+		// Only hand out one connection (net.Dial can return more even after the
+		// listener is closed).  This is not thread-safe, but Dial should never be
+		// called concurrently in this environment.
+		return nil, fmt.Errorf("no more conns")
+	}
+	d.dialed = true
+	return (&net.Dialer{}).DialContext(ctx, "tcp", d.Listener.Addr().String())
+}
+
+// TestGracefulStop ensures GracefulStop causes new connections to fail.
+//
+// Steps of this test:
+//  1. Start Server
+//  2. GracefulStop() Server after listener's Accept is called, but don't
+//     allow Accept() to exit when Close() is called on it.
+//  3. Create a new connection to the server after listener.Close() is called.
+//     Server should close this connection immediately, before handshaking.
+//  4. Send an RPC on the new connection.  Should see Unavailable error
+//     because the ClientConn is in transient failure.
+func (s) TestGracefulStop(t *testing.T) {
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error listenening: %v", err)
+	}
+	dlis := &delayListener{
+		Listener:     lis,
+		acceptCalled: make(chan struct{}),
+		closeCalled:  make(chan struct{}),
+		allowCloseCh: make(chan struct{}),
+	}
+
+	ss := &stubserver.StubServer{
+		Listener: dlis,
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			if _, err := stream.Recv(); err != nil {
+				return err
+			}
+			return stream.Send(&testpb.StreamingOutputCallResponse{})
+		},
+		S: grpc.NewServer(),
+	}
+	// 1. Start Server and start serving by calling Serve().
+	stubserver.StartTestService(t, ss)
+
+	// 2. Call GracefulStop from a goroutine. It will trigger Close on the
+	// listener, but the listener will not actually close until a connection
+	// is accepted.
+	gracefulStopDone := make(chan struct{})
+	<-dlis.acceptCalled
+	go func() {
+		ss.S.GracefulStop()
+		close(gracefulStopDone)
+	}()
+
+	// 3. Create a new connection to the server after listener.Close() is called.
+	// Server should close this connection immediately, before handshaking.
+
+	<-dlis.closeCalled // Block until GracefulStop calls dlis.Close()
+
+	dialer := func(ctx context.Context, _ string) (net.Conn, error) { return dlis.Dial(ctx) }
+	cc, err := grpc.NewClient("passthrough:///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(dialer))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) = %v", lis.Addr().String(), err)
+	}
+	client := testgrpc.NewTestServiceClient(cc)
+	defer cc.Close()
+
+	// 4. Make an RPC.
+	// The server would send a GOAWAY first, but we are delaying the server's
+	// writes for now until the client writes more than the preface.
+	// This will cause a connection to be accepted. This will
+	// also unblock the Close method.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	if _, err = client.FullDuplexCall(ctx); err == nil || status.Code(err) != codes.Unavailable {
+		t.Fatalf("FullDuplexCall= _, %v; want _, <status code Unavailable>", err)
+	}
+	cancel()
+	<-gracefulStopDone
+}
+
+// TestGracefulStopClosesConnAfterLastStream ensures that a server closes the
+// connections to its clients when the final stream has completed after
+// a GOAWAY.
+func (s) TestGracefulStopClosesConnAfterLastStream(t *testing.T) {
+
+	handlerCalled := make(chan struct{})
+	gracefulStopCalled := make(chan struct{})
+
+	ts := &funcServer{streamingInputCall: func(testgrpc.TestService_StreamingInputCallServer) error {
+		close(handlerCalled) // Initiate call to GracefulStop.
+		<-gracefulStopCalled // Wait for GOAWAYs to be received by the client.
+		return nil
+	}}
+
+	te := newTest(t, tcpClearEnv)
+	te.startServer(ts)
+	defer te.tearDown()
+
+	te.withServerTester(func(st *serverTester) {
+		st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", false)
+
+		<-handlerCalled // Wait for the server to invoke its handler.
+
+		// Gracefully stop the server.
+		gracefulStopDone := make(chan struct{})
+		go func() {
+			te.srv.GracefulStop()
+			close(gracefulStopDone)
+		}()
+		st.wantGoAway(http2.ErrCodeNo) // Server sends a GOAWAY due to GracefulStop.
+		pf := st.wantPing()            // Server sends a ping to verify client receipt.
+		st.writePing(true, pf.Data)    // Send ping ack to confirm.
+		st.wantGoAway(http2.ErrCodeNo) // Wait for subsequent GOAWAY to indicate no new stream processing.
+
+		close(gracefulStopCalled) // Unblock server handler.
+
+		fr := st.wantAnyFrame() // Wait for trailer.
+		hdr, ok := fr.(*http2.MetaHeadersFrame)
+		if !ok {
+			t.Fatalf("Received unexpected frame of type (%T) from server: %v; want HEADERS", fr, fr)
+		}
+		if !hdr.StreamEnded() {
+			t.Fatalf("Received unexpected HEADERS frame from server: %v; want END_STREAM set", fr)
+		}
+
+		st.wantRSTStream(http2.ErrCodeNo) // Server should send RST_STREAM because client did not half-close.
+
+		<-gracefulStopDone // Wait for GracefulStop to return.
+	})
+}
+
+// TestGracefulStopBlocksUntilGRPCConnectionsTerminate ensures that
+// GracefulStop() blocks until all ongoing RPCs finished.
+func (s) TestGracefulStopBlocksUntilGRPCConnectionsTerminate(t *testing.T) {
+	unblockGRPCCall := make(chan struct{})
+	grpcCallExecuting := make(chan struct{})
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			close(grpcCallExecuting)
+			<-unblockGRPCCall
+			return &testpb.SimpleResponse{}, nil
+		},
+	}
+
+	err := ss.Start(nil)
+	if err != nil {
+		t.Fatalf("StubServer.start failed: %s", err)
+	}
+	t.Cleanup(ss.Stop)
+
+	grpcClientCallReturned := make(chan struct{})
+	go func() {
+		clt := ss.Client
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		_, err := clt.UnaryCall(ctx, &testpb.SimpleRequest{})
+		if err != nil {
+			t.Errorf("rpc failed with error: %s", err)
+		}
+		close(grpcClientCallReturned)
+	}()
+
+	gracefulStopReturned := make(chan struct{})
+	<-grpcCallExecuting
+	go func() {
+		ss.S.GracefulStop()
+		close(gracefulStopReturned)
+	}()
+
+	select {
+	case <-gracefulStopReturned:
+		t.Error("GracefulStop returned before rpc method call ended")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	unblockGRPCCall <- struct{}{}
+	<-grpcClientCallReturned
+	<-gracefulStopReturned
+}
+
+// TestStopAbortsBlockingGRPCCall ensures that when Stop() is called while an ongoing RPC
+// is blocking that:
+// - Stop() returns
+// - and the RPC fails with an connection  closed error on the client-side
+func (s) TestStopAbortsBlockingGRPCCall(t *testing.T) {
+	unblockGRPCCall := make(chan struct{})
+	grpcCallExecuting := make(chan struct{})
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			close(grpcCallExecuting)
+			<-unblockGRPCCall
+			return &testpb.SimpleResponse{}, nil
+		},
+	}
+
+	err := ss.Start(nil)
+	if err != nil {
+		t.Fatalf("StubServer.start failed: %s", err)
+	}
+	t.Cleanup(ss.Stop)
+
+	grpcClientCallReturned := make(chan struct{})
+	go func() {
+		clt := ss.Client
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		defer cancel()
+		_, err := clt.UnaryCall(ctx, &testpb.SimpleRequest{})
+		if err == nil || !isConnClosedErr(err) {
+			t.Errorf("expected rpc to fail with connection closed error, got: %v", err)
+		}
+		close(grpcClientCallReturned)
+	}()
+
+	<-grpcCallExecuting
+	ss.S.Stop()
+
+	unblockGRPCCall <- struct{}{}
+	<-grpcClientCallReturned
+}
+
+// TestServeHTTP_GracefulStop verifies that calling GracefulStop on a server
+// using ServeHTTP returns properly.
+func (s) TestServeHTTP_GracefulStop(t *testing.T) {
+	gs := grpc.NewServer()
+	defer gs.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Register a service that blocks until context is done to make sure the
+	// server is active when GracefulStop is called.
+	callStarted := make(chan struct{})
+	testgrpc.RegisterTestServiceServer(gs, &stubserver.StubServer{
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			close(callStarted)
+			<-ctx.Done()
+			return nil
+		},
+	})
+
+	// Create a handler that delegates to ServeHTTP and signals when it returns.
+	serveHTTPDone := make(chan struct{})
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gs.ServeHTTP(w, r)
+		close(serveHTTPDone)
+	})
+
+	// Start an HTTP/2 server with the wrapped handler.
+	ts := httptest.NewUnstartedServer(handler)
+	ts.EnableHTTP2 = true
+	ts.StartTLS()
+	defer ts.Close()
+
+	// Make a new request to the gRPC server via HTTP/2 to ensure HandleStreams is called.
+	req, err := http.NewRequest("POST", ts.URL+"/grpc.testing.TestService/FullDuplexCall", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/grpc")
+
+	client := ts.Client()
+	// Send the request in a goroutine.
+	go func() {
+		resp, err := client.Do(req)
+		if err != nil {
+			return
+		}
+		resp.Body.Close()
+	}()
+
+	<-callStarted
+	// Call GracefulStop in a goroutine to avoid blocking the main thread until the RPC is done.
+	go gs.GracefulStop()
+
+	// Verify that ServeHTTP returned because GracefulStop is called and ensure it doesn't panic.
+	select {
+	case <-serveHTTPDone:
+	case <-ctx.Done():
+		t.Fatal("Context timed out waiting for ServeHTTP to return after GracefulStop")
+	}
+}

test/http_header_end2end_test.go

@@ -0,0 +1,259 @@
+/*
+*
+* Copyright 2022 gRPC authors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+ */
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/transport"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestHTTPHeaderFrameErrorHandlingHTTPMode(t *testing.T) {
+	type test struct {
+		name    string
+		header  []string
+		errCode codes.Code
+	}
+
+	var tests []test
+
+	// Non-gRPC content-type fallback path.
+	for httpCode := range transport.HTTPStatusConvTab {
+		tests = append(tests, test{
+			name: fmt.Sprintf("Non-gRPC content-type fallback path with httpCode: %v", httpCode),
+			header: []string{
+				":status", fmt.Sprintf("%d", httpCode),
+				"content-type", "text/html", // non-gRPC content type to switch to HTTP mode.
+				"grpc-status", "1", // Make up a gRPC status error
+				"grpc-status-details-bin", "???", // Make up a gRPC field parsing error
+			},
+			errCode: transport.HTTPStatusConvTab[int(httpCode)],
+		})
+	}
+
+	// Missing content-type fallback path.
+	for httpCode := range transport.HTTPStatusConvTab {
+		tests = append(tests, test{
+			name: fmt.Sprintf("Missing content-type fallback path with httpCode: %v", httpCode),
+			header: []string{
+				":status", fmt.Sprintf("%d", httpCode),
+				// Omitting content type to switch to HTTP mode.
+				"grpc-status", "1", // Make up a gRPC status error
+				"grpc-status-details-bin", "???", // Make up a gRPC field parsing error
+			},
+			errCode: transport.HTTPStatusConvTab[int(httpCode)],
+		})
+	}
+
+	// Malformed HTTP status when fallback.
+	tests = append(tests, test{
+		name: "Malformed HTTP status when fallback",
+		header: []string{
+			":status", "abc",
+			// Omitting content type to switch to HTTP mode.
+			"grpc-status", "1", // Make up a gRPC status error
+			"grpc-status-details-bin", "???", // Make up a gRPC field parsing error
+		},
+		errCode: codes.Internal,
+	})
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			serverAddr, cleanup, err := startServer(t, test.header)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+// Testing erroneous ResponseHeader or Trailers-only (delivered in the first HEADERS frame).
+func (s) TestHTTPHeaderFrameErrorHandlingInitialHeader(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		header  []string
+		errCode codes.Code
+	}{
+		{
+			name: "missing gRPC content type",
+			header: []string{
+				":status", "403",
+			},
+			errCode: codes.PermissionDenied,
+		},
+		{
+			name: "malformed grpc-status",
+			header: []string{
+				":status", "502",
+				"content-type", "application/grpc",
+				"grpc-status", "abc",
+			},
+			errCode: codes.Unknown,
+		},
+		{
+			name: "Malformed grpc-tags-bin field ignores http status",
+			header: []string{
+				":status", "502",
+				"content-type", "application/grpc",
+				"grpc-status", "0",
+				"grpc-tags-bin", "???",
+			},
+			errCode: codes.Internal,
+		},
+		{
+			name: "gRPC status error ignoring http status",
+			header: []string{
+				":status", "502",
+				"content-type", "application/grpc",
+				"grpc-status", "3",
+			},
+			errCode: codes.InvalidArgument,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			serverAddr, cleanup, err := startServer(t, test.header)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil {
+				t.Error(err)
+			}
+		})
+	}
+}
+
+// Testing non-Trailers-only Trailers (delivered in second HEADERS frame)
+func (s) TestHTTPHeaderFrameErrorHandlingNormalTrailer(t *testing.T) {
+	tests := []struct {
+		name           string
+		responseHeader []string
+		trailer        []string
+		errCode        codes.Code
+	}{
+		{
+			name: "trailer missing grpc-status to ignore http status",
+			responseHeader: []string{
+				":status", "200",
+				"content-type", "application/grpc",
+			},
+			trailer: []string{
+				// trailer missing grpc-status
+				":status", "502",
+			},
+			errCode: codes.Unknown,
+		},
+		{
+			name: "malformed grpc-status-details-bin field with status 404 to be ignored due to content type",
+			responseHeader: []string{
+				":status", "404",
+				"content-type", "application/grpc",
+			},
+			trailer: []string{
+				// malformed grpc-status-details-bin field
+				"grpc-status", "0",
+				"grpc-status-details-bin", "????",
+			},
+			errCode: codes.Internal,
+		},
+		{
+			name: "malformed grpc-status-details-bin field with status 404 and no content type",
+			responseHeader: []string{
+				":status", "404",
+			},
+			trailer: []string{
+				// malformed grpc-status-details-bin field
+				"grpc-status", "0",
+				"grpc-status-details-bin", "????",
+			},
+			errCode: codes.Unimplemented,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			serverAddr, cleanup, err := startServer(t, test.responseHeader, test.trailer)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil {
+				t.Error(err)
+			}
+		})
+
+	}
+}
+
+func (s) TestHTTPHeaderFrameErrorHandlingMoreThanTwoHeaders(t *testing.T) {
+	header := []string{
+		":status", "200",
+		"content-type", "application/grpc",
+	}
+	serverAddr, cleanup, err := startServer(t, header, header, header)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanup()
+	if err := doHTTPHeaderTest(serverAddr, codes.Internal); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func startServer(t *testing.T, headerFields ...[]string) (serverAddr string, cleanup func(), err error) {
+	t.Helper()
+
+	lis, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", nil, fmt.Errorf("listening on %q: %v", "localhost:0", err)
+	}
+	server := &httpServer{responses: []httpServerResponse{{trailers: headerFields}}}
+	server.start(t, lis)
+	return lis.Addr().String(), func() { lis.Close() }, nil
+}
+
+func doHTTPHeaderTest(lisAddr string, errCode codes.Code) error {
+	cc, err := grpc.NewClient(lisAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return fmt.Errorf("NewClient(%q): %v", lisAddr, err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	stream, err := client.FullDuplexCall(ctx)
+	if err != nil {
+		return fmt.Errorf("creating FullDuplex stream: %v", err)
+	}
+	if _, err := stream.Recv(); err == nil || status.Code(err) != errCode {
+		return fmt.Errorf("stream.Recv() = %v, want error code: %v", err, errCode)
+	}
+	return nil
+}

test/insecure_creds_test.go

@@ -0,0 +1,198 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// testLegacyPerRPCCredentials is a PerRPCCredentials that has yet incorporated security level.
+type testLegacyPerRPCCredentials struct{}
+
+func (cr testLegacyPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) {
+	return nil, nil
+}
+
+func (cr testLegacyPerRPCCredentials) RequireTransportSecurity() bool {
+	return true
+}
+
+func getSecurityLevel(ai credentials.AuthInfo) credentials.SecurityLevel {
+	if c, ok := ai.(interface {
+		GetCommonAuthInfo() credentials.CommonAuthInfo
+	}); ok {
+		return c.GetCommonAuthInfo().SecurityLevel
+	}
+	return credentials.InvalidSecurityLevel
+}
+
+// TestInsecureCreds tests the use of insecure creds on the server and client
+// side, and verifies that expect security level and auth info are returned.
+// Also verifies that this credential can interop with existing `WithInsecure`
+// DialOption.
+func (s) TestInsecureCreds(t *testing.T) {
+	tests := []struct {
+		desc                string
+		clientInsecureCreds bool
+		serverInsecureCreds bool
+	}{
+		{
+			desc:                "client and server insecure creds",
+			clientInsecureCreds: true,
+			serverInsecureCreds: true,
+		},
+		{
+			desc:                "client only insecure creds",
+			clientInsecureCreds: true,
+		},
+		{
+			desc:                "server only insecure creds",
+			serverInsecureCreds: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			lis, err := testutils.LocalTCPListener()
+			if err != nil {
+				t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+			}
+
+			ss := &stubserver.StubServer{
+				Listener: lis,
+				EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+					if !test.serverInsecureCreds {
+						return &testpb.Empty{}, nil
+					}
+
+					pr, ok := peer.FromContext(ctx)
+					if !ok {
+						return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
+					}
+					// Check security level.
+					secLevel := getSecurityLevel(pr.AuthInfo)
+					if secLevel == credentials.InvalidSecurityLevel {
+						return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()")
+					}
+					if secLevel != credentials.NoSecurity {
+						return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity)
+					}
+					return &testpb.Empty{}, nil
+				},
+			}
+			sOpts := []grpc.ServerOption{}
+			if test.serverInsecureCreds {
+				ss.S = grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
+			} else {
+				ss.S = grpc.NewServer(sOpts...)
+			}
+			stubserver.StartTestService(t, ss)
+			defer ss.S.Stop()
+			addr := lis.Addr().String()
+			opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+			cc, err := grpc.NewClient(addr, opts...)
+			if err != nil {
+				t.Fatalf("grpc.NewClient(%q) failed: %v", addr, err)
+			}
+			defer cc.Close()
+
+			c := testgrpc.NewTestServiceClient(cc)
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall(_, _) = _, %v; want _, <nil>", err)
+			}
+		})
+	}
+}
+
+func (s) TestInsecureCreds_WithPerRPCCredentials_AsCallOption(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+	}
+
+	ss := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	addr := lis.Addr().String()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	dopts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+	copts := []grpc.CallOption{grpc.PerRPCCredentials(testLegacyPerRPCCredentials{})}
+	cc, err := grpc.NewClient(addr, dopts...)
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%q) failed: %v", addr, err)
+	}
+	defer cc.Close()
+
+	const wantErr = "transport: cannot send secure credentials on an insecure connection"
+	c := testgrpc.NewTestServiceClient(cc)
+	if _, err = c.EmptyCall(ctx, &testpb.Empty{}, copts...); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("insecure credentials with per-RPC credentials requiring transport security returned error: %v; want %s", err, wantErr)
+	}
+}
+
+func (s) TestInsecureCreds_WithPerRPCCredentials_AsDialOption(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err)
+	}
+	ss := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.Creds(insecure.NewCredentials())),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	addr := lis.Addr().String()
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithPerRPCCredentials(testLegacyPerRPCCredentials{}),
+	}
+	const wantErr = "the credentials require transport level security"
+	if _, err := grpc.NewClient(addr, dopts...); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("grpc.NewClient(%q) returned err %v, want: %v", addr, err, wantErr)
+	}
+}

test/interceptor_test.go

@@ -0,0 +1,281 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+type parentCtxkey struct{}
+type firstInterceptorCtxkey struct{}
+type secondInterceptorCtxkey struct{}
+type baseInterceptorCtxKey struct{}
+
+const (
+	parentCtxVal            = "parent"
+	firstInterceptorCtxVal  = "firstInterceptor"
+	secondInterceptorCtxVal = "secondInterceptor"
+	baseInterceptorCtxVal   = "baseInterceptor"
+)
+
+// TestUnaryClientInterceptor_ContextValuePropagation verifies that a unary
+// interceptor receives context values specified in the context passed to the
+// RPC call.
+func (s) TestUnaryClientInterceptor_ContextValuePropagation(t *testing.T) {
+	errCh := testutils.NewChannel()
+	unaryInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.Send(fmt.Errorf("unaryInt got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		errCh.Send(nil)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}
+
+	// Start a stub server and use the above unary interceptor while creating a
+	// ClientConn to it.
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := ss.Start(nil, grpc.WithUnaryInterceptor(unaryInt)); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall() failed: %v", err)
+	}
+	val, err := errCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err)
+	}
+	if val != nil {
+		t.Fatalf("unary interceptor failed: %v", val)
+	}
+}
+
+// TestChainUnaryClientInterceptor_ContextValuePropagation verifies that a chain
+// of unary interceptors receive context values specified in the original call
+// as well as the ones specified by prior interceptors in the chain.
+func (s) TestChainUnaryClientInterceptor_ContextValuePropagation(t *testing.T) {
+	errCh := testutils.NewChannel()
+	firstInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if ctx.Value(firstInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{}))
+		}
+		if ctx.Value(secondInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{}))
+		}
+		firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal)
+		return invoker(firstCtx, method, req, reply, cc, opts...)
+	}
+
+	secondInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal))
+		}
+		if ctx.Value(secondInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{}))
+		}
+		secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal)
+		return invoker(secondCtx, method, req, reply, cc, opts...)
+	}
+
+	lastInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal))
+		}
+		if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal))
+		}
+		errCh.SendContext(ctx, nil)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}
+
+	// Start a stub server and use the above chain of interceptors while creating
+	// a ClientConn to it.
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := ss.Start(nil, grpc.WithChainUnaryInterceptor(firstInt, secondInt, lastInt)); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall() failed: %v", err)
+	}
+	val, err := errCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err)
+	}
+	if val != nil {
+		t.Fatalf("unary interceptor failed: %v", val)
+	}
+}
+
+// TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation verifies that
+// unary interceptors specified as a base interceptor or as a chain interceptor
+// receive context values specified in the original call as well as the ones
+// specified by interceptors in the chain.
+func (s) TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation(t *testing.T) {
+	errCh := testutils.NewChannel()
+	baseInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("base interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if ctx.Value(baseInterceptorCtxKey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("baseinterceptor should not have %T in context", baseInterceptorCtxKey{}))
+		}
+		baseCtx := context.WithValue(ctx, baseInterceptorCtxKey{}, baseInterceptorCtxVal)
+		return invoker(baseCtx, method, req, reply, cc, opts...)
+	}
+
+	chainInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if got, ok := ctx.Value(baseInterceptorCtxKey{}).(string); !ok || got != baseInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, baseInterceptorCtxVal))
+		}
+		errCh.SendContext(ctx, nil)
+		return invoker(ctx, method, req, reply, cc, opts...)
+	}
+
+	// Start a stub server and use the above chain of interceptors while creating
+	// a ClientConn to it.
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := ss.Start(nil, grpc.WithUnaryInterceptor(baseInt), grpc.WithChainUnaryInterceptor(chainInt)); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall() failed: %v", err)
+	}
+	val, err := errCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err)
+	}
+	if val != nil {
+		t.Fatalf("unary interceptor failed: %v", val)
+	}
+}
+
+// TestChainStreamClientInterceptor_ContextValuePropagation verifies that a
+// chain of stream interceptors receive context values specified in the original
+// call as well as the ones specified by the prior interceptors in the chain.
+func (s) TestChainStreamClientInterceptor_ContextValuePropagation(t *testing.T) {
+	errCh := testutils.NewChannel()
+	firstInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if ctx.Value(firstInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{}))
+		}
+		if ctx.Value(secondInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{}))
+		}
+		firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal)
+		return streamer(firstCtx, desc, cc, method, opts...)
+	}
+
+	secondInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal))
+		}
+		if ctx.Value(secondInterceptorCtxkey{}) != nil {
+			errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{}))
+		}
+		secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal)
+		return streamer(secondCtx, desc, cc, method, opts...)
+	}
+
+	lastInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+		if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal))
+		}
+		if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal))
+		}
+		if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal {
+			errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal))
+		}
+		errCh.SendContext(ctx, nil)
+		return streamer(ctx, desc, cc, method, opts...)
+	}
+
+	// Start a stub server and use the above chain of interceptors while creating
+	// a ClientConn to it.
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			if _, err := stream.Recv(); err != nil {
+				return err
+			}
+			return stream.Send(&testpb.StreamingOutputCallResponse{})
+		},
+	}
+	if err := ss.Start(nil, grpc.WithChainStreamInterceptor(firstInt, secondInt, lastInt)); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.FullDuplexCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal)); err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall() failed: %v", err)
+	}
+	val, err := errCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for stream interceptor to be invoked: %v", err)
+	}
+	if val != nil {
+		t.Fatalf("stream interceptor failed: %v", val)
+	}
+}

test/invoke_test.go

@@ -0,0 +1,152 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/status"
+)
+
+// TestInvoke verifies a straightforward invocation of ClientConn.Invoke().
+func (s) TestInvoke(t *testing.T) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}); err != nil {
+		t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed: %v", err)
+	}
+}
+
+// TestInvokeLargeErr verifies an invocation of ClientConn.Invoke() where the
+// server returns a really large error message.
+func (s) TestInvokeLargeErr(t *testing.T) {
+	largeErrorStr := strings.Repeat("A", 1024*1024)
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, status.Error(codes.Internal, largeErrorStr)
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{})
+	if err == nil {
+		t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail")
+	}
+	st, ok := status.FromError(err)
+	if !ok {
+		t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error")
+	}
+	if status.Code(err) != codes.Internal || st.Message() != largeErrorStr {
+		t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want an error of code %d and desc size %d", err, codes.Internal, len(largeErrorStr))
+	}
+}
+
+// TestInvokeErrorSpecialChars tests an invocation of ClientConn.Invoke() and
+// verifies that error messages don't get mangled.
+func (s) TestInvokeErrorSpecialChars(t *testing.T) {
+	const weirdError = "format verbs: %v%s"
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, status.Error(codes.Internal, weirdError)
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{})
+	if err == nil {
+		t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail")
+	}
+	st, ok := status.FromError(err)
+	if !ok {
+		t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error")
+	}
+	if status.Code(err) != codes.Internal || st.Message() != weirdError {
+		t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want %v", err, weirdError)
+	}
+}
+
+// TestInvokeCancel tests an invocation of ClientConn.Invoke() with a cancelled
+// context and verifies that the request is not actually sent to the server.
+func (s) TestInvokeCancel(t *testing.T) {
+	cancelled := 0
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			cancelled++
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	for i := 0; i < 100; i++ {
+		ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+		cancel()
+		ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{})
+	}
+	if cancelled != 0 {
+		t.Fatalf("server received %d of 100 cancelled requests", cancelled)
+	}
+}
+
+// TestInvokeCancelClosedNonFail tests an invocation of ClientConn.Invoke() with
+// a cancelled non-failfast RPC on a closed ClientConn and verifies that the
+// call terminates with an error.
+func (s) TestInvokeCancelClosedNonFailFast(t *testing.T) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Failed to start stub server: %v", err)
+	}
+	defer ss.Stop()
+
+	ss.CC.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	cancel()
+	if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil {
+		t.Fatal("ClientConn.Invoke() on closed connection succeeded when expected to fail")
+	}
+}

test/kokoro/README.md

@@ -0,0 +1 @@
+The scripts in this directory are intended to be run by Kokoro CI jobs.

test/kokoro/psm-csm.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 120
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "csm"
+}

test/kokoro/psm-dualstack.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 360
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "dualstack"
+}

test/kokoro/psm-interop-build-go.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -eo pipefail
+
+#######################################
+# Builds test app Docker images and pushes them to GCR.
+# Called from psm_interop_kokoro_lib.sh.
+#
+# Globals:
+#   SRC_DIR: Absolute path to the source repo on Kokoro VM
+#   SERVER_IMAGE_NAME: Test server Docker image name
+#   CLIENT_IMAGE_NAME: Test client Docker image name
+#   GIT_COMMIT: SHA-1 of git commit being built
+#   DOCKER_REGISTRY: Docker registry to push to
+# Outputs:
+#   Writes the output of docker image build stdout, stderr
+#######################################
+psm::lang::build_docker_images() {
+  local client_dockerfile="interop/xds/client/Dockerfile"
+  local server_dockerfile="interop/xds/server/Dockerfile"
+  psm::build::docker_images_generic "${client_dockerfile}" "${server_dockerfile}"
+}

test/kokoro/psm-interop-test-go.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -eo pipefail
+
+# Input parameters to psm:: methods of the install script.
+readonly GRPC_LANGUAGE="go"
+readonly BUILD_SCRIPT_DIR="$(dirname "$0")"
+
+# Used locally.
+readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh"
+
+psm::lang::source_install_lib() {
+  echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  local install_lib
+  # Download to a tmp file.
+  install_lib="$(mktemp -d)/psm_interop_kokoro_lib.sh"
+  curl -s --retry-connrefused --retry 5 -o "${install_lib}" "${TEST_DRIVER_INSTALL_SCRIPT_URL}"
+  # Checksum.
+  if command -v sha256sum &> /dev/null; then
+    echo "Install script checksum:"
+    sha256sum "${install_lib}"
+  fi
+  source "${install_lib}"
+}
+
+psm::lang::source_install_lib
+source "${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh"
+psm::run "${PSM_TEST_SUITE}"

test/kokoro/psm-light.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 30
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "light"
+}

test/kokoro/psm-regional_td.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 30
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "regional_td"
+}

test/kokoro/psm-security.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 240
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "security"
+}

test/kokoro/psm-spiffe.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 240
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "spiffe"
+}

test/kokoro/xds.cfg

@@ -0,0 +1,11 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/xds.sh"
+timeout_mins: 360
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}

test/kokoro/xds.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -exu -o pipefail
+[[ -f /VERSION ]] && cat /VERSION
+
+cd github
+
+export GOPATH="${HOME}/gopath"
+pushd grpc-go/interop/xds/client
+# Install a version of Go supported by gRPC for the new features, e.g.
+# errors.Is()
+gofilename=go1.25.0.linux-amd64.tar.gz
+curl --retry 3 -O -L "https://go.dev/dl/${gofilename}"
+sudo tar -C /usr/local -xf "${gofilename}"
+sudo ln -s /usr/local/go/bin/go /usr/bin/go
+# Retry go build on errors (e.g. go get connection errors), for at most 3 times
+for i in 1 2 3; do go build && break || sleep 5; done
+popd
+
+git clone -b master --single-branch --depth=1 https://github.com/grpc/grpc.git
+
+grpc/tools/run_tests/helper_scripts/prep_xds.sh
+
+# Test cases "path_matching" and "header_matching" are not included in "all",
+# because not all interop clients in all languages support these new tests.
+#
+# TODO: remove "path_matching" and "header_matching" from --test_case after
+# they are added into "all".
+GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \
+  python3 grpc/tools/run_tests/run_xds_tests.py \
+    --test_case="ping_pong,circuit_breaking" \
+    --project_id=grpc-testing \
+    --project_num=830293263384 \
+    --source_image=projects/grpc-testing/global/images/xds-test-server-5 \
+    --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \
+    --gcp_suffix=$(date '+%s') \
+    --verbose \
+    ${XDS_V3_OPT-} \
+    --client_cmd="grpc-go/interop/xds/client/client \
+      --server=xds:///{server_uri} \
+      --stats_port={stats_port} \
+      --qps={qps} \
+      {fail_on_failed_rpc} \
+      {rpcs_to_send} \
+      {metadata_to_send}"

test/kokoro/xds_k8s_lb.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 300
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "lb"
+}

test/kokoro/xds_url_map.cfg

@@ -0,0 +1,17 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/psm-interop-test-go.sh"
+timeout_mins: 60
+
+action {
+  define_artifacts {
+    regex: "artifacts/**/*sponge_log.xml"
+    regex: "artifacts/**/*.log"
+    strip_prefix: "artifacts"
+  }
+}
+env_vars {
+  key: "PSM_TEST_SUITE"
+  value: "url_map"
+}

test/kokoro/xds_v3.cfg

@@ -0,0 +1,11 @@
+# Config file for internal CI
+
+# Location of the continuous shell script in repository.
+build_file: "grpc-go/test/kokoro/xds_v3.sh"
+timeout_mins: 360
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}

test/kokoro/xds_v3.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+XDS_V3_OPT="--xds_v3_support" `dirname $0`/xds.sh

test/local_creds_test.go

@@ -0,0 +1,218 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/netip"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/credentials/local"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func testLocalCredsE2ESucceed(t *testing.T, network, address string) error {
+	lis, err := net.Listen(network, address)
+	if err != nil {
+		return fmt.Errorf("Failed to create listener: %v", err)
+	}
+	ss := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			pr, ok := peer.FromContext(ctx)
+			if !ok {
+				return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx")
+			}
+			type internalInfo interface {
+				GetCommonAuthInfo() credentials.CommonAuthInfo
+			}
+			var secLevel credentials.SecurityLevel
+			if info, ok := (pr.AuthInfo).(internalInfo); ok {
+				secLevel = info.GetCommonAuthInfo().SecurityLevel
+			} else {
+				return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()")
+			}
+			// Check security level
+			switch network {
+			case "unix":
+				if secLevel != credentials.PrivacyAndIntegrity {
+					return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.PrivacyAndIntegrity)
+				}
+			case "tcp":
+				if secLevel != credentials.NoSecurity {
+					return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity)
+				}
+			}
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.Creds(local.NewCredentials())),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	var cc *grpc.ClientConn
+	lisAddr := lis.Addr().String()
+
+	switch network {
+	case "unix":
+		cc, err = grpc.NewClient("passthrough:///"+lisAddr, grpc.WithTransportCredentials(local.NewCredentials()), grpc.WithContextDialer(
+			func(_ context.Context, addr string) (net.Conn, error) {
+				return net.Dial("unix", addr)
+			}))
+	case "tcp":
+		cc, err = grpc.NewClient(lisAddr, grpc.WithTransportCredentials(local.NewCredentials()))
+	default:
+		return fmt.Errorf("unsupported network %q", network)
+	}
+	if err != nil {
+		return fmt.Errorf("Failed to create a client for server: %v, %v", err, lisAddr)
+	}
+	defer cc.Close()
+
+	c := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("EmptyCall(_, _) = _, %v; want _, <nil>", err)
+	}
+	return nil
+}
+
+func (s) TestLocalCredsLocalhost(t *testing.T) {
+	if err := testLocalCredsE2ESucceed(t, "tcp", "localhost:0"); err != nil {
+		t.Fatalf("Failed e2e test for localhost: %v", err)
+	}
+}
+
+func (s) TestLocalCredsUDS(t *testing.T) {
+	addr := fmt.Sprintf("/tmp/grpc_fullstck_test%d", time.Now().UnixNano())
+	if err := testLocalCredsE2ESucceed(t, "unix", addr); err != nil {
+		t.Fatalf("Failed e2e test for UDS: %v", err)
+	}
+}
+
+type connWrapper struct {
+	net.Conn
+	remote net.Addr
+}
+
+func (c connWrapper) RemoteAddr() net.Addr {
+	return c.remote
+}
+
+type lisWrapper struct {
+	net.Listener
+	remote net.Addr
+}
+
+func spoofListener(l net.Listener, remote net.Addr) net.Listener {
+	return &lisWrapper{l, remote}
+}
+
+func (l *lisWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	return connWrapper{c, l.remote}, nil
+}
+
+func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn, error) {
+	return func(t string, d time.Duration) (net.Conn, error) {
+		c, err := net.DialTimeout("tcp", t, d)
+		if err != nil {
+			return nil, err
+		}
+		return connWrapper{c, addr}, nil
+	}
+}
+
+func testLocalCredsE2EFail(t *testing.T, dopts []grpc.DialOption) error {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		return fmt.Errorf("Failed to create listener: %v", err)
+	}
+	var fakeClientAddr, fakeServerAddr net.Addr
+	fakeClientAddr = &net.IPAddr{
+		IP:   netip.MustParseAddr("10.8.9.10").AsSlice(),
+		Zone: "",
+	}
+	fakeServerAddr = &net.IPAddr{
+		IP:   netip.MustParseAddr("10.8.9.11").AsSlice(),
+		Zone: "",
+	}
+	ss := &stubserver.StubServer{
+		Listener: spoofListener(lis, fakeClientAddr),
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(grpc.Creds(local.NewCredentials())),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	cc, err := grpc.NewClient(lis.Addr().String(), append(dopts, grpc.WithDialer(spoofDialer(fakeServerAddr)))...)
+	if err != nil {
+		return fmt.Errorf("Failed to dial server: %v, %v", err, lis.Addr().String())
+	}
+	defer cc.Close()
+
+	c := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	_, err = c.EmptyCall(ctx, &testpb.Empty{})
+	return err
+}
+
+func isExpected(got, want error) bool {
+	return status.Code(got) == status.Code(want) && strings.Contains(status.Convert(got).Message(), status.Convert(want).Message())
+}
+
+func (s) TestLocalCredsClientFail(t *testing.T) {
+	// Use local creds at client-side which should lead to client-side failure.
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(local.NewCredentials())}
+	want := status.Error(codes.Unavailable, "transport: authentication handshake failed: local credentials rejected connection to non-local address")
+	if err := testLocalCredsE2EFail(t, opts); !isExpected(err, want) {
+		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, want)
+	}
+}
+
+func (s) TestLocalCredsServerFail(t *testing.T) {
+	// Use insecure at client-side which should lead to server-side failure.
+	opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}
+	if err := testLocalCredsE2EFail(t, opts); status.Code(err) != codes.Unavailable {
+		t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, codes.Unavailable)
+	}
+}

test/logging.go

@@ -0,0 +1,23 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import "google.golang.org/grpc/grpclog"
+
+var logger = grpclog.Component("testing")

test/malformed_method_test.go

@@ -0,0 +1,159 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"bytes"
+	"context"
+	"net"
+	"testing"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
+	"google.golang.org/grpc/internal/stubserver"
+
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestMalformedMethodPath tests that the server responds with Unimplemented
+// when the method path is malformed. This verifies that the server does not
+// route requests with a malformed method path to the application handler.
+func (s) TestMalformedMethodPath(t *testing.T) {
+	tests := []struct {
+		name       string
+		path       string
+		wantStatus string // string representation of codes.Code
+	}{
+		{
+			name:       "missing_leading_slash",
+			path:       "grpc.testing.TestService/UnaryCall",
+			wantStatus: "12", // Unimplemented
+		},
+		{
+			name:       "empty_path",
+			path:       "",
+			wantStatus: "12", // Unimplemented
+		},
+		{
+			name:       "just_slash",
+			path:       "/",
+			wantStatus: "12", // Unimplemented
+		},
+		{
+			name:       "double_slash",
+			path:       "//grpc.testing.TestService/UnaryCall",
+			wantStatus: "12", // Unimplemented
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			ss := &stubserver.StubServer{
+				UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+					return &testpb.SimpleResponse{Payload: &testpb.Payload{Body: []byte("pwned")}}, nil
+				},
+			}
+			if err := ss.Start(nil); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			// Open a raw TCP connection to the server and speak HTTP/2 directly.
+			tcpConn, err := net.Dial("tcp", ss.Address)
+			if err != nil {
+				t.Fatalf("Failed to dial tcp: %v", err)
+			}
+			defer tcpConn.Close()
+
+			// Write the HTTP/2 connection preface and the initial settings frame.
+			if _, err := tcpConn.Write([]byte("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")); err != nil {
+				t.Fatalf("Failed to write preface: %v", err)
+			}
+			framer := http2.NewFramer(tcpConn, tcpConn)
+			if err := framer.WriteSettings(); err != nil {
+				t.Fatalf("Failed to write settings: %v", err)
+			}
+
+			// Encode and write the HEADERS frame.
+			var headerBuf bytes.Buffer
+			enc := hpack.NewEncoder(&headerBuf)
+			writeHeader := func(name, value string) {
+				enc.WriteField(hpack.HeaderField{Name: name, Value: value})
+			}
+			writeHeader(":method", "POST")
+			writeHeader(":scheme", "http")
+			writeHeader(":authority", ss.Address)
+			writeHeader(":path", tc.path)
+			writeHeader("content-type", "application/grpc")
+			writeHeader("te", "trailers")
+			if err := framer.WriteHeaders(http2.HeadersFrameParam{
+				StreamID:      1,
+				BlockFragment: headerBuf.Bytes(),
+				EndStream:     false,
+				EndHeaders:    true,
+			}); err != nil {
+				t.Fatalf("Failed to write headers: %v", err)
+			}
+
+			// Send a small gRPC-encoded data frame (0 length).
+			if err := framer.WriteData(1, true, []byte{0, 0, 0, 0, 0}); err != nil {
+				t.Fatalf("Failed to write data: %v", err)
+			}
+
+			// Read responses and look for grpc-status.
+			gotStatus := ""
+			dec := hpack.NewDecoder(4096, func(f hpack.HeaderField) {
+				if f.Name == "grpc-status" {
+					gotStatus = f.Value
+				}
+			})
+			done := make(chan struct{})
+			go func() {
+				defer close(done)
+				for {
+					frame, err := framer.ReadFrame()
+					if err != nil {
+						return
+					}
+					if headers, ok := frame.(*http2.HeadersFrame); ok {
+						if _, err := dec.Write(headers.HeaderBlockFragment()); err != nil {
+							return
+						}
+						if headers.StreamEnded() {
+							return
+						}
+					}
+				}
+			}()
+
+			select {
+			case <-done:
+			case <-ctx.Done():
+				t.Fatalf("Timed out waiting for response")
+			}
+
+			if gotStatus != tc.wantStatus {
+				t.Errorf("Got grpc-status %v, want %v", gotStatus, tc.wantStatus)
+			}
+		})
+	}
+}

test/metadata_test.go

@@ -0,0 +1,155 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestInvalidMetadata(t *testing.T) {
+	grpctest.ExpectErrorN("stream: failed to validate md when setting trailer", 5)
+
+	tests := []struct {
+		name     string
+		md       metadata.MD
+		appendMD []string
+		want     error
+		recv     error
+	}{
+		{
+			name: "invalid key",
+			md:   map[string][]string{string(rune(0x19)): {"testVal"}},
+			want: status.Error(codes.Internal, "header key \"\\x19\" contains illegal characters not in [0-9a-z-_.]"),
+			recv: status.Error(codes.Internal, "invalid header field"),
+		},
+		{
+			name: "invalid value",
+			md:   map[string][]string{"test": {string(rune(0x19))}},
+			want: status.Error(codes.Internal, "header key \"test\" contains value with non-printable ASCII characters"),
+			recv: status.Error(codes.Internal, "invalid header field"),
+		},
+		{
+			name:     "invalid appended value",
+			md:       map[string][]string{"test": {"test"}},
+			appendMD: []string{"/", "value"},
+			want:     status.Error(codes.Internal, "header key \"/\" contains illegal characters not in [0-9a-z-_.]"),
+			recv:     status.Error(codes.Internal, "invalid header field"),
+		},
+		{
+			name:     "empty appended key",
+			md:       map[string][]string{"test": {"test"}},
+			appendMD: []string{"", "value"},
+			want:     status.Error(codes.Internal, "there is an empty key in the header"),
+			recv:     status.Error(codes.Internal, "invalid header field"),
+		},
+		{
+			name: "empty key",
+			md:   map[string][]string{"": {"test"}},
+			want: status.Error(codes.Internal, "there is an empty key in the header"),
+			recv: status.Error(codes.Internal, "invalid header field"),
+		},
+		{
+			name: "-bin key with arbitrary value",
+			md:   map[string][]string{"test-bin": {string(rune(0x19))}},
+			want: nil,
+			recv: io.EOF,
+		},
+		{
+			name: "valid key and value",
+			md:   map[string][]string{"test": {"value"}},
+			want: nil,
+			recv: io.EOF,
+		},
+	}
+
+	testNum := 0
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			_, err := stream.Recv()
+			if err != nil {
+				return err
+			}
+			test := tests[testNum]
+			testNum++
+			// merge original md and added md.
+			md := metadata.Join(test.md, metadata.Pairs(test.appendMD...))
+
+			if err := stream.SetHeader(md); !reflect.DeepEqual(test.want, err) {
+				return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want)
+			}
+			if err := stream.SendHeader(md); !reflect.DeepEqual(test.want, err) {
+				return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want)
+			}
+			stream.SetTrailer(md)
+			return nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting ss endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	for _, test := range tests {
+		t.Run("unary "+test.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			ctx = metadata.NewOutgoingContext(ctx, test.md)
+			ctx = metadata.AppendToOutgoingContext(ctx, test.appendMD...)
+			if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !reflect.DeepEqual(test.want, err) {
+				t.Errorf("call ss.Client.EmptyCall() validate metadata which is %v got err :%v, want err :%v", test.md, err, test.want)
+			}
+		})
+	}
+
+	// call the stream server's api to drive the server-side unit testing
+	for _, test := range tests {
+		t.Run("streaming "+test.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			stream, err := ss.Client.FullDuplexCall(ctx)
+			if err != nil {
+				t.Errorf("call ss.Client.FullDuplexCall got err :%v", err)
+				return
+			}
+			if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+				t.Errorf("call ss.Client stream Send(nil) will success but got err :%v", err)
+			}
+			if _, err := stream.Recv(); status.Code(err) != status.Code(test.recv) || !strings.Contains(err.Error(), test.recv.Error()) {
+				t.Errorf("stream.Recv() = _, get err :%v, want err :%v", err, test.recv)
+			}
+		})
+	}
+}

test/parse_config.go

@@ -0,0 +1,38 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"testing"
+
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+)
+
+// parseServiceConfig is a test helper which uses the manual resolver to parse
+// the given service config. It calls t.Fatal() if service config parsing fails.
+func parseServiceConfig(t *testing.T, r *manual.Resolver, sc string) *serviceconfig.ParseResult {
+	t.Helper()
+
+	scpr := r.CC().ParseServiceConfig(sc)
+	if scpr.Err != nil {
+		t.Fatalf("Failed to parse service config %q: %v", sc, scpr.Err)
+	}
+	return scpr
+}

test/race_test.go

@@ -0,0 +1,25 @@
+//go:build race
+// +build race
+
+/*
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+func init() {
+	raceMode = true
+}

test/rawConnWrapper.go

@@ -0,0 +1,247 @@
+/*
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
+)
+
+type listenerWrapper struct {
+	net.Listener
+	mu  sync.Mutex
+	rcw *rawConnWrapper
+}
+
+func listenWithConnControl(network, address string) (net.Listener, error) {
+	l, err := net.Listen(network, address)
+	if err != nil {
+		return nil, err
+	}
+	return &listenerWrapper{Listener: l}, nil
+}
+
+// Accept blocks until Dial is called, then returns a net.Conn for the server
+// half of the connection.
+func (l *listenerWrapper) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	l.mu.Lock()
+	l.rcw = newRawConnWrapperFromConn(c)
+	l.mu.Unlock()
+	return c, nil
+}
+
+func (l *listenerWrapper) getLastConn() *rawConnWrapper {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.rcw
+}
+
+type dialerWrapper struct {
+	c   net.Conn
+	rcw *rawConnWrapper
+}
+
+func (d *dialerWrapper) dialer(target string, t time.Duration) (net.Conn, error) {
+	c, err := net.DialTimeout("tcp", target, t)
+	d.c = c
+	d.rcw = newRawConnWrapperFromConn(c)
+	return c, err
+}
+
+func (d *dialerWrapper) getRawConnWrapper() *rawConnWrapper {
+	return d.rcw
+}
+
+type rawConnWrapper struct {
+	cc io.ReadWriteCloser
+	fr *http2.Framer
+
+	// writing headers:
+	headerBuf bytes.Buffer
+	hpackEnc  *hpack.Encoder
+
+	// reading frames:
+	frc    chan http2.Frame
+	frErrc chan error
+}
+
+func newRawConnWrapperFromConn(cc io.ReadWriteCloser) *rawConnWrapper {
+	rcw := &rawConnWrapper{
+		cc:     cc,
+		frc:    make(chan http2.Frame, 1),
+		frErrc: make(chan error, 1),
+	}
+	rcw.hpackEnc = hpack.NewEncoder(&rcw.headerBuf)
+	rcw.fr = http2.NewFramer(cc, cc)
+	rcw.fr.ReadMetaHeaders = hpack.NewDecoder(4096 /*initialHeaderTableSize*/, nil)
+
+	return rcw
+}
+
+func (rcw *rawConnWrapper) Close() error {
+	return rcw.cc.Close()
+}
+
+func (rcw *rawConnWrapper) encodeHeaderField(k, v string) error {
+	err := rcw.hpackEnc.WriteField(hpack.HeaderField{Name: k, Value: v})
+	if err != nil {
+		return fmt.Errorf("HPACK encoding error for %q/%q: %v", k, v, err)
+	}
+	return nil
+}
+
+// encodeRawHeader is for usage on both client and server side to construct header based on the input
+// key, value pairs.
+func (rcw *rawConnWrapper) encodeRawHeader(headers ...string) []byte {
+	if len(headers)%2 == 1 {
+		panic("odd number of kv args")
+	}
+
+	rcw.headerBuf.Reset()
+
+	pseudoCount := map[string]int{}
+	var keys []string
+	vals := map[string][]string{}
+
+	for len(headers) > 0 {
+		k, v := headers[0], headers[1]
+		headers = headers[2:]
+		if _, ok := vals[k]; !ok {
+			keys = append(keys, k)
+		}
+		if strings.HasPrefix(k, ":") {
+			pseudoCount[k]++
+			if pseudoCount[k] == 1 {
+				vals[k] = []string{v}
+			} else {
+				// Allows testing of invalid headers w/ dup pseudo fields.
+				vals[k] = append(vals[k], v)
+			}
+		} else {
+			vals[k] = append(vals[k], v)
+		}
+	}
+	for _, k := range keys {
+		for _, v := range vals[k] {
+			rcw.encodeHeaderField(k, v)
+		}
+	}
+	return rcw.headerBuf.Bytes()
+}
+
+// encodeHeader is for usage on client side to write request header.
+//
+// encodeHeader encodes headers and returns their HPACK bytes. headers
+// must contain an even number of key/value pairs.  There may be
+// multiple pairs for keys (e.g. "cookie").  The :method, :path, and
+// :scheme headers default to GET, / and https.
+func (rcw *rawConnWrapper) encodeHeader(headers ...string) []byte {
+	if len(headers)%2 == 1 {
+		panic("odd number of kv args")
+	}
+
+	rcw.headerBuf.Reset()
+
+	if len(headers) == 0 {
+		// Fast path, mostly for benchmarks, so test code doesn't pollute
+		// profiles when we're looking to improve server allocations.
+		rcw.encodeHeaderField(":method", "GET")
+		rcw.encodeHeaderField(":path", "/")
+		rcw.encodeHeaderField(":scheme", "https")
+		return rcw.headerBuf.Bytes()
+	}
+
+	if len(headers) == 2 && headers[0] == ":method" {
+		// Another fast path for benchmarks.
+		rcw.encodeHeaderField(":method", headers[1])
+		rcw.encodeHeaderField(":path", "/")
+		rcw.encodeHeaderField(":scheme", "https")
+		return rcw.headerBuf.Bytes()
+	}
+
+	pseudoCount := map[string]int{}
+	keys := []string{":method", ":path", ":scheme"}
+	vals := map[string][]string{
+		":method": {"GET"},
+		":path":   {"/"},
+		":scheme": {"https"},
+	}
+	for len(headers) > 0 {
+		k, v := headers[0], headers[1]
+		headers = headers[2:]
+		if _, ok := vals[k]; !ok {
+			keys = append(keys, k)
+		}
+		if strings.HasPrefix(k, ":") {
+			pseudoCount[k]++
+			if pseudoCount[k] == 1 {
+				vals[k] = []string{v}
+			} else {
+				// Allows testing of invalid headers w/ dup pseudo fields.
+				vals[k] = append(vals[k], v)
+			}
+		} else {
+			vals[k] = append(vals[k], v)
+		}
+	}
+	for _, k := range keys {
+		for _, v := range vals[k] {
+			rcw.encodeHeaderField(k, v)
+		}
+	}
+	return rcw.headerBuf.Bytes()
+}
+
+func (rcw *rawConnWrapper) writeHeaders(p http2.HeadersFrameParam) error {
+	if err := rcw.fr.WriteHeaders(p); err != nil {
+		return fmt.Errorf("error writing HEADERS: %v", err)
+	}
+	return nil
+}
+
+func (rcw *rawConnWrapper) writeRSTStream(streamID uint32, code http2.ErrCode) error {
+	if err := rcw.fr.WriteRSTStream(streamID, code); err != nil {
+		return fmt.Errorf("error writing RST_STREAM: %v", err)
+	}
+	return nil
+}
+
+func (rcw *rawConnWrapper) writeGoAway(maxStreamID uint32, code http2.ErrCode, debugData []byte) error {
+	if err := rcw.fr.WriteGoAway(maxStreamID, code, debugData); err != nil {
+		return fmt.Errorf("error writing GoAway: %v", err)
+	}
+	return nil
+}
+
+func (rcw *rawConnWrapper) writeDataFrame(streamID uint32, payload []byte) error {
+	if err := rcw.fr.WriteData(streamID, false, payload); err != nil {
+		return fmt.Errorf("error writing Raw Frame: %v", err)
+	}
+	return nil
+}

test/resolver_update_test.go

@@ -0,0 +1,263 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestResolverUpdateDuringBuild_ServiceConfigParseError makes the
+// resolver.Builder call into the ClientConn, during the Build call, with a
+// service config parsing error.
+//
+// We use two separate mutexes in the code which make sure there is no data race
+// in this code path, and also that there is no deadlock.
+func (s) TestResolverUpdateDuringBuild_ServiceConfigParseError(t *testing.T) {
+	// Setting InitialState on the manual resolver makes it call into the
+	// ClientConn during the Build call.
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Err: errors.New("resolver build err")}})
+
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	const wantMsg = "error parsing service config"
+	const wantCode = codes.Unavailable
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) {
+		t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg)
+	}
+}
+
+type fakeConfig struct {
+	serviceconfig.Config
+}
+
+// TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError makes the
+// resolver.Builder call into the ClientConn, during the Build call, with an
+// invalid service config type.
+//
+// We use two separate mutexes in the code which make sure there is no data race
+// in this code path, and also that there is no deadlock.
+func (s) TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError(t *testing.T) {
+	// Setting InitialState on the manual resolver makes it call into the
+	// ClientConn during the Build call.
+	r := manual.NewBuilderWithScheme("whatever")
+	r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Config: fakeConfig{}}})
+
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err)
+	}
+	defer cc.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	const wantMsg = "illegal service config type"
+	const wantCode = codes.Unavailable
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) {
+		t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg)
+	}
+}
+
+// TestResolverUpdate_InvalidServiceConfigAsFirstUpdate makes the resolver send
+// an update with an invalid service config as its first update. This should
+// make the ClientConn apply the failing LB policy, and should result in RPC
+// errors indicating the failing service config.
+func (s) TestResolverUpdate_InvalidServiceConfigAsFirstUpdate(t *testing.T) {
+	r := manual.NewBuilderWithScheme("whatever")
+
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err)
+	}
+	cc.Connect()
+	defer cc.Close()
+
+	scpr := r.CC().ParseServiceConfig("bad json service config")
+	r.UpdateState(resolver.State{ServiceConfig: scpr})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	const wantMsg = "error parsing service config"
+	const wantCode = codes.Unavailable
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) {
+		t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg)
+	}
+}
+
+func verifyClientConnStateUpdate(got, want balancer.ClientConnState) error {
+	if got, want := got.ResolverState.Addresses, want.ResolverState.Addresses; !cmp.Equal(got, want) {
+		return fmt.Errorf("update got unexpected addresses: %v, want %v", got, want)
+	}
+	if got, want := got.ResolverState.ServiceConfig.Config, want.ResolverState.ServiceConfig.Config; !internal.EqualServiceConfigForTesting(got, want) {
+		return fmt.Errorf("received unexpected service config: \ngot: %v \nwant: %v", got, want)
+	}
+	if got, want := got.BalancerConfig, want.BalancerConfig; !cmp.Equal(got, want) {
+		return fmt.Errorf("received unexpected balancer config: \ngot: %v \nwant: %v", cmp.Diff(nil, got), cmp.Diff(nil, want))
+	}
+	return nil
+}
+
+// TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate tests the scenario
+// where the resolver sends an update with an invalid service config after
+// having sent a good update. This should result in the ClientConn discarding
+// the new invalid service config, and continuing to use the old good config.
+func (s) TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate(t *testing.T) {
+	type wrappingBalancerConfig struct {
+		serviceconfig.LoadBalancingConfig
+		Config string `json:"config,omitempty"`
+	}
+
+	// Register a stub balancer which uses a "pick_first" balancer underneath and
+	// signals on a channel when it receives ClientConn updates.
+	ccUpdateCh := testutils.NewChannel()
+	stub.Register(t.Name(), stub.BalancerFuncs{
+		Init: func(bd *stub.BalancerData) {
+			pf := balancer.Get(pickfirst.Name)
+			bd.ChildBalancer = pf.Build(bd.ClientConn, bd.BuildOptions)
+		},
+		Close: func(bd *stub.BalancerData) {
+			bd.ChildBalancer.Close()
+		},
+		ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) {
+			cfg := &wrappingBalancerConfig{}
+			if err := json.Unmarshal(lbCfg, cfg); err != nil {
+				return nil, err
+			}
+			return cfg, nil
+		},
+		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
+			if _, ok := ccs.BalancerConfig.(*wrappingBalancerConfig); !ok {
+				return fmt.Errorf("received balancer config of unsupported type %T", ccs.BalancerConfig)
+			}
+			ccUpdateCh.Send(ccs)
+			ccs.BalancerConfig = nil
+			return bd.ChildBalancer.UpdateClientConnState(ccs)
+		},
+	})
+
+	// Start a backend exposing the test service.
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started TestService backend at: %q", backend.Address)
+	defer backend.Stop()
+
+	r := manual.NewBuilderWithScheme("whatever")
+
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r))
+	if err != nil {
+		t.Fatalf("NewClient(_, _) = _, %v; want _, nil", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	// Push a resolver update and verify that our balancer receives the update.
+	addrs := []resolver.Address{{Addr: backend.Address}}
+	const lbCfg = "wrapping balancer LB policy config"
+	goodSC := r.CC().ParseServiceConfig(fmt.Sprintf(`
+{
+  "loadBalancingConfig": [
+    {
+      "%v": {
+        "config": "%s"
+      }
+    }
+  ]
+}`, t.Name(), lbCfg))
+	r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: goodSC})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantCCS := balancer.ClientConnState{
+		ResolverState: resolver.State{
+			Addresses:     addrs,
+			ServiceConfig: goodSC,
+		},
+		BalancerConfig: &wrappingBalancerConfig{Config: lbCfg},
+	}
+	ccs, err := ccUpdateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for ClientConnState update from grpc")
+	}
+	gotCCS := ccs.(balancer.ClientConnState)
+	if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure RPCs are successful.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall RPC failed: %v", err)
+	}
+
+	// Push a bad resolver update and ensure that the update is propagated to our
+	// stub balancer. But since the pushed update contains an invalid service
+	// config, our balancer should continue to see the old loadBalancingConfig.
+	badSC := r.CC().ParseServiceConfig("bad json service config")
+	wantCCS.ResolverState.ServiceConfig = badSC
+	r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: badSC})
+	ccs, err = ccUpdateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for ClientConnState update from grpc")
+	}
+	gotCCS = ccs.(balancer.ClientConnState)
+	if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil {
+		t.Fatal(err)
+	}
+
+	// RPCs should continue to be successful since the ClientConn is using the old
+	// good service config.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall RPC failed: %v", err)
+	}
+}

test/roundrobin_test.go

@@ -0,0 +1,318 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/channelz"
+	imetadata "google.golang.org/grpc/internal/metadata"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	rrutil "google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const rrServiceConfig = `{"loadBalancingConfig": [{"round_robin":{}}]}`
+
+func testRoundRobinBasic(ctx context.Context, t *testing.T, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) {
+	t.Helper()
+
+	r := manual.NewBuilderWithScheme("whatever")
+
+	const backendCount = 5
+	backends := make([]*stubserver.StubServer, backendCount)
+	endpoints := make([]resolver.Endpoint, backendCount)
+	addrs := make([]resolver.Address, backendCount)
+	for i := 0; i < backendCount; i++ {
+		backend := &stubserver.StubServer{
+			EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+		}
+		if err := backend.StartServer(); err != nil {
+			t.Fatalf("Failed to start backend: %v", err)
+		}
+		t.Logf("Started TestService backend at: %q", backend.Address)
+		t.Cleanup(func() { backend.Stop() })
+
+		backends[i] = backend
+		addrs[i] = resolver.Address{Addr: backend.Address}
+		endpoints[i] = resolver.Endpoint{Addresses: []resolver.Address{addrs[i]}}
+	}
+
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithResolvers(r),
+		grpc.WithDefaultServiceConfig(rrServiceConfig),
+	}
+	dopts = append(dopts, opts...)
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...)
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// At this point, the resolver has not returned any addresses to the channel.
+	// This RPC must block until the context expires.
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded)
+	}
+
+	r.UpdateState(resolver.State{Addresses: addrs})
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+	return cc, r, backends
+}
+
+// TestRoundRobin_Basic tests the most basic scenario for round_robin. It brings
+// up a bunch of backends and verifies that RPCs are getting round robin-ed
+// across these backends.
+func (s) TestRoundRobin_Basic(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testRoundRobinBasic(ctx, t)
+}
+
+// TestRoundRobin_AddressesRemoved tests the scenario where a bunch of backends
+// are brought up, and round_robin is configured as the LB policy and RPCs are
+// being correctly round robin-ed across these backends. We then send a resolver
+// update with no addresses and verify that the channel enters TransientFailure
+// and RPCs fail with an expected error message.
+func (s) TestRoundRobin_AddressesRemoved(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, r, _ := testRoundRobinBasic(ctx, t)
+
+	// Send a resolver update with no addresses. This should push the channel into
+	// TransientFailure.
+	r.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{}})
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	const msgWant = "no children to pick from"
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); !strings.Contains(status.Convert(err).Message(), msgWant) {
+		t.Fatalf("EmptyCall() = %v, want Contains(Message(), %q)", err, msgWant)
+	}
+}
+
+// TestRoundRobin_NewAddressWhileBlocking tests the case where round_robin is
+// configured on a channel, things are working as expected and then a resolver
+// updates removes all addresses. An RPC attempted at this point in time will be
+// blocked because there are no valid ¡ds. This test verifies that when new
+// backends are added, the RPC is able to complete.
+func (s) TestRoundRobin_NewAddressWhileBlocking(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, r, backends := testRoundRobinBasic(ctx, t)
+
+	// Send a resolver update with no addresses. This should push the channel into
+	// TransientFailure.
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{}})
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	client := testgrpc.NewTestServiceClient(cc)
+	doneCh := make(chan struct{})
+	go func() {
+		// The channel is currently in TransientFailure and this RPC will block
+		// until the channel becomes Ready, which will only happen when we push a
+		// resolver update with a valid backend address.
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+			t.Errorf("EmptyCall() = %v, want <nil>", err)
+		}
+		close(doneCh)
+	}()
+
+	// Make sure that there is one pending RPC on the ClientConn before attempting
+	// to push new addresses through the name resolver. If we don't do this, the
+	// resolver update can happen before the above goroutine gets to make the RPC.
+	for {
+		if err := ctx.Err(); err != nil {
+			t.Fatal(err)
+		}
+		tcs, _ := channelz.GetTopChannels(0, 0)
+		if len(tcs) != 1 {
+			t.Fatalf("there should only be one top channel, not %d", len(tcs))
+		}
+		started := tcs[0].ChannelMetrics.CallsStarted.Load()
+		completed := tcs[0].ChannelMetrics.CallsSucceeded.Load() + tcs[0].ChannelMetrics.CallsFailed.Load()
+		if (started - completed) == 1 {
+			break
+		}
+		time.Sleep(defaultTestShortTimeout)
+	}
+
+	// Send a resolver update with a valid backend to push the channel to Ready
+	// and unblock the above RPC.
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}})
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for blocked RPC to complete")
+	case <-doneCh:
+	}
+}
+
+// TestRoundRobin_OneServerDown tests the scenario where a channel is configured
+// to round robin across a set of backends, and things are working correctly.
+// One backend goes down. The test verifies that going forward, RPCs are round
+// robin-ed across the remaining set of backends.
+func (s) TestRoundRobin_OneServerDown(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, _, backends := testRoundRobinBasic(ctx, t)
+
+	// Stop one backend. RPCs should round robin across the remaining backends.
+	backends[len(backends)-1].Stop()
+
+	addrs := make([]resolver.Address, len(backends)-1)
+	for i := 0; i < len(backends)-1; i++ {
+		addrs[i] = resolver.Address{Addr: backends[i].Address}
+	}
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatalf("RPCs are not being round robined across remaining servers: %v", err)
+	}
+}
+
+// TestRoundRobin_AllServersDown tests the scenario where a channel is
+// configured to round robin across a set of backends, and things are working
+// correctly. Then, all backends go down. The test verifies that the channel
+// moves to TransientFailure and failfast RPCs fail with Unavailable.
+func (s) TestRoundRobin_AllServersDown(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, _, backends := testRoundRobinBasic(ctx, t)
+
+	// Stop all backends.
+	for _, b := range backends {
+		b.Stop()
+	}
+
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Failfast RPCs should fail with Unavailable.
+	client := testgrpc.NewTestServiceClient(cc)
+
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+		t.Fatalf("EmptyCall got err: %v; want Unavailable", err)
+	}
+}
+
+// TestRoundRobin_UpdateAddressAttributes tests the scenario where the addresses
+// returned by the resolver contain attributes. The test verifies that the
+// attributes contained in the addresses show up as RPC metadata in the backend.
+func (s) TestRoundRobin_UpdateAddressAttributes(t *testing.T) {
+	const (
+		testMDKey   = "test-md"
+		testMDValue = "test-md-value"
+	)
+	r := manual.NewBuilderWithScheme("whatever")
+
+	// Spin up a StubServer to serve as a backend. The implementation verifies
+	// that the expected metadata is received.
+	testMDChan := make(chan []string, 1)
+	backend := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			md, ok := metadata.FromIncomingContext(ctx)
+			if ok {
+				select {
+				case testMDChan <- md[testMDKey]:
+				case <-ctx.Done():
+					return nil, ctx.Err()
+				}
+			}
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := backend.StartServer(); err != nil {
+		t.Fatalf("Failed to start backend: %v", err)
+	}
+	t.Logf("Started TestService backend at: %q", backend.Address)
+	t.Cleanup(func() { backend.Stop() })
+
+	// Dial the backend with round_robin as the LB policy.
+	dopts := []grpc.DialOption{
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithResolvers(r),
+		grpc.WithDefaultServiceConfig(rrServiceConfig),
+	}
+	// Set an initial resolver update with no address attributes.
+	addr := resolver.Address{Addr: backend.Address}
+	r.InitialState(resolver.State{Addresses: []resolver.Address{addr}})
+	cc, err := grpc.NewClient(r.Scheme()+":///test.server", dopts...)
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	// Make an RPC and ensure it does not contain the metadata we are looking for.
+	client := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() = %v, want <nil>", err)
+	}
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for metadata received in RPC")
+	case md := <-testMDChan:
+		if len(md) != 0 {
+			t.Fatalf("received metadata %v, want nil", md)
+		}
+	}
+
+	// Send a resolver update with address attributes.
+	addrWithAttributes := imetadata.Set(addr, metadata.Pairs(testMDKey, testMDValue))
+	r.UpdateState(resolver.State{Addresses: []resolver.Address{addrWithAttributes}})
+
+	// Make an RPC and ensure it contains the metadata we are looking for. The
+	// resolver update isn't processed synchronously, so we wait some time before
+	// failing if some RPCs do not contain it.
+Done:
+	for {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() = %v, want <nil>", err)
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatalf("Timeout when waiting for metadata received in RPC")
+		case md := <-testMDChan:
+			if len(md) == 1 && md[0] == testMDValue {
+				break Done
+			}
+		}
+		time.Sleep(defaultTestShortTimeout)
+	}
+}

test/server_test.go

@@ -0,0 +1,587 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"io"
+	"runtime/pprof"
+	"sync/atomic"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+type ctxKey string
+
+// TestServerReturningContextError verifies that if a context error is returned
+// by the service handler, the status will have the correct status code, not
+// Unknown.
+func (s) TestServerReturningContextError(t *testing.T) {
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return nil, context.DeadlineExceeded
+		},
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			return context.DeadlineExceeded
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	_, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
+	if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded {
+		t.Fatalf("ss.Client.EmptyCall() got error %v; want <status with Code()=DeadlineExceeded>", err)
+	}
+
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("unexpected error starting the stream: %v", err)
+	}
+	_, err = stream.Recv()
+	if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded {
+		t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want <status with Code()=DeadlineExceeded>", err)
+	}
+
+}
+
+func pprofCtxCollectLabels(ctx context.Context) map[string]string {
+	seenLabels := map[string]string{}
+	pprof.ForLabels(ctx, func(k, val string) bool {
+		seenLabels[k] = val
+		return true
+	})
+	return seenLabels
+}
+
+// TestServerSetGoroutineLabelsInContext verifies that when enabled, the
+// grpc.method runtime/pprof goroutine label gets set in the context that's
+// passed to the handlers.
+func (s) TestServerSetGoroutineLabelsInContext(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, envconfig.GoroutineLabelServerMethod)
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			ctxLabels := pprofCtxCollectLabels(ctx)
+			if val, ok := ctxLabels["grpc.method"]; !ok {
+				t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels)
+			} else if wantVal := "/grpc.testing.TestService/EmptyCall"; val != wantVal {
+				t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal)
+			}
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			ctxLabels := pprofCtxCollectLabels(stream.Context())
+			if val, ok := ctxLabels["grpc.method"]; !ok {
+				t.Errorf("missing \"grpc.method\" label; found labels: %v", ctxLabels)
+			} else if wantVal := "/grpc.testing.TestService/FullDuplexCall"; val != wantVal {
+				t.Errorf("unexpected value for \"grpc.method\" label %q; want %q", ctxLabels["grpc.method"], wantVal)
+			}
+			return nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err)
+	}
+
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("unexpected error starting the stream: %v", err)
+	}
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err)
+	}
+}
+
+// TestServerSetGoroutineLabelsInContextEnvVarDisabled verifies that when
+// disable, the grpc.method runtime/pprof goroutine label does _not_ get set in
+// the context that's passed to the handlers.
+func (s) TestServerSetGoroutineLabelsInContextEnvVarDisabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.LabelServerGoroutines, 0)
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			ctxLabels := pprofCtxCollectLabels(ctx)
+			if val, ok := ctxLabels["grpc.method"]; ok {
+				t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels)
+			}
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			ctxLabels := pprofCtxCollectLabels(stream.Context())
+			if val, ok := ctxLabels["grpc.method"]; ok {
+				t.Errorf("\"grpc.method\" label set with value %q; found labels: %v", val, ctxLabels)
+			}
+			return nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("ss.Client.EmptyCall() got error %v; want OK", err)
+	}
+
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("unexpected error starting the stream: %v", err)
+	}
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want io.EOF", err)
+	}
+}
+
+func (s) TestChainUnaryServerInterceptor(t *testing.T) {
+	var (
+		firstIntKey  = ctxKey("firstIntKey")
+		secondIntKey = ctxKey("secondIntKey")
+	)
+
+	firstInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if ctx.Value(firstIntKey) != nil {
+			return nil, status.Errorf(codes.Internal, "first interceptor should not have %v in context", firstIntKey)
+		}
+		if ctx.Value(secondIntKey) != nil {
+			return nil, status.Errorf(codes.Internal, "first interceptor should not have %v in context", secondIntKey)
+		}
+
+		firstCtx := context.WithValue(ctx, firstIntKey, 0)
+		resp, err := handler(firstCtx, req)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "failed to handle request at firstInt")
+		}
+
+		simpleResp, ok := resp.(*testpb.SimpleResponse)
+		if !ok {
+			return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at firstInt")
+		}
+		return &testpb.SimpleResponse{
+			Payload: &testpb.Payload{
+				Type: simpleResp.GetPayload().GetType(),
+				Body: append(simpleResp.GetPayload().GetBody(), '1'),
+			},
+		}, nil
+	}
+
+	secondInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if ctx.Value(firstIntKey) == nil {
+			return nil, status.Errorf(codes.Internal, "second interceptor should have %v in context", firstIntKey)
+		}
+		if ctx.Value(secondIntKey) != nil {
+			return nil, status.Errorf(codes.Internal, "second interceptor should not have %v in context", secondIntKey)
+		}
+
+		secondCtx := context.WithValue(ctx, secondIntKey, 1)
+		resp, err := handler(secondCtx, req)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "failed to handle request at secondInt")
+		}
+
+		simpleResp, ok := resp.(*testpb.SimpleResponse)
+		if !ok {
+			return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at secondInt")
+		}
+		return &testpb.SimpleResponse{
+			Payload: &testpb.Payload{
+				Type: simpleResp.GetPayload().GetType(),
+				Body: append(simpleResp.GetPayload().GetBody(), '2'),
+			},
+		}, nil
+	}
+
+	lastInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if ctx.Value(firstIntKey) == nil {
+			return nil, status.Errorf(codes.Internal, "last interceptor should have %v in context", firstIntKey)
+		}
+		if ctx.Value(secondIntKey) == nil {
+			return nil, status.Errorf(codes.Internal, "last interceptor should not have %v in context", secondIntKey)
+		}
+
+		resp, err := handler(ctx, req)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, "failed to handle request at lastInt at lastInt")
+		}
+
+		simpleResp, ok := resp.(*testpb.SimpleResponse)
+		if !ok {
+			return nil, status.Errorf(codes.Internal, "failed to get *testpb.SimpleResponse at lastInt")
+		}
+		return &testpb.SimpleResponse{
+			Payload: &testpb.Payload{
+				Type: simpleResp.GetPayload().GetType(),
+				Body: append(simpleResp.GetPayload().GetBody(), '3'),
+			},
+		}, nil
+	}
+
+	sopts := []grpc.ServerOption{
+		grpc.ChainUnaryInterceptor(firstInt, secondInt, lastInt),
+	}
+
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 0)
+			if err != nil {
+				return nil, status.Errorf(codes.Aborted, "failed to make payload: %v", err)
+			}
+
+			return &testpb.SimpleResponse{
+				Payload: payload,
+			}, nil
+		},
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{})
+	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
+		t.Fatalf("ss.Client.UnaryCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+	}
+
+	respBytes := resp.Payload.GetBody()
+	if string(respBytes) != "321" {
+		t.Fatalf("invalid response: want=%s, but got=%s", "321", resp)
+	}
+}
+
+func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) {
+	baseIntKey := ctxKey("baseIntKey")
+
+	baseInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if ctx.Value(baseIntKey) != nil {
+			return nil, status.Errorf(codes.Internal, "base interceptor should not have %v in context", baseIntKey)
+		}
+
+		baseCtx := context.WithValue(ctx, baseIntKey, 1)
+		return handler(baseCtx, req)
+	}
+
+	chainInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		if ctx.Value(baseIntKey) == nil {
+			return nil, status.Errorf(codes.Internal, "chain interceptor should have %v in context", baseIntKey)
+		}
+
+		return handler(ctx, req)
+	}
+
+	sopts := []grpc.ServerOption{
+		grpc.UnaryInterceptor(baseInt),
+		grpc.ChainUnaryInterceptor(chainInt),
+	}
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	resp, err := ss.Client.EmptyCall(ctx, &testpb.Empty{})
+	if s, ok := status.FromError(err); !ok || s.Code() != codes.OK {
+		t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, <status with Code()=OK>", resp, err)
+	}
+}
+
+func (s) TestChainStreamServerInterceptor(t *testing.T) {
+	callCounts := make([]atomic.Int32, 4)
+
+	firstInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		if callCounts[0].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[0] should be 0, but got=%d", callCounts[0].Load())
+		}
+		if callCounts[1].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[1] should be 0, but got=%d", callCounts[1].Load())
+		}
+		if callCounts[2].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load())
+		}
+		if callCounts[3].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load())
+		}
+		callCounts[0].Add(1)
+		return handler(srv, stream)
+	}
+
+	secondInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		if callCounts[0].Load() != 1 {
+			return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load())
+		}
+		if callCounts[1].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[1] should be 0, but got=%d", callCounts[1].Load())
+		}
+		if callCounts[2].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load())
+		}
+		if callCounts[3].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load())
+		}
+		callCounts[1].Add(1)
+		return handler(srv, stream)
+	}
+
+	lastInt := func(srv any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		if callCounts[0].Load() != 1 {
+			return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load())
+		}
+		if callCounts[1].Load() != 1 {
+			return status.Errorf(codes.Internal, "callCounts[1] should be 1, but got=%d", callCounts[1].Load())
+		}
+		if callCounts[2].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load())
+		}
+		if callCounts[3].Load() != 0 {
+			return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load())
+		}
+		callCounts[2].Add(1)
+		return handler(srv, stream)
+	}
+
+	sopts := []grpc.ServerOption{
+		grpc.ChainStreamInterceptor(firstInt, secondInt, lastInt),
+	}
+
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			if callCounts[0].Load() != 1 {
+				return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0].Load())
+			}
+			if callCounts[1].Load() != 1 {
+				return status.Errorf(codes.Internal, "callCounts[1] should be 1, but got=%d", callCounts[1].Load())
+			}
+			if callCounts[2].Load() != 1 {
+				return status.Errorf(codes.Internal, "callCounts[2] should be 0, but got=%d", callCounts[2].Load())
+			}
+			if callCounts[3].Load() != 0 {
+				return status.Errorf(codes.Internal, "callCounts[3] should be 0, but got=%d", callCounts[3].Load())
+			}
+			callCounts[3].Add(1)
+			return nil
+		},
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("failed to FullDuplexCall: %v", err)
+	}
+
+	_, err = stream.Recv()
+	if err != io.EOF {
+		t.Fatalf("failed to recv from stream: %v", err)
+	}
+
+	if callCounts[3].Load() != 1 {
+		t.Fatalf("callCounts[3] should be 1, but got=%d", callCounts[3].Load())
+	}
+}
+
+// Test verifies that only unary interceptors are invoked for unary RPCs and
+// only streaming interceptors are invoked for streaming RPCs.
+func (s) TestInterceptorSegregation(t *testing.T) {
+	var unaryCalled, streamCalled atomic.Bool
+
+	unaryInt := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		unaryCalled.Store(true)
+		return handler(ctx, req)
+	}
+	streamInt := func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		streamCalled.Store(true)
+		return handler(srv, ss)
+	}
+	sopts := []grpc.ServerOption{grpc.UnaryInterceptor(unaryInt), grpc.StreamInterceptor(streamInt)}
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			return nil
+		},
+	}
+	if err := ss.Start(sopts); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	// Make a unary RPC and ensure that only the unary interceptor was invoked.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+		t.Fatalf("ss.Client.UnaryCall failed: %v", err)
+	}
+	if !unaryCalled.Load() {
+		t.Error("Unary interceptor was not called for Unary RPC")
+	}
+	if streamCalled.Load() {
+		t.Error("Stream interceptor was called for Unary RPC")
+	}
+
+	// Make a streaming RPC and ensure that only the streaming interceptor was
+	// invoked.
+	unaryCalled.Store(false)
+	streamCalled.Store(false)
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("ss.Client.FullDuplexCall failed: %v", err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv() returned %v, want io.EOF", err)
+	}
+	if unaryCalled.Load() {
+		t.Error("Unary interceptor was called for Streaming RPC")
+	}
+	if !streamCalled.Load() {
+		t.Error("Stream interceptor was not called for Streaming RPC")
+	}
+}
+
+type wrappedTestStreamKey struct{}
+
+type wrappedTestStream struct {
+	grpc.ServerStream
+}
+
+func (w *wrappedTestStream) Context() context.Context {
+	return context.WithValue(w.ServerStream.Context(), wrappedTestStreamKey{}, w)
+}
+
+// Test verifies that an internal xDS filter wrapper option configured on the
+// server gets invoked and can wrap the ServerStream for both Unary and
+// Streaming RPCs.
+func (s) TestXDSFilterWrapperOption(t *testing.T) {
+	var wrapperCallCount atomic.Int32
+	wrapper := func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		wrapperCallCount.Add(1)
+		return &wrappedTestStream{ServerStream: ss}, nil
+	}
+
+	opt := internal.XDSFilterWrapperOption.(func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption)(wrapper)
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+			if _, ok := ctx.Value(wrappedTestStreamKey{}).(*wrappedTestStream); !ok {
+				return nil, status.Errorf(codes.Internal, "context value is %T, want *wrappedTestStream", ctx.Value(wrappedTestStreamKey{}))
+			}
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			if _, ok := stream.Context().Value(wrappedTestStreamKey{}).(*wrappedTestStream); !ok {
+				return status.Errorf(codes.Internal, "context value is %T, want *wrappedTestStream", stream.Context().Value(wrappedTestStreamKey{}))
+			}
+			return nil
+		},
+	}
+	if err := ss.Start([]grpc.ServerOption{opt}); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall failed: %v", err)
+	}
+	if got := wrapperCallCount.Load(); got != 1 {
+		t.Fatalf("XDSFilterWrapperOption callback call count for Unary RPC got %d, want 1", got)
+	}
+
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("FullDuplexCall failed: %v", err)
+	}
+	if _, err = stream.Recv(); err != io.EOF {
+		t.Fatalf("Recv failed: %v", err)
+	}
+	if got := wrapperCallCount.Load(); got != 2 {
+		t.Fatalf("XDSFilterWrapperOption callback call count for Streaming RPC got %d, want 2", got)
+	}
+}
+
+// Test verifies that if an internal xDS filter wrapper returns an error,
+// the RPC is rejected early with that status error before executing
+// handlers.
+func (s) TestXDSFilterWrapperOption_EarlyRejection(t *testing.T) {
+	wrapper := func(grpc.ServerStream) (grpc.ServerStream, error) {
+		return nil, status.Error(codes.PermissionDenied, "early rejection by internal wrapper")
+	}
+
+	opt := internal.XDSFilterWrapperOption.(func(func(grpc.ServerStream) (grpc.ServerStream, error)) grpc.ServerOption)(wrapper)
+
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		FullDuplexCallF: func(testgrpc.TestService_FullDuplexCallServer) error {
+			return nil
+		},
+	}
+	if err := ss.Start([]grpc.ServerOption{opt}); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.PermissionDenied {
+		t.Fatalf("EmptyCall failed with error %v; want PermissionDenied", err)
+	}
+
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("FullDuplexCall failed: %v", err)
+	}
+	if _, err = stream.Recv(); status.Code(err) != codes.PermissionDenied {
+		t.Fatalf("stream.Recv() got error %v; want PermissionDenied", err)
+	}
+}

test/servertester.go

@@ -0,0 +1,320 @@
+/*
+ * Copyright 2016 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package test contains tests.
+package test
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/hpack"
+)
+
+// This is a subset of http2's serverTester type.
+//
+// serverTester wraps a io.ReadWriter (acting like the underlying
+// network connection) and provides utility methods to read and write
+// http2 frames.
+//
+// NOTE(bradfitz): this could eventually be exported somewhere. Others
+// have asked for it too. For now I'm still experimenting with the
+// API and don't feel like maintaining a stable testing API.
+
+type serverTester struct {
+	cc io.ReadWriteCloser // client conn
+	t  testing.TB
+	fr *http2.Framer
+
+	// writing headers:
+	headerBuf bytes.Buffer
+	hpackEnc  *hpack.Encoder
+
+	// reading frames:
+	frc    chan http2.Frame
+	frErrc chan error
+}
+
+func newServerTesterFromConn(t testing.TB, cc io.ReadWriteCloser) *serverTester {
+	st := &serverTester{
+		t:      t,
+		cc:     cc,
+		frc:    make(chan http2.Frame, 1),
+		frErrc: make(chan error, 1),
+	}
+	st.hpackEnc = hpack.NewEncoder(&st.headerBuf)
+	st.fr = http2.NewFramer(cc, cc)
+	st.fr.ReadMetaHeaders = hpack.NewDecoder(4096 /*initialHeaderTableSize*/, nil)
+
+	return st
+}
+
+func (st *serverTester) readFrame() (http2.Frame, error) {
+	go func() {
+		fr, err := st.fr.ReadFrame()
+		if err != nil {
+			st.frErrc <- err
+		} else {
+			st.frc <- fr
+		}
+	}()
+	t := time.NewTimer(2 * time.Second)
+	defer t.Stop()
+	select {
+	case f := <-st.frc:
+		return f, nil
+	case err := <-st.frErrc:
+		return nil, err
+	case <-t.C:
+		return nil, errors.New("timeout waiting for frame")
+	}
+}
+
+// greet initiates the client's HTTP/2 connection into a state where
+// frames may be sent.
+func (st *serverTester) greet() {
+	st.greetWithSettings()
+}
+
+// greetWithSettings initiates the client's HTTP/2 connection with custom settings.
+func (st *serverTester) greetWithSettings(settings ...http2.Setting) {
+	st.writePreface()
+	if len(settings) > 0 {
+		if err := st.fr.WriteSettings(settings...); err != nil {
+			st.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err)
+		}
+	} else {
+		st.writeInitialSettings()
+	}
+	st.wantSettings()
+	st.writeSettingsAck()
+	for {
+		f, err := st.readFrame()
+		if err != nil {
+			st.t.Fatal(err)
+		}
+		switch f := f.(type) {
+		case *http2.WindowUpdateFrame:
+			// grpc's transport/http2_server sends this
+			// before the settings ack. The Go http2
+			// server uses a setting instead.
+		case *http2.SettingsFrame:
+			if f.IsAck() {
+				return
+			}
+			st.t.Fatalf("during greet, got non-ACK settings frame")
+		default:
+			st.t.Fatalf("during greet, unexpected frame type %T", f)
+		}
+	}
+}
+
+func (st *serverTester) writePreface() {
+	n, err := st.cc.Write([]byte(http2.ClientPreface))
+	if err != nil {
+		st.t.Fatalf("Error writing client preface: %v", err)
+	}
+	if n != len(http2.ClientPreface) {
+		st.t.Fatalf("Writing client preface, wrote %d bytes; want %d", n, len(http2.ClientPreface))
+	}
+}
+
+func (st *serverTester) writeInitialSettings() {
+	if err := st.fr.WriteSettings(); err != nil {
+		st.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err)
+	}
+}
+
+func (st *serverTester) writeSettingsAck() {
+	if err := st.fr.WriteSettingsAck(); err != nil {
+		st.t.Fatalf("Error writing ACK of server's SETTINGS: %v", err)
+	}
+}
+
+func (st *serverTester) wantGoAway(errCode http2.ErrCode) *http2.GoAwayFrame {
+	f, err := st.readFrame()
+	if err != nil {
+		st.t.Fatalf("Error while expecting an RST frame: %v", err)
+	}
+	gaf, ok := f.(*http2.GoAwayFrame)
+	if !ok {
+		st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f)
+	}
+	if gaf.ErrCode != errCode {
+		st.t.Fatalf("expected GOAWAY error code '%v', got '%v'", errCode.String(), gaf.ErrCode.String())
+	}
+	return gaf
+}
+
+func (st *serverTester) wantPing() *http2.PingFrame {
+	f, err := st.readFrame()
+	if err != nil {
+		st.t.Fatalf("Error while expecting an RST frame: %v", err)
+	}
+	pf, ok := f.(*http2.PingFrame)
+	if !ok {
+		st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f)
+	}
+	return pf
+}
+
+func (st *serverTester) wantRSTStream(errCode http2.ErrCode) *http2.RSTStreamFrame {
+	f, err := st.readFrame()
+	if err != nil {
+		st.t.Fatalf("Error while expecting an RST frame: %v", err)
+	}
+	rf, ok := f.(*http2.RSTStreamFrame)
+	if !ok {
+		st.t.Fatalf("got a %T; want *http2.RSTStreamFrame", f)
+	}
+	if rf.ErrCode != errCode {
+		st.t.Fatalf("expected RST error code '%v', got '%v'", errCode.String(), rf.ErrCode.String())
+	}
+	return rf
+}
+
+func (st *serverTester) wantSettings() *http2.SettingsFrame {
+	f, err := st.readFrame()
+	if err != nil {
+		st.t.Fatalf("Error while expecting a SETTINGS frame: %v", err)
+	}
+	sf, ok := f.(*http2.SettingsFrame)
+	if !ok {
+		st.t.Fatalf("got a %T; want *SettingsFrame", f)
+	}
+	return sf
+}
+
+// wait for any activity from the server
+func (st *serverTester) wantAnyFrame() http2.Frame {
+	f, err := st.fr.ReadFrame()
+	if err != nil {
+		st.t.Fatal(err)
+	}
+	return f
+}
+
+func (st *serverTester) encodeHeaderField(k, v string) {
+	err := st.hpackEnc.WriteField(hpack.HeaderField{Name: k, Value: v})
+	if err != nil {
+		st.t.Fatalf("HPACK encoding error for %q/%q: %v", k, v, err)
+	}
+}
+
+// encodeHeader encodes headers and returns their HPACK bytes. headers
+// must contain an even number of key/value pairs.  There may be
+// multiple pairs for keys (e.g. "cookie").  The :method, :path, and
+// :scheme headers default to GET, / and https.
+func (st *serverTester) encodeHeader(headers ...string) []byte {
+	if len(headers)%2 == 1 {
+		panic("odd number of kv args")
+	}
+
+	st.headerBuf.Reset()
+
+	if len(headers) == 0 {
+		// Fast path, mostly for benchmarks, so test code doesn't pollute
+		// profiles when we're looking to improve server allocations.
+		st.encodeHeaderField(":method", "GET")
+		st.encodeHeaderField(":path", "/")
+		st.encodeHeaderField(":scheme", "https")
+		return st.headerBuf.Bytes()
+	}
+
+	if len(headers) == 2 && headers[0] == ":method" {
+		// Another fast path for benchmarks.
+		st.encodeHeaderField(":method", headers[1])
+		st.encodeHeaderField(":path", "/")
+		st.encodeHeaderField(":scheme", "https")
+		return st.headerBuf.Bytes()
+	}
+
+	pseudoCount := map[string]int{}
+	keys := []string{":method", ":path", ":scheme"}
+	vals := map[string][]string{
+		":method": {"GET"},
+		":path":   {"/"},
+		":scheme": {"https"},
+	}
+	for len(headers) > 0 {
+		k, v := headers[0], headers[1]
+		headers = headers[2:]
+		if _, ok := vals[k]; !ok {
+			keys = append(keys, k)
+		}
+		if strings.HasPrefix(k, ":") {
+			pseudoCount[k]++
+			if pseudoCount[k] == 1 {
+				vals[k] = []string{v}
+			} else {
+				// Allows testing of invalid headers w/ dup pseudo fields.
+				vals[k] = append(vals[k], v)
+			}
+		} else {
+			vals[k] = append(vals[k], v)
+		}
+	}
+	for _, k := range keys {
+		for _, v := range vals[k] {
+			st.encodeHeaderField(k, v)
+		}
+	}
+	return st.headerBuf.Bytes()
+}
+
+func (st *serverTester) writeHeadersGRPC(streamID uint32, path string, endStream bool) {
+	st.writeHeaders(http2.HeadersFrameParam{
+		StreamID: streamID,
+		BlockFragment: st.encodeHeader(
+			":method", "POST",
+			":path", path,
+			":authority", "localhost",
+			"content-type", "application/grpc",
+			"te", "trailers",
+		),
+		EndStream:  endStream,
+		EndHeaders: true,
+	})
+}
+
+func (st *serverTester) writeHeaders(p http2.HeadersFrameParam) {
+	if err := st.fr.WriteHeaders(p); err != nil {
+		st.t.Fatalf("Error writing HEADERS: %v", err)
+	}
+}
+
+func (st *serverTester) writeData(streamID uint32, endStream bool, data []byte) {
+	if err := st.fr.WriteData(streamID, endStream, data); err != nil {
+		st.t.Fatalf("Error writing DATA: %v", err)
+	}
+}
+
+func (st *serverTester) writeRSTStream(streamID uint32, code http2.ErrCode) {
+	if err := st.fr.WriteRSTStream(streamID, code); err != nil {
+		st.t.Fatalf("Error writing RST_STREAM: %v", err)
+	}
+}
+
+func (st *serverTester) writePing(ack bool, data [8]byte) {
+	if err := st.fr.WritePing(ack, data); err != nil {
+		st.t.Fatalf("Error writing PING: %v", err)
+	}
+}

test/stats_test.go

@@ -0,0 +1,142 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/interop"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/stats"
+)
+
+// TestPeerForClientStatsHandler configures a stats handler that
+// verifies that peer is sent all stats handler callouts instead
+// of Begin and PickerUpdated.
+func (s) TestPeerForClientStatsHandler(t *testing.T) {
+	psh := &peerStatsHandler{}
+
+	// Stats callouts & peer object population.
+	// Note:
+	// * Begin stats lack peer info (RPC starts pre-resolution).
+	// * PickerUpdated: no peer info (picker lacks transport details).
+	expectedCallouts := map[stats.RPCStats]bool{
+		&stats.OutPayload{}:          true,
+		&stats.InHeader{}:            true,
+		&stats.OutHeader{}:           true,
+		&stats.InTrailer{}:           true,
+		&stats.OutTrailer{}:          true,
+		&stats.End{}:                 true,
+		&stats.Begin{}:               false,
+		&stats.DelayedPickComplete{}: false,
+	}
+
+	// Start server.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ss := &stubserver.StubServer{
+		Listener: l,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		S: grpc.NewServer(),
+	}
+	stubserver.StartTestService(t, ss)
+	defer ss.S.Stop()
+
+	// Create client with stats handler and do some calls.
+	cc, err := grpc.NewClient(
+		l.Addr().String(),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithStatsHandler(psh))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cc.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	interop.DoEmptyUnaryCall(ctx, client)
+
+	psh.mu.Lock()
+	pshArgs := psh.args
+	psh.mu.Unlock()
+
+	// Fetch the total unique stats handlers with peer != nil
+	uniqueStatsTypes := make(map[string]struct{})
+	for _, callbackArgs := range pshArgs {
+		key := fmt.Sprintf("%T", callbackArgs.rpcStats)
+		if _, exists := uniqueStatsTypes[key]; exists {
+			continue
+		}
+		uniqueStatsTypes[fmt.Sprintf("%T", callbackArgs.rpcStats)] = struct{}{}
+	}
+	if len(uniqueStatsTypes) != len(expectedCallouts) {
+		t.Errorf("Unexpected number of stats handler callouts. Got %v, want %v", len(uniqueStatsTypes), len(expectedCallouts))
+	}
+
+	for _, callbackArgs := range pshArgs {
+		expectedPeer, found := expectedCallouts[callbackArgs.rpcStats]
+		// In case expectation is set to false and still we got the peer,
+		// then it's good to have it. So no need to assert those conditions.
+		if found && expectedPeer && callbackArgs.peer != nil {
+			continue
+		} else if expectedPeer && callbackArgs.peer == nil {
+			t.Errorf("peer not populated for: %T", callbackArgs.rpcStats)
+		}
+	}
+}
+
+type peerStats struct {
+	rpcStats stats.RPCStats
+	peer     *peer.Peer
+}
+
+type peerStatsHandler struct {
+	args []peerStats
+	mu   sync.Mutex
+}
+
+func (h *peerStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context {
+	return ctx
+}
+
+func (h *peerStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	p, _ := peer.FromContext(ctx)
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	h.args = append(h.args, peerStats{rs, p})
+}
+
+func (h *peerStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+func (h *peerStatsHandler) HandleConn(context.Context, stats.ConnStats) {}

test/stream_cleanup_test.go

@@ -0,0 +1,139 @@
+/*
+ *
+ * Copyright 2019 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestStreamCleanup(t *testing.T) {
+	const initialWindowSize uint = 70 * 1024 // Must be higher than default 64K, ignored otherwise
+	const bodySize = 2 * initialWindowSize   // Something that is not going to fit in a single window
+	const callRecvMsgSize uint = 1           // The maximum message size the client can receive
+
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Payload: &testpb.Payload{
+				Body: make([]byte, bodySize),
+			}}, nil
+		},
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(callRecvMsgSize))), grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted {
+		t.Fatalf("should fail with ResourceExhausted, message's body size: %v, maximum message size the client can receive: %v", bodySize, callRecvMsgSize)
+	}
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("should succeed, err: %v", err)
+	}
+}
+
+func (s) TestStreamCleanupAfterSendStatus(t *testing.T) {
+	const initialWindowSize uint = 70 * 1024 // Must be higher than default 64K, ignored otherwise
+	const bodySize = 2 * initialWindowSize   // Something that is not going to fit in a single window
+
+	serverReturnedStatus := make(chan struct{})
+
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			defer func() {
+				close(serverReturnedStatus)
+			}()
+			return stream.Send(&testpb.StreamingOutputCallResponse{
+				Payload: &testpb.Payload{
+					Body: make([]byte, bodySize),
+				},
+			})
+		},
+	}
+	if err := ss.Start(nil, grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	// This test makes sure we don't delete stream from server transport's
+	// activeStreams list too aggressively.
+
+	// 1. Make a long living stream RPC. So server's activeStream list is not
+	// empty.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	stream, err := ss.Client.FullDuplexCall(ctx)
+	if err != nil {
+		t.Fatalf("FullDuplexCall= _, %v; want _, <nil>", err)
+	}
+
+	// 2. Wait for service handler to return status.
+	//
+	// This will trigger a stream cleanup code, which will eventually remove
+	// this stream from activeStream.
+	//
+	// But the stream removal won't happen because it's supposed to be done
+	// after the status is sent by loopyWriter, and the status send is blocked
+	// by flow control.
+	<-serverReturnedStatus
+
+	// 3. GracefulStop (besides sending goaway) checks the number of
+	// activeStreams.
+	//
+	// It will close the connection if there's no active streams. This won't
+	// happen because of the pending stream. But if there's a bug in stream
+	// cleanup that causes stream to be removed too aggressively, the connection
+	// will be closed and the stream will be broken.
+	gracefulStopDone := make(chan struct{})
+	go func() {
+		defer close(gracefulStopDone)
+		ss.S.GracefulStop()
+	}()
+
+	// 4. Make sure the stream is not broken.
+	if _, err := stream.Recv(); err != nil {
+		t.Fatalf("stream.Recv() = _, %v, want _, <nil>", err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		t.Fatalf("stream.Recv() = _, %v, want _, io.EOF", err)
+	}
+
+	timer := time.NewTimer(time.Second)
+	select {
+	case <-gracefulStopDone:
+		timer.Stop()
+	case <-timer.C:
+		t.Fatalf("s.GracefulStop() didn't finish within 1 second after the last RPC")
+	}
+}

test/subconn_test.go

@@ -0,0 +1,126 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/internal/balancer/stub"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+)
+
+type tsccPicker struct {
+	sc balancer.SubConn
+}
+
+func (p *tsccPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) {
+	return balancer.PickResult{SubConn: p.sc}, nil
+}
+
+// TestSubConnEmpty tests that removing all addresses from a SubConn and then
+// re-adding them does not cause a panic and properly reconnects.
+func (s) TestSubConnEmpty(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// sc is the one SubConn used throughout the test.  Created on demand and
+	// re-used on every update.
+	var sc balancer.SubConn
+
+	// Simple custom balancer that sets the address list to empty if the
+	// resolver produces no addresses.  Pickfirst, by default, will remove the
+	// SubConn in this case instead.
+	bal := stub.BalancerFuncs{
+		UpdateClientConnState: func(d *stub.BalancerData, ccs balancer.ClientConnState) error {
+			if sc == nil {
+				var err error
+				sc, err = d.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{
+					StateListener: func(state balancer.SubConnState) {
+						switch state.ConnectivityState {
+						case connectivity.Ready:
+							d.ClientConn.UpdateState(balancer.State{
+								ConnectivityState: connectivity.Ready,
+								Picker:            &tsccPicker{sc: sc},
+							})
+						case connectivity.TransientFailure:
+							d.ClientConn.UpdateState(balancer.State{
+								ConnectivityState: connectivity.TransientFailure,
+								Picker:            base.NewErrPicker(fmt.Errorf("error connecting: %v", state.ConnectionError)),
+							})
+						}
+					},
+				})
+				if err != nil {
+					t.Errorf("error creating initial subconn: %v", err)
+				}
+			} else {
+				d.ClientConn.UpdateAddresses(sc, ccs.ResolverState.Addresses)
+			}
+			sc.Connect()
+
+			if len(ccs.ResolverState.Addresses) == 0 {
+				d.ClientConn.UpdateState(balancer.State{
+					ConnectivityState: connectivity.TransientFailure,
+					Picker:            base.NewErrPicker(errors.New("no addresses")),
+				})
+			} else {
+				d.ClientConn.UpdateState(balancer.State{
+					ConnectivityState: connectivity.Connecting,
+					Picker:            &tsccPicker{sc: sc},
+				})
+			}
+			return nil
+		},
+	}
+	stub.Register("tscc", bal)
+
+	// Start the stub server with our stub balancer.
+	ss := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+	}
+	if err := ss.Start(nil, grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"tscc":{}}]}`)); err != nil {
+		t.Fatalf("Error starting server: %v", err)
+	}
+	defer ss.Stop()
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall failed: %v", err)
+	}
+
+	t.Log("Removing addresses from resolver and SubConn")
+	ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{}})
+	testutils.AwaitState(ctx, t, ss.CC, connectivity.TransientFailure)
+
+	t.Log("Re-adding addresses to resolver and SubConn")
+	ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}})
+	if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall failed: %v", err)
+	}
+}

test/timeouts.go

@@ -0,0 +1,29 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package test
+
+import "time"
+
+const (
+	// Default timeout for tests in this package.
+	defaultTestTimeout = 10 * time.Second
+	// Default short timeout, to be used when waiting for events which are not
+	// expected to happen.
+	defaultTestShortTimeout = 100 * time.Millisecond
+)

test/tools/go.mod

@@ -0,0 +1,29 @@
+module google.golang.org/grpc/test/tools
+
+go 1.25.0
+
+require (
+	github.com/client9/misspell v0.3.4
+	github.com/mgechev/revive v1.15.0
+	golang.org/x/tools v0.49.0
+	google.golang.org/protobuf v1.36.12
+	honnef.co/go/tools v0.7.0
+)
+
+require (
+	codeberg.org/chavacava/garif v0.2.1 // indirect
+	github.com/BurntSushi/toml v1.6.0 // indirect
+	github.com/fatih/color v1.19.0 // indirect
+	github.com/fatih/structtag v1.2.0 // indirect
+	github.com/hashicorp/go-version v1.9.0 // indirect
+	github.com/mattn/go-colorable v0.1.15 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/mgechev/dots v1.0.0 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297 // indirect
+	golang.org/x/mod v0.40.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1 // indirect
+	golang.org/x/text v0.41.0 // indirect
+)

test/tools/go.sum

@@ -0,0 +1,44 @@
+codeberg.org/chavacava/garif v0.2.1 h1:K9oYxSlvlXrHXyW26Z4q61bTpJDo1wbvXcYKar/F/LM=
+codeberg.org/chavacava/garif v0.2.1/go.mod h1:oHnDSmc0f9K1MeE+MQD/yjkiIB5Xsn5y3S9Dg96Xk84=
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
+github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
+github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
+github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
+github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
+github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
+github.com/mgechev/dots v1.0.0 h1:o+4OJ3OjWzgQHGJXKfJ8rbH4dqDugu5BiEy84nxg0k4=
+github.com/mgechev/dots v1.0.0/go.mod h1:rykuMydC9t3wfkM+ccYH3U3ss03vZGg6h3hmOznXLH0=
+github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q=
+github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297 h1:5MlQaTIH1l83SH0C4Xd5VtW0muW6qnokTvh0uybX1Lw=
+golang.org/x/exp/typeparams v0.0.0-20260813180055-c1d0aacb2297/go.mod h1:PqrXSW65cXDZH0k4IeUbhmg/bcAZDbzNz3byBpKCsXo=
+golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
+golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1 h1:tw/dhR3C+0y5Y8sTyXtV5GzjFE9hWblSlUUjF6Xs+Ko=
+golang.org/x/telemetry v0.0.0-20260814151720-d8c169486af1/go.mod h1:LVehoXe41cL5SCVQilsV7Gg6BNG+Js6P9PhSbYTIUkQ=
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
+golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM=
+golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU=
+honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc=

test/tools/tools.go

@@ -0,0 +1,34 @@
+//go:build tools
+// +build tools
+
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// This file is not intended to be compiled.  Because some of these imports are
+// not actual go packages, we use a build constraint at the top of this file to
+// prevent tools from inspecting the imports.
+
+package tools
+
+import (
+	_ "github.com/client9/misspell/cmd/misspell"
+	_ "github.com/mgechev/revive"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "google.golang.org/protobuf/cmd/protoc-gen-go"
+	_ "honnef.co/go/tools/cmd/staticcheck"
+)

test/tools/tools_vet.go

@@ -0,0 +1,21 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package tools is used to pin specific versions of external tools in this
+// module's go.mod that gRPC uses for internal testing.
+package tools

test/transport_test.go

@@ -0,0 +1,355 @@
+/*
+*
+* Copyright 2023 gRPC authors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+ */
+package test
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/net/http2"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/transport"
+	"google.golang.org/grpc/status"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// connWrapperWithCloseCh wraps a net.Conn and fires an event when closed.
+type connWrapperWithCloseCh struct {
+	net.Conn
+	close *grpcsync.Event
+}
+
+// Close closes the connection and sends a value on the close channel.
+func (cw *connWrapperWithCloseCh) Close() error {
+	cw.close.Fire()
+	return cw.Conn.Close()
+}
+
+// These custom creds are used for storing the connections made by the client.
+// The closeCh in conn can be used to detect when conn is closed.
+type transportRestartCheckCreds struct {
+	mu          sync.Mutex
+	connections []*connWrapperWithCloseCh
+}
+
+func (c *transportRestartCheckCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	return rawConn, nil, nil
+}
+func (c *transportRestartCheckCreds) ClientHandshake(_ context.Context, _ string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	conn := &connWrapperWithCloseCh{Conn: rawConn, close: grpcsync.NewEvent()}
+	c.connections = append(c.connections, conn)
+	return conn, nil, nil
+}
+func (c *transportRestartCheckCreds) Info() credentials.ProtocolInfo {
+	return credentials.ProtocolInfo{}
+}
+func (c *transportRestartCheckCreds) Clone() credentials.TransportCredentials {
+	return c
+}
+func (c *transportRestartCheckCreds) OverrideServerName(string) error {
+	return nil
+}
+
+// Tests that the client transport drains and restarts when next stream ID exceeds
+// MaxStreamID. This test also verifies that subsequent RPCs use a new client
+// transport and the old transport is closed.
+func (s) TestClientTransportRestartsAfterStreamIDExhausted(t *testing.T) {
+	// Set the transport's MaxStreamID to 4 to cause connection to drain after 2 RPCs.
+	originalMaxStreamID := transport.MaxStreamID
+	transport.MaxStreamID = 4
+	defer func() {
+		transport.MaxStreamID = originalMaxStreamID
+	}()
+
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			if _, err := stream.Recv(); err != nil {
+				return status.Errorf(codes.Internal, "unexpected error receiving: %v", err)
+			}
+			if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+				return status.Errorf(codes.Internal, "unexpected error sending: %v", err)
+			}
+			if recv, err := stream.Recv(); err != io.EOF {
+				return status.Errorf(codes.Internal, "Recv = %v, %v; want _, io.EOF", recv, err)
+			}
+			return nil
+		},
+	}
+
+	creds := &transportRestartCheckCreds{}
+	if err := ss.Start(nil, grpc.WithTransportCredentials(creds)); err != nil {
+		t.Fatalf("Starting stubServer: %v", err)
+	}
+	defer ss.Stop()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	var streams []testgrpc.TestService_FullDuplexCallClient
+
+	const numStreams = 3
+	// expected number of conns when each stream is created i.e., 3rd stream is created
+	// on a new connection.
+	expectedNumConns := [numStreams]int{1, 1, 2}
+
+	// Set up 3 streams.
+	for i := 0; i < numStreams; i++ {
+		s, err := ss.Client.FullDuplexCall(ctx)
+		if err != nil {
+			t.Fatalf("Creating FullDuplex stream: %v", err)
+		}
+		streams = append(streams, s)
+		// Verify expected num of conns after each stream is created.
+		if len(creds.connections) != expectedNumConns[i] {
+			t.Fatalf("Got number of connections created: %v, want: %v", len(creds.connections), expectedNumConns[i])
+		}
+	}
+
+	// Verify all streams still work.
+	for i, stream := range streams {
+		if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+			t.Fatalf("Sending on stream %d: %v", i, err)
+		}
+		if _, err := stream.Recv(); err != nil {
+			t.Fatalf("Receiving on stream %d: %v", i, err)
+		}
+	}
+
+	for i, stream := range streams {
+		if err := stream.CloseSend(); err != nil {
+			t.Fatalf("CloseSend() on stream %d: %v", i, err)
+		}
+	}
+
+	// Verifying first connection was closed.
+	select {
+	case <-creds.connections[0].close.Done():
+	case <-ctx.Done():
+		t.Fatal("Timeout expired when waiting for first client transport to close")
+	}
+}
+
+// Tests that an RST_STREAM frame that causes an io.ErrUnexpectedEOF while
+// reading a gRPC message is correctly converted to a gRPC status with code
+// CANCELLED. The test sends a data frame with a partial gRPC message, followed
+// by an RST_STREAM frame with HTTP/2 code CANCELLED. The test asserts the
+// client receives the correct status.
+func (s) TestRSTDuringMessageRead(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer lis.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%s) = %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+
+	go func() {
+		conn, err := lis.Accept()
+		if err != nil {
+			t.Errorf("lis.Accept() = %v", err)
+			return
+		}
+		defer conn.Close()
+		framer := http2.NewFramer(conn, conn)
+
+		if _, err := io.ReadFull(conn, make([]byte, len(clientPreface))); err != nil {
+			t.Errorf("Error while reading client preface: %v", err)
+			return
+		}
+		if err := framer.WriteSettings(); err != nil {
+			t.Errorf("Error while writing settings: %v", err)
+			return
+		}
+		if err := framer.WriteSettingsAck(); err != nil {
+			t.Errorf("Error while writing settings: %v", err)
+			return
+		}
+		for ctx.Err() == nil {
+			frame, err := framer.ReadFrame()
+			if err != nil {
+				return
+			}
+			switch frame := frame.(type) {
+			case *http2.HeadersFrame:
+				// When the client creates a stream, write a partial gRPC
+				// message followed by an RST_STREAM.
+				const messageLen = 2048
+				buf := make([]byte, messageLen/2)
+				// Write the gRPC message length header.
+				binary.BigEndian.PutUint32(buf[1:5], uint32(messageLen))
+				if err := framer.WriteData(1, false, buf); err != nil {
+					return
+				}
+				framer.WriteRSTStream(1, http2.ErrCodeCancel)
+			default:
+				t.Logf("Server received frame: %v", frame)
+			}
+		}
+	}()
+
+	// The server will send a partial gRPC message before cancelling the stream.
+	// The client should get a gRPC status with code CANCELLED.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled {
+		t.Fatalf("client.EmptyCall() returned %v; want status with code %v", err, codes.Canceled)
+	}
+}
+
+// Test verifies that a client-side cancellation correctly frees up resources on
+// the server. The test setup is designed to simulate a scenario where a server
+// is blocked from sending a large message due to a full client-side flow
+// control window. The client-side cancellation of this blocked RPC then frees
+// up the max concurrent streams quota on the server, allowing a new RPC to be
+// created successfully.
+func (s) TestCancelWhileServerWaitingForFlowControl(t *testing.T) {
+	serverDoneCh := make(chan struct{}, 2)
+	const flowControlWindowSize = 65535
+	ss := &stubserver.StubServer{
+		StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
+			// Send a large message to exhaust the client's flow control window.
+			stream.Send(&testpb.StreamingOutputCallResponse{
+				Payload: &testpb.Payload{
+					Body: make([]byte, flowControlWindowSize+1),
+				},
+			})
+			serverDoneCh <- struct{}{}
+			return nil
+		},
+	}
+
+	// Create a server that allows only 1 stream at a time.
+	ss = stubserver.StartTestService(t, ss, grpc.MaxConcurrentStreams(1))
+	defer ss.Stop()
+	// Use a static flow control window.
+	if err := ss.StartClient(grpc.WithStaticStreamWindowSize(flowControlWindowSize)); err != nil {
+		t.Fatalf("Error while start test service client: %v", err)
+	}
+	client := ss.Client
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	streamCtx, streamCancel := context.WithCancel(ctx)
+	defer streamCancel()
+
+	if _, err := client.StreamingOutputCall(streamCtx, &testpb.StreamingOutputCallRequest{}); err != nil {
+		t.Fatalf("Failed to create server streaming RPC: %v", err)
+	}
+
+	// Wait for the server handler to return. This should cause the trailers to
+	// be buffered on the server, waiting for flow control quota to first send
+	// the data frame.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Context timed out waiting for server handler to return.")
+	case <-serverDoneCh:
+	}
+
+	// Attempt to create a stream. It should fail since the previous stream is
+	// still blocked.
+	shortCtx, shortCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer shortCancel()
+	_, err := client.StreamingOutputCall(shortCtx, &testpb.StreamingOutputCallRequest{})
+	if status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("Server stream creation returned error with unexpected status code: %v, want code: %v", err, codes.DeadlineExceeded)
+	}
+
+	// Cancel the RPC, this should free up concurrent stream quota on the
+	// server.
+	streamCancel()
+
+	// Attempt to create another stream.
+	stream, err := client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{})
+	if err != nil {
+		t.Fatalf("Failed to create server streaming RPC: %v", err)
+	}
+	_, err = stream.Recv()
+	if err != nil {
+		t.Fatalf("Failed to read from the stream: %v", err)
+	}
+}
+
+// Tests that when a client sends a HEADERS frame with EndStream=true, the
+// server-side stream receives an io.EOF on Recv() and does not hang waiting
+// for data frames.
+func (s) TestHeadersEndStreamNoHang(t *testing.T) {
+	receivedErr := make(chan error, 1)
+	ss := &stubserver.StubServer{
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			_, err := stream.Recv()
+			receivedErr <- err
+			return nil
+		},
+	}
+	if err := ss.Start(nil); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+	defer ss.Stop()
+
+	conn, err := net.DialTimeout("tcp", ss.Address, defaultTestTimeout)
+	if err != nil {
+		t.Fatalf("Failed to dial: %v", err)
+	}
+	defer conn.Close()
+
+	st := newServerTesterFromConn(t, conn)
+	st.greet()
+
+	// Send HEADERS with EndStream = true and no grpc-timeout header.
+	st.writeHeaders(http2.HeadersFrameParam{
+		StreamID: 1,
+		BlockFragment: st.encodeHeader(
+			":method", "POST",
+			":path", "/grpc.testing.TestService/FullDuplexCall",
+			":authority", "localhost",
+			"content-type", "application/grpc",
+			"te", "trailers",
+		),
+		EndStream:  true,
+		EndHeaders: true,
+	})
+
+	select {
+	case err := <-receivedErr:
+		if err != io.EOF {
+			t.Errorf("Streaming handler returned error = %v, expected io.EOF", err)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatalf("Timed out waiting for Recv() on the server to complete")
+	}
+}

test/xds/xds_client_ack_nack_test.go

@@ -0,0 +1,202 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/resolver"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// We are interested in LDS, RDS, CDS and EDS resources as part of the regular
+// xDS flow on the client.
+const wantResources = 4
+
+// seenAllACKs returns true if the provided ackVersions map contains valid acks
+// for all the resources that we are interested in. If `wantNonEmpty` is true,
+// only non-empty ack versions are considered valid.
+func seenAllACKs(acksVersions map[string]string, wantNonEmpty bool) bool {
+	if len(acksVersions) != wantResources {
+		return false
+	}
+	for _, ack := range acksVersions {
+		if wantNonEmpty && ack == "" {
+			return false
+		}
+	}
+	return true
+}
+
+// TestClientResourceVersionAfterStreamRestart tests the scenario where the
+// xdsClient's ADS stream to the management server gets broken. This test
+// verifies that the version number on the initial request on the new stream
+// indicates the most recent version seen by the client on the previous stream.
+func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) {
+	// Create a restartable listener which can close existing connections.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	// We depend on the fact that the management server assigns monotonically
+	// increasing stream IDs starting at 1.
+	const (
+		idBeforeRestart = 1
+		idAfterRestart  = 2
+	)
+
+	// Events of importance in the test, in the order in which they are expected
+	// to happen.
+	acksReceivedBeforeRestart := grpcsync.NewEvent()
+	streamRestarted := grpcsync.NewEvent()
+	acksReceivedAfterRestart := grpcsync.NewEvent()
+
+	// Map from stream id to a map of resource type to resource version.
+	ackVersionsMap := make(map[int64]map[string]string)
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error {
+			// Return early under the following circumstances:
+			// - Received all the requests we wanted to see. This is to avoid
+			//   any stray requests leading to test flakes.
+			// - Request contains no resource names. Such requests are usually
+			//   seen when the xdsclient is shutting down and is no longer
+			//   interested in the resources that it had subscribed to earlier.
+			if acksReceivedAfterRestart.HasFired() || len(req.GetResourceNames()) == 0 {
+				return nil
+			}
+			// Create a stream specific map to store ack versions if this is the
+			// first time we are seeing this stream id.
+			if ackVersionsMap[id] == nil {
+				ackVersionsMap[id] = make(map[string]string)
+			}
+			ackVersionsMap[id][req.GetTypeUrl()] = req.GetVersionInfo()
+			// Prior to stream restart, we are interested only in non-empty
+			// resource versions. The xdsclient first sends out requests with an
+			// empty version string. After receipt of requested resource, it
+			// sends out another request for the same resource, but this time
+			// with a non-empty version string, to serve as an ACK.
+			if seenAllACKs(ackVersionsMap[idBeforeRestart], true) {
+				acksReceivedBeforeRestart.Fire()
+			}
+			// After stream restart, we expect the xdsclient to send out
+			// requests with version string set to the previously ACKed
+			// versions. If it sends out requests with empty version string, it
+			// is a bug and we want this test to catch it.
+			if seenAllACKs(ackVersionsMap[idAfterRestart], false) {
+				acksReceivedAfterRestart.Fire()
+			}
+			return nil
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			streamRestarted.Fire()
+		},
+	})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// A successful RPC means that the xdsclient received all requested
+	// resources. The ACKs from the xdsclient may get a little delayed. So, we
+	// need to wait for all ACKs to be received on the management server before
+	// restarting the stream.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for all resources to be ACKed prior to stream restart")
+	case <-acksReceivedBeforeRestart.Done():
+	}
+
+	// Stop the listener on the management server. This will cause the client to
+	// backoff and recreate the stream.
+	lis.Stop()
+
+	// Wait for the stream to be closed on the server.
+	<-streamRestarted.Done()
+
+	// Restart the listener on the management server to be able to accept
+	// reconnect attempts from the client.
+	lis.Restart()
+
+	// Wait for all the previously sent resources to be re-requested.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for all resources to be ACKed post stream restart")
+	case <-acksReceivedAfterRestart.Done():
+	}
+
+	if diff := cmp.Diff(ackVersionsMap[idBeforeRestart], ackVersionsMap[idAfterRestart]); diff != "" {
+		t.Fatalf("unexpected diff in ack versions before and after stream restart (-want, +got):\n%s", diff)
+	}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_affinity_test.go

@@ -0,0 +1,128 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// hashRouteConfig returns a RouteConfig resource with hash policy set to
+// header "session_id".
+func hashRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration {
+	return &v3routepb.RouteConfiguration{
+		Name: routeName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{ldsTarget},
+			Routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+				Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+					ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+					HashPolicy: []*v3routepb.RouteAction_HashPolicy{{
+						PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{
+							Header: &v3routepb.RouteAction_HashPolicy_Header{
+								HeaderName: "session_id",
+							},
+						},
+						Terminal: true,
+					}},
+				}},
+			}},
+		}},
+	}
+}
+
+// ringhashCluster returns a Cluster resource that picks ringhash as the lb
+// policy.
+func ringhashCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster {
+	return &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig: &v3corepb.ConfigSource{
+				ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{
+					Ads: &v3corepb.AggregatedConfigSource{},
+				},
+			},
+			ServiceName: edsServiceName,
+		},
+		LbPolicy: v3clusterpb.Cluster_RING_HASH,
+	}
+}
+
+// TestClientSideAffinitySanityCheck tests that the affinity config can be
+// propagated to pick the ring_hash policy. It doesn't test the affinity
+// behavior in ring_hash policy.
+func (s) TestClientSideAffinitySanityCheck(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	// Replace RDS and CDS resources with ringhash config, but keep the resource
+	// names.
+	resources.Routes = []*v3routepb.RouteConfiguration{hashRouteConfig(
+		resources.Routes[0].Name,
+		resources.Listeners[0].Name,
+		resources.Clusters[0].Name,
+	)}
+	resources.Clusters = []*v3clusterpb.Cluster{ringhashCluster(
+		resources.Clusters[0].Name,
+		resources.Clusters[0].EdsClusterConfig.ServiceName,
+	)}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_certificate_providers_test.go

@@ -0,0 +1,508 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with no security configuration. The test verifies that the gRPC client is
+// able to make RPCs to the backend which is configured to accept plaintext
+// connections. This ensures that the insecure fallback credentials are getting
+// used on the client.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, resolverBuilder := setup.ManagementServerAndResolver(t)
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with no
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains no certificate
+// providers, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server is configured to return client side xDS resources
+// with an mTLS security configuration. The test verifies that the gRPC client
+// moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and
+// string. This ensures that when the certificate provider instance name
+// specified in the security configuration is not present in the bootstrap,
+// channel creation does not fail, but it moves to TRANSIENT_FAILURE and
+// subsequent rpcs fail.
+func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) {
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server,
+	// with no certificate providers.
+	nodeID := uuid.New().String()
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	resolverBuilder, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server, with mTLS
+	// security configuration in the Cluster resource.
+	const serviceName = "my-service-client-side-xds"
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
+
+	// Make an RPC and ensure that expected error is returned.
+	wantErr := fmt.Sprintf("identity certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance)
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains one certificate
+// provider, and xDS credentials with an insecure fallback is specified at dial
+// time. The management server responds with three clusters:
+//  1. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap
+//  2. contains no security configuration, hence should use insecure fallback
+//  3. contains invalid security configuration pointing to a non-existent
+//     certificate provider instance
+//
+// The test verifies that RPCs to the first two clusters succeed, while RPCs to
+// the third cluster fails with an appropriate code and error message.
+func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) {
+	// Spin up an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var xdsResolver resolver.Builder
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		var err error
+		xdsResolver, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Create test backends for all three clusters
+	// backend1 configured with TLS creds, represents cluster1
+	// backend2 configured with insecure creds, represents cluster2
+	// backend3 configured with insecure creds, represents cluster3
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil)
+	defer server2.Stop()
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const clusterName3 = "cluster3-" + serviceName
+	const endpointsName1 = "endpoints1-" + serviceName
+	const endpointsName2 = "endpoints2-" + serviceName
+	const endpointsName3 = "endpoints3-" + serviceName
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
+					}},
+				},
+			},
+		}},
+	}}
+	// Clusters:
+	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance.
+	// - cluster2 with no security configuration.
+	// - cluster3 with non-existent cert provider name.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
+		func() *v3clusterpb.Cluster {
+			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
+			cluster3.TransportSocket = &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName: "non-existent-certificate-provider-instance-name",
+								},
+							},
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: "non-existent-certificate-provider-instance-name",
+							},
+						},
+					}),
+				},
+			}
+			return cluster3
+		}(),
+	}
+	// Endpoints for each of the above clusters with backends created earlier.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC to be routed to cluster1 and verify that it succeeds.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server1.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+
+	}
+
+	// Make an RPC to be routed to cluster2 and verify that it succeeds.
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("UnaryCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server2.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// Make an RPC to be routed to cluster3 and verify that it fails.
+	const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
+	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests the case where the bootstrap configuration contains one certificate
+// provider configured with SPIFFE Bundle Map roots on the client side, and xDS
+// credentials with an insecure fallback is specified at dial time. The
+// management server responds with three clusters:
+//  1. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap, and the server uses a SPIFFE cert.
+//  2. contains valid security configuration pointing to the certificate provider
+//     instance specified in the bootstrap, and the server uses a SPIFFE cert chain.
+//  3. contains invalid security configuration pointing to a non-existent
+//     certificate provider instance
+//
+// The test verifies that RPCs to the first two clusters succeed, while RPCs to
+// the third cluster fails with an appropriate code and error message.
+func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfigurationSPIFFE(t *testing.T) {
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t)
+
+	// Create test backends for all three clusters
+	// backend1 configured with a SPIFFE cert, represents cluster1
+	// backend2 configured with a SPIFFE cert chain, represents cluster2
+	// backend3 configured with insecure creds, represents cluster3
+	serverCreds := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFE(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	serverCreds2 := testutils.CreateServerTLSCredentialsCompatibleWithSPIFFEChain(t, tls.RequireAndVerifyClientCert)
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds2))
+	defer server2.Stop()
+	server3 := stubserver.StartTestService(t, nil)
+	defer server3.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const clusterName3 = "cluster3-" + serviceName
+	const endpointsName1 = "endpoints1-" + serviceName
+	const endpointsName2 = "endpoints2-" + serviceName
+	const endpointsName3 = "endpoints3-" + serviceName
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
+	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
+					}},
+				},
+				{
+					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
+					}},
+				},
+			},
+		}},
+	}}
+	// Clusters:
+	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance and mTLS.
+	// - cluster2 with cert provider name e2e.ClientSideCertProviderInstance and mTLS.
+	// - cluster3 with non-existent cert provider name.
+	clusters := []*v3clusterpb.Cluster{
+		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
+		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelMTLS),
+		func() *v3clusterpb.Cluster {
+			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
+			cluster3.TransportSocket = &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+									InstanceName: "non-existent-certificate-provider-instance-name",
+								},
+							},
+							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+								InstanceName: "non-existent-certificate-provider-instance-name",
+							},
+						},
+					}),
+				},
+			}
+			return cluster3
+		}(),
+	}
+	// Endpoints for each of the above clusters with backends created earlier.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+		e2e.DefaultEndpoint(endpointsName3, "localhost", []uint32{testutils.ParsePort(t, server3.Address)}),
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC to be routed to cluster1 and verify that it succeeds.
+	client := testgrpc.NewTestServiceClient(cc)
+	peer := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server1.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+	verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 1)
+
+	// Make an RPC to be routed to cluster2 and verify that it succeeds.
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("UnaryCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), server2.Address; got != want {
+		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+	// In this call the server contains a peer chain of length 2
+	verifySecurityInformationFromPeerSPIFFE(t, peer, e2e.SecurityLevelMTLS, 2)
+
+	// Make an RPC to be routed to cluster3 and verify that it fails.
+	const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
+	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}

test/xds/xds_client_custom_lb_test.go

@@ -0,0 +1,269 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	_ "google.golang.org/grpc/balancer/leastrequest"       // To register least_request
+	_ "google.golang.org/grpc/balancer/weightedroundrobin" // To register weighted_round_robin
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
+	v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3"
+	v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3"
+	v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// wrrLocality is a helper that takes a proto message and returns a
+// WrrLocalityProto with the proto message marshaled into a proto.Any as a
+// child.
+func wrrLocality(t *testing.T, m proto.Message) *v3wrrlocalitypb.WrrLocality {
+	return &v3wrrlocalitypb.WrrLocality{
+		EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{
+			Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+				{
+					TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+						TypedConfig: testutils.MarshalAny(t, m),
+					},
+				},
+			},
+		},
+	}
+}
+
+// clusterWithLBConfiguration returns a cluster resource with the proto message
+// passed Marshaled to an any and specified through the load_balancing_policy
+// field.
+func clusterWithLBConfiguration(t *testing.T, clusterName, edsServiceName string, secLevel e2e.SecurityLevel, m proto.Message) *v3clusterpb.Cluster {
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel)
+	cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{
+		Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{
+			{
+				TypedExtensionConfig: &v3corepb.TypedExtensionConfig{
+					TypedConfig: testutils.MarshalAny(t, m),
+				},
+			},
+		},
+	}
+	return cluster
+}
+
+// TestWRRLocality tests RPC distribution across a scenario with 5 backends,
+// with 2 backends in a locality with weight 1, and 3 backends in a second
+// locality with weight 2. Through xDS, the test configures a
+// wrr_locality_balancer with either a round robin or custom (specifying pick
+// first) child load balancing policy, and asserts the correct distribution
+// based on the locality weights and the endpoint picking policy specified.
+func (s) TestWrrLocality(t *testing.T) {
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+	backend3 := stubserver.StartTestService(t, nil)
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+	backend4 := stubserver.StartTestService(t, nil)
+	port4 := testutils.ParsePort(t, backend4.Address)
+	defer backend4.Stop()
+	backend5 := stubserver.StartTestService(t, nil)
+	port5 := testutils.ParsePort(t, backend5.Address)
+	defer backend5.Stop()
+	const serviceName = "my-service-client-side-xds"
+
+	tests := []struct {
+		name string
+		// Configuration will be specified through load_balancing_policy field.
+		wrrLocalityConfiguration *v3wrrlocalitypb.WrrLocality
+		addressDistributionWant  []struct {
+			addr  string
+			count int
+		}
+	}{
+		{
+			name:                     "rr_child",
+			wrrLocalityConfiguration: wrrLocality(t, &v3roundrobinpb.RoundRobin{}),
+			// Each addresses expected probability is locality weight of
+			// locality / total locality weights multiplied by 1 / number of
+			// endpoints in each locality (due to round robin across endpoints
+			// in a locality). Thus, address 1 and address 2 have 1/3 * 1/2
+			// probability, and addresses 3 4 5 have 2/3 * 1/3 probability of
+			// being routed to.
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+		// This configures custom lb as the child of wrr_locality, which points
+		// to our pick_first implementation. Thus, the expected distribution of
+		// addresses is locality weight of locality / total locality weights as
+		// the probability of picking the first backend within the locality
+		// (e.g. Address 1 for locality 1, and Address 3 for locality 2).
+		{
+			name: "custom_lb_child_pick_first",
+			wrrLocalityConfiguration: wrrLocality(t, &v3xdsxdstypepb.TypedStruct{
+				TypeUrl: "type.googleapis.com/pick_first",
+				Value:   &structpb.Struct{},
+			}),
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 1},
+				{addr: backend3.Address, count: 2},
+			},
+		},
+		// Sanity check for weighted round robin. Don't need to test super
+		// specific behaviors, as that is covered in unit tests. Set up weighted
+		// round robin as the endpoint picking policy with per RPC load reports
+		// enabled. Due the server not sending trailers with load reports, the
+		// weighted round robin policy should essentially function as round
+		// robin, and thus should have the same distribution as round robin
+		// above.
+		{
+			name: "custom_lb_child_wrr/",
+			wrrLocalityConfiguration: wrrLocality(t, &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{
+				EnableOobLoadReport: &wrapperspb.BoolValue{
+					Value: false,
+				},
+				// BlackoutPeriod long enough to cause load report weights to
+				// trigger in the scope of test case, but no load reports
+				// configured anyway.
+				BlackoutPeriod:          durationpb.New(10 * time.Second),
+				WeightExpirationPeriod:  durationpb.New(10 * time.Second),
+				WeightUpdatePeriod:      durationpb.New(time.Second),
+				ErrorUtilizationPenalty: &wrapperspb.FloatValue{Value: 1},
+			}),
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+		{
+			name: "custom_lb_least_request",
+			wrrLocalityConfiguration: wrrLocality(t, &v3leastrequestpb.LeastRequest{
+				ChoiceCount: wrapperspb.UInt32(2),
+			}),
+			// The test performs a Unary RPC, and blocks until the RPC returns,
+			// and then makes the next Unary RPC. Thus, over iterations, no RPC
+			// counts are present. This causes least request's randomness of
+			// indexes to sample to converge onto a round robin distribution per
+			// locality. Thus, expect the same distribution as round robin
+			// above.
+			addressDistributionWant: []struct {
+				addr  string
+				count int
+			}{
+				{addr: backend1.Address, count: 6},
+				{addr: backend2.Address, count: 6},
+				{addr: backend3.Address, count: 8},
+				{addr: backend4.Address, count: 8},
+				{addr: backend5.Address, count: 8},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Start an xDS management server.
+			managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			routeConfigName := "route-" + serviceName
+			clusterName := "cluster-" + serviceName
+			endpointsName := "endpoints-" + serviceName
+			resources := e2e.UpdateOptions{
+				NodeID:    nodeID,
+				Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+				Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+				Clusters:  []*v3clusterpb.Cluster{clusterWithLBConfiguration(t, clusterName, endpointsName, e2e.SecurityLevelNone, test.wrrLocalityConfiguration)},
+				Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+					ClusterName: endpointsName,
+					Host:        "localhost",
+					Localities: []e2e.LocalityOptions{
+						{
+							Backends: []e2e.BackendOptions{{Ports: []uint32{port1}}, {Ports: []uint32{port2}}},
+							Weight:   1,
+						},
+						{
+							Backends: []e2e.BackendOptions{{Ports: []uint32{port3}}, {Ports: []uint32{port4}}, {Ports: []uint32{port5}}},
+							Weight:   2,
+						},
+					},
+				})},
+			}
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("Failed to dial local test server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testgrpc.NewTestServiceClient(cc)
+			var addrDistWant []resolver.Address
+			for _, addrAndCount := range test.addressDistributionWant {
+				for i := 0; i < addrAndCount.count; i++ {
+					addrDistWant = append(addrDistWant, resolver.Address{Addr: addrAndCount.addr})
+				}
+			}
+			if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, t, client, addrDistWant); err != nil {
+				t.Fatalf("Error in expected round robin: %v", err)
+			}
+		})
+	}
+}

test/xds/xds_client_federation_test.go

@@ -0,0 +1,343 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestClientSideFederation tests that federation is supported.
+//
+// In this test, some xDS responses contain resource names in another authority
+// (in the new resource name style):
+// - LDS: old style, no authority (default authority)
+// - RDS: new style, in a different authority
+// - CDS: old style, no authority (default authority)
+// - EDS: new style, in a different authority
+func (s) TestClientSideFederation(t *testing.T) {
+	// Start a management server as the default authority.
+	serverDefaultAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Start another management server as the other authority.
+	const nonDefaultAuth = "non-default-auth"
+	serverAnotherAuth := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create a bootstrap file in a temporary directory.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverDefaultAuth.Address)),
+		Node:                               []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+		// Specify the address of the non-default authority.
+		Authorities: map[string]json.RawMessage{
+			nonDefaultAuth: []byte(fmt.Sprintf(`{
+				"xds_servers": [
+					{
+						"server_uri": %q,
+						"channel_creds": [{"type": "insecure"}]
+					}
+				]
+			}`, serverAnotherAuth.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	// LDS is old style name.
+	ldsName := serviceName
+	// RDS is new style, with the non default authority.
+	rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", nonDefaultAuth, "route-"+serviceName)
+	// CDS is old style name.
+	cdsName := "cluster-" + serviceName
+	// EDS is new style, with the non default authority.
+	edsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.ClusterLoadAssignment/%s", nonDefaultAuth, "endpoints-"+serviceName)
+
+	// Split resources, put LDS/CDS in the default authority, and put RDS/EDS in
+	// the other authority.
+	resourcesDefault := e2e.UpdateOptions{
+		NodeID: nodeID,
+		// This has only LDS and CDS.
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
+		SkipValidation: true,
+	}
+	resourcesAnother := e2e.UpdateOptions{
+		NodeID: nodeID,
+		// This has only RDS and EDS.
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// This has only LDS and CDS.
+	if err := serverDefaultAuth.Update(ctx, resourcesDefault); err != nil {
+		t.Fatal(err)
+	}
+	// This has only RDS and EDS.
+	if err := serverAnotherAuth.Update(ctx, resourcesAnother); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestClientSideFederationWithOnlyXDSTPStyleLDS tests that federation is
+// supported with new xdstp style names for LDS only while using the old style
+// for other resources. This test in addition also checks that when service name
+// contains escapable characters, we "fully" encode it for looking up
+// VirtualHosts in xDS RouteConfiguration.
+func (s) TestClientSideFederationWithOnlyXDSTPStyleLDS(t *testing.T) {
+	// Start a management server as a sophisticated authority.
+	const authority = "traffic-manager.xds.notgoogleapis.com"
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create a bootstrap file in a temporary directory.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		ClientDefaultListenerResourceNameTemplate: fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%%s", authority),
+		// Specify the address of the non-default authority.
+		Authorities: map[string]json.RawMessage{
+			authority: []byte(fmt.Sprintf(`{
+				"xds_servers": [
+					{
+						"server_uri": %q,
+						"channel_creds": [{"type": "insecure"}]
+					}
+				]
+			}`, mgmtServer.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	resolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// serviceName with escapable characters - ' ', and '/'.
+	const serviceName = "my-service-client-side-xds/2nd component"
+
+	// All other resources are with old style name.
+	const rdsName = "route-" + serviceName
+	const cdsName = "cluster-" + serviceName
+	const edsName = "endpoints-" + serviceName
+
+	// Resource update sent to go-control-plane mgmt server.
+	resourceUpdate := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: func() []*v3listenerpb.Listener {
+			// LDS is new style xdstp name. Since the LDS resource name is prefixed
+			// with xdstp, the string will be %-encoded excluding '/'s. See
+			// bootstrap.PopulateResourceTemplate().
+			const specialEscapedServiceName = "my-service-client-side-xds/2nd%20component" // same as bootstrap.percentEncode(serviceName)
+			ldsName := fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/%s", authority, specialEscapedServiceName)
+			return []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}
+		}(),
+		Routes: func() []*v3routepb.RouteConfiguration {
+			// RouteConfiguration will has one entry in []VirtualHosts that contains the
+			// "fully" escaped service name in []Domains. This is to assert that gRPC
+			// uses the escaped service name to lookup VirtualHosts. RDS is also with
+			// old style name.
+			const fullyEscapedServiceName = "my-service-client-side-xds%2F2nd%20component" // same as url.PathEscape(serviceName)
+			return []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, fullyEscapedServiceName, cdsName)}
+		}(),
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)},
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+		SkipValidation: true,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resourceUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestFederation_UnknownAuthorityInDialTarget tests the case where a ClientConn
+// is created with a dial target containing an authority which is not specified
+// in the bootstrap configuration. The test verifies that RPCs on the ClientConn
+// fail with an appropriate error.
+func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) {
+	// Setting up the management server is not *really* required for this test
+	// case. All we need is a bootstrap configuration which does not contain the
+	// authority mentioned in the dial target. But setting up the management
+	// server and actually making an RPC ensures that the xDS client is
+	// configured properly, and when we dial with an unknown authority in the
+	// next step, we can be sure that the error we receive is legitimate.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	target := fmt.Sprintf("xds:///%s", serviceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed %q: %v", target, err)
+	}
+	defer cc.Close()
+	t.Log("Created ClientConn to test service")
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() RPC: %v", err)
+	}
+	t.Log("Successfully performed an EmptyCall RPC")
+
+	target = fmt.Sprintf("xds://unknown-authority/%s", serviceName)
+	t.Logf("Creating a channel with unknown authority %q, expecting failure", target)
+	wantErr := fmt.Sprintf("authority \"unknown-authority\" specified in dial target %q is not found in the bootstrap file", target)
+	cc, err = grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Unexpected error while creating ClientConn: %v", err)
+	}
+	defer cc.Close()
+	client = testgrpc.NewTestServiceClient(cc)
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("EmptyCall(_, _) = _, %v; want _, %q", err, wantErr)
+	}
+}
+
+// TestFederation_UnknownAuthorityInReceivedResponse tests the case where the
+// LDS resource associated with the dial target contains an RDS resource name
+// with an authority which is not specified in the bootstrap configuration. The
+// test verifies that RPCs fail with an appropriate error.
+func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) {
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// LDS is old style name.
+	// RDS is new style, with an unknown authority.
+	const serviceName = "my-service-client-side-xds"
+	const unknownAuthority = "unknown-authority"
+	ldsName := serviceName
+	rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", unknownAuthority, "route-"+serviceName)
+
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)},
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "cluster-"+serviceName)},
+		SkipValidation: true, // This update has only LDS and RDS resources.
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	target := fmt.Sprintf("xds:///%s", serviceName)
+	cc, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed %q: %v", target, err)
+	}
+	defer cc.Close()
+	t.Log("Created ClientConn to test service")
+
+	client := testgrpc.NewTestServiceClient(cc)
+	_, err = client.EmptyCall(ctx, &testpb.Empty{})
+	if err == nil {
+		t.Fatal("EmptyCall RPC succeeded for target with unknown authority when expected to fail")
+	}
+	if got, want := status.Code(err), codes.Unavailable; got != want {
+		t.Fatalf("EmptyCall RPC returned status code: %v, want %v", got, want)
+	}
+}

test/xds/xds_client_filter_e2e_test.go

@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// Tests the interaction between the defaultStreamInterceptor and xDS filters.
+// It verifies that the SendMsg, RecvMsg, and CloseSend methods are called the
+// expected number of times for different RPC types (unary, client streaming,
+// server streaming, and bidi streaming) when an xDS filter is registered and
+// used in the call chain.
+func (s) TestDefaultStreamInterceptor_InteractionWithXDSFilters(t *testing.T) {
+	// Register a custom xDS filter builder for the test.
+	testFilterTypeURL := t.Name()
+	filterBuilder := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	httpfilter.Register(filterBuilder)
+	defer httpfilter.UnregisterForTesting(testFilterTypeURL)
+
+	// Setup a test backend implementing both all four RPC types.
+	testServer := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		StreamingOutputCallF: func(_ *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error {
+			return stream.Send(&testpb.StreamingOutputCallResponse{})
+		},
+		StreamingInputCallF: func(stream testgrpc.TestService_StreamingInputCallServer) error {
+			for {
+				_, err := stream.Recv()
+				if err == io.EOF {
+					break
+				}
+				if err != nil {
+					return err
+				}
+			}
+			return stream.SendAndClose(&testpb.StreamingInputCallResponse{})
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				_, err := stream.Recv()
+				if err == io.EOF {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+				if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	if err := testServer.Start(nil); err != nil {
+		t.Fatal("Error starting server:", err)
+	}
+	defer testServer.Stop()
+
+	// Start an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	const serviceName = "my-service-xds"
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: "cluster-A"}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "tracking-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster("cluster-A", "cluster-A", e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint("cluster-A", "localhost", []uint32{testutils.ParsePort(t, testServer.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+	client := testgrpc.NewTestServiceClient(cc)
+
+	tests := []struct {
+		name        string
+		run         func(ctx context.Context, client testgrpc.TestServiceClient)
+		wantSendMsg int32
+		wantRecvMsg int32
+	}{
+		{
+			name: "unary",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+					t.Fatalf("EmptyCall() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 2, // 1 response + 1 trailers
+		},
+		{
+			name: "client_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.StreamingInputCall(ctx)
+				if err != nil {
+					t.Fatalf("StreamingInputCall() failed: %v", err)
+				}
+				for i := 0; i < 2; i++ {
+					if err := stream.Send(&testpb.StreamingInputCallRequest{}); err != nil {
+						t.Fatalf("Send() failed: %v", err)
+					}
+				}
+				if _, err := stream.CloseAndRecv(); err != nil {
+					t.Fatalf("CloseAndRecv() failed: %v", err)
+				}
+			},
+			wantSendMsg: 2, // 2 requests
+			wantRecvMsg: 2, // 1 response + 1 trailers
+		},
+		{
+			name: "server_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{})
+				if err != nil {
+					t.Fatalf("StreamingOutputCall() failed: %v", err)
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("Recv() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 1, // 1 response (trailers not yet consumed since stream not read to EOF)
+		},
+		{
+			name: "bidi_streaming",
+			run: func(ctx context.Context, client testgrpc.TestServiceClient) {
+				stream, err := client.FullDuplexCall(ctx)
+				if err != nil {
+					t.Fatalf("FullDuplexCall() failed: %v", err)
+				}
+				if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+					t.Fatalf("Send() failed: %v", err)
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("Recv() failed: %v", err)
+				}
+				if err := stream.CloseSend(); err != nil {
+					t.Fatalf("CloseSend() failed: %v", err)
+				}
+			},
+			wantSendMsg: 1, // 1 request
+			wantRecvMsg: 1, // 1 response
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			filterBuilder.sendMsgCount.Store(0)
+			filterBuilder.recvMsgCount.Store(0)
+			filterBuilder.closeSendCount.Store(0)
+
+			tc.run(ctx, client)
+
+			if got := filterBuilder.sendMsgCount.Load(); got != tc.wantSendMsg {
+				t.Fatalf("SendMsg() count = %d, want %d", got, tc.wantSendMsg)
+			}
+			if got := filterBuilder.recvMsgCount.Load(); got != tc.wantRecvMsg {
+				t.Fatalf("RecvMsg() count = %d, want %d", got, tc.wantRecvMsg)
+			}
+			if got := filterBuilder.closeSendCount.Load(); got != 1 {
+				t.Fatalf("CloseSend() count = %d, want 1", got)
+			}
+		})
+	}
+}

test/xds/xds_client_http_connect_test.go

@@ -0,0 +1,642 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/proxyserver"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3http11proxypb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+)
+
+func enableXDSHTTPConnect(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSHTTPConnectEnabled, true)
+	cleanup, err := xdsresource.RegisterMetadataConverterForTesting(version.V3AddressURL)
+	if err != nil {
+		t.Fatalf("RegisterMetadataConverterForTesting(%q) failed: %v", version.V3AddressURL, err)
+	}
+	t.Cleanup(cleanup)
+}
+
+func splitHostPort(t *testing.T, addr string) (string, int) {
+	t.Helper()
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("Failed to split host and port for address %q: %v", addr, err)
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		t.Fatalf("Failed to parse port %q: %v", portStr, err)
+	}
+	return host, port
+}
+
+// Test veifies client-side HTTP CONNECT proxying support where the proxy
+// address is configured via endpoint metadata. It verifies that gRPC requests
+// are successfully routed through the HTTP CONNECT proxy to the correct
+// backend server.
+func (s) TestXDSHTTPConnect(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support where the proxy
+// address is configured via metadata at the locality level. It verifies that
+// the client successfully fallback to using the locality-level proxy address
+// if no endpoint-level metadata is present.
+func (s) TestXDSHTTPConnect_LocalityFallback(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				// Locality-level metadata contains the proxy address.
+				Metadata: &v3corepb.Metadata{
+					TypedFilterMetadata: map[string]*anypb.Any{
+						"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+							Address: &v3corepb.Address_SocketAddress{
+								SocketAddress: &v3corepb.SocketAddress{
+									Address:       proxyHost,
+									PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+								},
+							},
+						}),
+					},
+				},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when the client
+// uses XDS credentials. It verifies that when a plaintext Http11Proxy wrapper
+// is present in CDS, the XDS client successfully falls back to insecure
+// credentials and establishes a proxy CONNECT tunnel to the correct backend
+// server without security handshaking.
+func (s) TestXDSHTTPConnect_WithInsecureXDSCredentials(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name:       "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create client XDS credentials: %v", err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when the cluster
+// uses an Http11ProxyUpstreamTransport wrapping an inner UpstreamTlsContext.
+// It verifies that a secure TLS session is established with the backend over
+// the HTTP CONNECT proxy connection.
+func (s) TestXDSHTTPConnect_WithUpstreamTLSContext(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	// Spin up a mock HTTP CONNECT proxy server to capture client connections.
+	connectCh := make(chan string, 1)
+	pServer := proxyserver.New(t, func(req *http.Request) {
+		connectCh <- req.URL.Host
+	}, false)
+
+	proxyHost, proxyPort := splitHostPort(t, pServer.Addr)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	backendHost, backendPort := splitHostPort(t, server.Address)
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{
+					TransportSocket: &v3corepb.TransportSocket{
+						Name: "envoy.transport_sockets.tls",
+						ConfigType: &v3corepb.TransportSocket_TypedConfig{
+							TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+								CommonTlsContext: &v3tlspb.CommonTlsContext{
+									ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+										ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+											InstanceName: e2e.ClientSideCertProviderInstance,
+										},
+									},
+									TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							}),
+						},
+					},
+				}),
+			},
+		},
+	}
+
+	loadAssignment := &v3endpointpb.ClusterLoadAssignment{
+		ClusterName: endpointsName,
+		Endpoints: []*v3endpointpb.LocalityLbEndpoints{
+			{
+				Locality:            &v3corepb.Locality{SubZone: "locality-1"},
+				LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+				LbEndpoints: []*v3endpointpb.LbEndpoint{
+					{
+						HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{
+							Endpoint: &v3endpointpb.Endpoint{
+								Address: &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       backendHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(backendPort)},
+										},
+									},
+								},
+							},
+						},
+						Metadata: &v3corepb.Metadata{
+							TypedFilterMetadata: map[string]*anypb.Any{
+								"envoy.http11_proxy_transport_socket.proxy_address": testutils.MarshalAny(t, &v3corepb.Address{
+									Address: &v3corepb.Address_SocketAddress{
+										SocketAddress: &v3corepb.SocketAddress{
+											Address:       proxyHost,
+											PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: uint32(proxyPort)},
+										},
+									},
+								}),
+							},
+						},
+						LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1},
+					},
+				},
+			},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{loadAssignment},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatalf("Failed to create client XDS credentials: %v", err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	// Verify that the request was routed through the proxy to the expected
+	// backend.
+	select {
+	case gotTarget := <-connectCh:
+		if gotTarget != server.Address {
+			t.Errorf("Unexpected server address from proxy server, got %s  want %s", gotTarget, server.Address)
+		}
+	case <-time.After(defaultTestTimeout):
+		t.Fatal("Timeout while waiting for server address from proxy server")
+	}
+}
+
+// Test verifies client-side HTTP CONNECT proxying support when cluster
+// specifies an HTTP CONNECT proxy transport socket, but no proxy address
+// metadata is supplied. It verifies that the client gracefully fallback
+// to dialing the backend directly.
+func (s) TestXDSHTTPConnect_NoMetadata(t *testing.T) {
+	enableXDSHTTPConnect(t)
+
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName     = "my-service-xds-http-connect"
+		routeConfigName = "route-my-service-xds-http-connect"
+		clusterName     = "cluster-my-service-xds-http-connect"
+		endpointsName   = "endpoints-my-service-xds-http-connect"
+	)
+
+	cluster := &v3clusterpb.Cluster{
+		Name:                 clusterName,
+		ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS},
+		EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{
+			EdsConfig:   &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}},
+			ServiceName: endpointsName,
+		},
+		LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN,
+		TransportSocket: &v3corepb.TransportSocket{
+			Name: "envoy.transport_sockets.http_11_proxy",
+			ConfigType: &v3corepb.TransportSocket_TypedConfig{
+				TypedConfig: testutils.MarshalAny(t, &v3http11proxypb.Http11ProxyUpstreamTransport{})},
+		},
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_client_ignore_resource_deletion_test.go

@@ -0,0 +1,502 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const (
+	serviceName = "my-service-xds"
+	rdsName     = "route-" + serviceName
+	cdsName1    = "cluster1-" + serviceName
+	cdsName2    = "cluster2-" + serviceName
+	edsName1    = "eds1-" + serviceName
+	edsName2    = "eds2-" + serviceName
+)
+
+var (
+	// This route configuration resource contains two routes:
+	// - a route for the EmptyCall rpc, to be sent to cluster1
+	// - a route for the UnaryCall rpc, to be sent to cluster2
+	defaultRouteConfigWithTwoRoutes = &routepb.RouteConfiguration{
+		Name: rdsName,
+		VirtualHosts: []*routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*routepb.Route{
+				{
+					Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &routepb.Route_Route{Route: &routepb.RouteAction{
+						ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName1},
+					}},
+				},
+				{
+					Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &routepb.Route_Route{Route: &routepb.RouteAction{
+						ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName2},
+					}},
+				},
+			},
+		}},
+	}
+)
+
+// This test runs subtest each for a Listener resource and a Cluster resource deletion
+// in the response from the server for the following cases:
+//   - testResourceDeletionIgnored: When ignore_resource_deletion is set, the
+//     xDSClient should not delete the resource.
+//   - testResourceDeletionNotIgnored: When ignore_resource_deletion is unset,
+//     the xDSClient should delete the resource.
+//
+// Resource deletion is only applicable to Listener and Cluster resources.
+func (s) TestIgnoreResourceDeletionOnClient(t *testing.T) {
+	server1 := stubserver.StartTestService(t, nil)
+	t.Cleanup(server1.Stop)
+
+	server2 := stubserver.StartTestService(t, nil)
+	t.Cleanup(server2.Stop)
+
+	initialResourceOnServer := func(nodeID string) e2e.UpdateOptions {
+		return e2e.UpdateOptions{
+			NodeID:    nodeID,
+			Listeners: []*listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)},
+			Routes:    []*routepb.RouteConfiguration{defaultRouteConfigWithTwoRoutes},
+			Clusters: []*clusterpb.Cluster{
+				e2e.DefaultCluster(cdsName1, edsName1, e2e.SecurityLevelNone),
+				e2e.DefaultCluster(cdsName2, edsName2, e2e.SecurityLevelNone),
+			},
+			Endpoints: []*endpointpb.ClusterLoadAssignment{
+				e2e.DefaultEndpoint(edsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+				e2e.DefaultEndpoint(edsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+			},
+			SkipValidation: true,
+		}
+	}
+
+	tests := []struct {
+		name           string
+		updateResource func(r *e2e.UpdateOptions)
+	}{
+		{
+			name: "listener",
+			updateResource: func(r *e2e.UpdateOptions) {
+				r.Listeners = nil
+			},
+		},
+		{
+			name: "cluster",
+			updateResource: func(r *e2e.UpdateOptions) {
+				r.Clusters = nil
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s resource deletion ignored", test.name), func(t *testing.T) {
+			testResourceDeletionIgnored(t, initialResourceOnServer, test.updateResource)
+		})
+		t.Run(fmt.Sprintf("%s resource deletion not ignored", test.name), func(t *testing.T) {
+			testResourceDeletionNotIgnored(t, initialResourceOnServer, test.updateResource)
+		})
+	}
+}
+
+// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion"
+// set in "server_features" field. This subtest verifies that the resource was
+// not deleted by the xDSClient when a resource is missing the xDS response and
+// RPCs continue to succeed.
+func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	t.Cleanup(cancel)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources := initialResource(nodeID)
+
+	// Update the management server with initial resources setup.
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v.", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Mutate resource and update on the server.
+	updateResource(&resources)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make an RPC every 50ms for the next 500ms. This is to ensure that the
+	// updated resource is received from the management server and is processed by
+	// gRPC. Since resource deletions are ignored by the xDS client, we expect RPCs
+	// to all endpoints to keep succeeding.
+	timer := time.NewTimer(500 * time.Millisecond)
+	ticker := time.NewTicker(50 * time.Millisecond)
+	t.Cleanup(ticker.Stop)
+	for {
+		if err := verifyRPCtoAllEndpoints(cc); err != nil {
+			t.Fatal(err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-timer.C:
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion"
+// not set in "server_features" field. This subtest verifies that the resource was
+// deleted by the xDSClient when a resource is missing the xDS response and subsequent
+// RPCs fail.
+func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	t.Cleanup(cancel)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources := initialResource(nodeID)
+
+	// Update the management server with initial resources setup.
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Mutate resource and update on the server.
+	updateResource(&resources)
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Spin up go routines to verify RPCs fail after the update. The xDS node ID
+	// needs to be part of the error seen by the RPC caller.
+	client := testgrpc.NewTestServiceClient(cc)
+	wg := sync.WaitGroup{}
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+			_, err := client.EmptyCall(ctx, &testpb.Empty{})
+			if err == nil {
+				continue
+			}
+			if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) {
+				return
+			}
+		}
+	}()
+	go func() {
+		defer wg.Done()
+		for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) {
+			_, err := client.UnaryCall(ctx, &testpb.SimpleRequest{})
+			if err == nil {
+				continue
+			}
+			if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), nodeID) {
+				return
+			}
+		}
+	}()
+
+	wg.Wait()
+	if ctx.Err() != nil {
+		t.Fatal("Context expired before RPCs failed.")
+	}
+}
+
+// This helper generates a custom bootstrap config for the test.
+func generateBootstrapContents(t *testing.T, serverURI string, ignoreResourceDeletion bool, nodeID string) []byte {
+	t.Helper()
+	var serverCfgs json.RawMessage
+	if ignoreResourceDeletion {
+		serverCfgs = []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}],
+			"server_features": ["ignore_resource_deletion"]
+		}]`, serverURI))
+	} else {
+		serverCfgs = []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverURI))
+
+	}
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers:                            serverCfgs,
+		Node:                               fmt.Appendf(nil, `{"id": "%s"}`, nodeID),
+		ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return bootstrapContents
+}
+
+// This helper creates an XDS resolver Builder from the bootstrap config passed
+// as parameter.
+func xdsResolverBuilder(t *testing.T, bs []byte) resolver.Builder {
+	t.Helper()
+	xdsR, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bs)
+	if err != nil {
+		t.Fatalf("Creating xDS resolver for testing failed for config %q: %v", string(bs), err)
+	}
+	return xdsR
+}
+
+// This helper creates an xDS-enabled gRPC server using the listener and the
+// bootstrap config passed. It then registers the test service on the newly
+// created gRPC server and starts serving.
+func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContents []byte, lis net.Listener) chan connectivity.ServingMode {
+	t.Helper()
+	updateCh := make(chan connectivity.ServingMode, 1)
+
+	// Create a server option to get notified about serving mode changes.
+	modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
+		t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
+		updateCh <- args.Mode
+	})
+	stub := &stubserver.StubServer{
+		Listener: lis,
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+	}
+	server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents))
+	if err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+	stub.S = server
+	t.Cleanup(stub.S.Stop)
+
+	stubserver.StartTestService(t, stub)
+
+	return updateCh
+}
+
+// This helper creates a new TCP listener. This helper also uses this listener to
+// create a resource update with a listener resource. This helper returns the
+// resource update and the TCP listener.
+func resourceWithListenerForGRPCServer(t *testing.T, nodeID string) (e2e.UpdateOptions, net.Listener) {
+	t.Helper()
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	t.Cleanup(func() { lis.Close() })
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("Failed to retrieve host and port of listener at %q: %v", lis.Addr(), err)
+	}
+	listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{listener},
+	}
+	return resources, lis
+}
+
+// This test creates a gRPC server which provides server-side xDS functionality
+// by talking to a custom management server. This tests the scenario where bootstrap
+// config with "server_features" includes "ignore_resource_deletion". In which
+// case, when the listener resource is deleted on the management server, the gRPC
+// server should continue to serve RPCs.
+func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources, lis := resourceWithListenerForGRPCServer(t, nodeID)
+	modeChangeCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the server to update to ServingModeServing mode.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Test timed out waiting for a server to change to ServingModeServing.")
+	case mode := <-modeChangeCh:
+		if mode != connectivity.ServingModeServing {
+			t.Fatalf("Server switched to mode %v, want %v", mode, connectivity.ServingModeServing)
+		}
+	}
+
+	// Create a ClientConn and make a successful RPCs.
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update without a listener resource.
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Perform RPCs every 100 ms for 1s and verify that the serving mode does not
+	// change on gRPC server.
+	timer := time.NewTimer(500 * time.Millisecond)
+	ticker := time.NewTicker(50 * time.Millisecond)
+	t.Cleanup(ticker.Stop)
+	for {
+		if err := verifyRPCtoAllEndpoints(cc); err != nil {
+			t.Fatal(err)
+		}
+		select {
+		case <-timer.C:
+			return
+		case mode := <-modeChangeCh:
+			t.Fatalf("Server switched to mode: %v when no switch was expected", mode)
+		case <-ticker.C:
+		}
+	}
+}
+
+// This test creates a gRPC server which provides server-side xDS functionality
+// by talking to a custom management server. This tests the scenario where bootstrap
+// config with "server_features" does not include "ignore_resource_deletion". In
+// which case, when the listener resource is deleted on the management server, the
+// gRPC server should stop serving RPCs and switch mode to ServingModeNotServing.
+func (s) TestListenerResourceDeletionOnServerNotIgnored(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID)
+	xdsR := xdsResolverBuilder(t, bs)
+	resources, lis := resourceWithListenerForGRPCServer(t, nodeID)
+	updateCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the listener to move to "serving" mode.
+	select {
+	case <-ctx.Done():
+		t.Fatal("Test timed out waiting for a mode change update.")
+	case mode := <-updateCh:
+		if mode != connectivity.ServingModeServing {
+			t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing)
+		}
+	}
+
+	// Create a ClientConn and make a successful RPCs.
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+	if err := verifyRPCtoAllEndpoints(cc); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*listenerpb.Listener{}, // empty listener resource
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	select {
+	case <-ctx.Done():
+		t.Fatalf("timed out waiting for a mode change update: %v", err)
+	case mode := <-updateCh:
+		if mode != connectivity.ServingModeNotServing {
+			t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing)
+		}
+	}
+}
+
+// This helper makes both UnaryCall and EmptyCall RPCs using the ClientConn that
+// is passed to this function. This helper panics for any failed RPCs.
+func verifyRPCtoAllEndpoints(cc grpc.ClientConnInterface) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("rpc EmptyCall() failed: %v", err)
+	}
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+		return fmt.Errorf("rpc UnaryCall() failed: %v", err)
+	}
+	return nil
+}

test/xds/xds_client_integration_test.go

@@ -0,0 +1,244 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/peer"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+func (s) TestClientSideXDS(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestClient_ConcurrentRPC ensures thread safety for xDS clients executing
+// concurrent RPCs, particularly verifying that the regex matchers do not cause
+// data races.
+func (s) TestClient_ConcurrentRPC(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{{
+			Name: routeConfigName,
+			VirtualHosts: []*v3routepb.VirtualHost{{
+				Domains: []string{serviceName},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_SafeRegex{
+							SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/.*"},
+						},
+					},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+					}},
+				}},
+			}},
+		}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, clusterName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	const numRPCs = 20
+	wg := sync.WaitGroup{}
+	for i := 0; i < numRPCs; i++ {
+		wg.Go(func() {
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Errorf("EmptyCall() failed: %v", err)
+			}
+		})
+	}
+	wg.Wait()
+}
+
+// Test verifies that client-side xDS routing strictly evaluates
+// OutgoingContext metadata and ignores IncomingContext metadata. This
+// prevents incoming headers on a server handler context from leaking
+// into and contaminating outgoing client routing decisions.
+func (s) TestClientSideXDS_RouteMatching_IgnoreIncomingMetadata(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName     = "my-service-client"
+		routeConfigName = "route-" + serviceName
+		clusterA        = "cluster-A-" + serviceName
+		clusterB        = "cluster-B-" + serviceName
+		headerName      = "x-route-target"
+		headerValue     = "cluster-a"
+	)
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{{
+			Name: routeConfigName,
+			VirtualHosts: []*v3routepb.VirtualHost{{
+				Domains: []string{serviceName},
+				Routes: []*v3routepb.Route{
+					{
+						Match: &v3routepb.RouteMatch{
+							PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+							Headers: []*v3routepb.HeaderMatcher{{
+								Name: headerName,
+								HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{
+									ExactMatch: headerValue,
+								},
+							}},
+						},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA},
+						}},
+					},
+					{
+						Match: &v3routepb.RouteMatch{
+							PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+						},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB},
+						}},
+					},
+				},
+			}},
+		}},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone),
+			e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
+			e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)}),
+			e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}),
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Simulate an intermediary server that receives an incoming RPC containing
+	// the header "x-route-target: cluster-a" in its IncomingContext, and then
+	// initiates an outgoing client RPC using that same context without setting
+	// any OutgoingContext metadata. Client-side routing  must strictly evaluate
+	// OutgoingContext (which is empty) and select Route 2.
+	incomingCtx := metadata.NewIncomingContext(ctx, metadata.Pairs(headerName, headerValue))
+	var p peer.Peer
+	if _, err := client.EmptyCall(incomingCtx, &testpb.Empty{}, grpc.Peer(&p)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if p.Addr.String() != serverB.Address {
+		t.Fatalf("Unexpected client routing to peer %s, want fallback Route 2 (clusterB at %s)", p.Addr.String(), serverB.Address)
+	}
+}

test/xds/xds_client_outlier_detection_test.go

@@ -0,0 +1,323 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/protobuf/types/known/durationpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// TestOutlierDetection_NoopConfig tests the scenario where the Outlier
+// Detection feature is enabled on the gRPC client, but it receives no Outlier
+// Detection configuration from the management server. This should result in a
+// no-op Outlier Detection configuration being used to configure the Outlier
+// Detection balancer. This test verifies that an RPC is able to proceed
+// normally with this configuration.
+func (s) TestOutlierDetection_NoopConfig(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
+	}
+	server.StartServer()
+	t.Logf("Started test service backend at %q", server.Address)
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// clientResourcesMultipleBackendsAndOD returns xDS resources which correspond
+// to multiple upstreams, corresponding different backends listening on
+// different localhost:port combinations. The resources also configure an
+// Outlier Detection Balancer configured through the passed in Outlier Detection
+// proto.
+func clientResourcesMultipleBackendsAndOD(params e2e.ResourceParams, ports []uint32, od *v3clusterpb.OutlierDetection) e2e.UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
+	return e2e.UpdateOptions{
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{clusterWithOutlierDetection(clusterName, endpointsName, params.SecLevel, od)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, ports)},
+	}
+}
+
+func clusterWithOutlierDetection(clusterName, edsServiceName string, secLevel e2e.SecurityLevel, od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster {
+	cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel)
+	cluster.OutlierDetection = od
+	return cluster
+}
+
+// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn,
+// connected to a server exposing the test.grpc_testing.TestService, are
+// roundrobined across the given backend addresses.
+//
+// Returns a non-nil error if context deadline expires before RPCs start to get
+// roundrobined across the given backends.
+func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error {
+	wantAddrCount := make(map[string]int)
+	for _, addr := range addrs {
+		wantAddrCount[addr.Addr]++
+	}
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		// Perform 3 iterations.
+		var iterations [][]string
+		for i := 0; i < 3; i++ {
+			iteration := make([]string, len(addrs))
+			for c := 0; c < len(addrs); c++ {
+				var peer peer.Peer
+				client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer))
+				if peer.Addr != nil {
+					iteration[c] = peer.Addr.String()
+				}
+			}
+			iterations = append(iterations, iteration)
+		}
+		// Ensure the first iteration contains all addresses in addrs.
+		gotAddrCount := make(map[string]int)
+		for _, addr := range iterations[0] {
+			gotAddrCount[addr]++
+		}
+		if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" {
+			continue
+		}
+		// Ensure all three iterations contain the same addresses.
+		if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) {
+			continue
+		}
+		return nil
+	}
+	return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs)
+}
+
+// TestOutlierDetectionWithOutlier tests the Outlier Detection Balancer e2e. It
+// spins up three backends, one which consistently errors, and configures the
+// ClientConn using xDS to connect to all three of those backends. The Outlier
+// Detection Balancer should eject the connection to the backend which
+// constantly errors, causing RPC's to not be routed to that upstream, and only
+// be Round Robined across the two healthy upstreams. Other than the intervals
+// the unhealthy upstream is ejected, RPC's should regularly round robin across
+// all three upstreams.
+func (s) TestOutlierDetectionWithOutlier(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Working backend 1.
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+
+	// Working backend 2.
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+
+	// Backend 3 that will always return an error and eventually ejected.
+	backend3 := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") },
+	})
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		SecLevel:   e2e.SecurityLevelNone,
+	}, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{
+		Interval:                       &durationpb.Duration{Nanos: 50000000}, // .5 seconds
+		BaseEjectionTime:               &durationpb.Duration{Seconds: 30},
+		MaxEjectionTime:                &durationpb.Duration{Seconds: 300},
+		MaxEjectionPercent:             &wrapperspb.UInt32Value{Value: 1},
+		FailurePercentageThreshold:     &wrapperspb.UInt32Value{Value: 50},
+		EnforcingFailurePercentage:     &wrapperspb.UInt32Value{Value: 100},
+		FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 8},
+		FailurePercentageMinimumHosts:  &wrapperspb.UInt32Value{Value: 3},
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	fullAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+		{Addr: backend3.Address},
+	}
+	// At first, due to no statistics on each of the backends, the 3
+	// upstreams should all be round robined across.
+	if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+	}
+	// After calling the three upstreams, one of them constantly error
+	// and should eventually be ejected for a period of time. This
+	// period of time should cause the RPC's to be round robined only
+	// across the two that are healthy.
+	if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}
+
+// TestOutlierDetectionXDSDefaultOn tests that Outlier Detection is by default
+// configured on in the xDS Flow. If the Outlier Detection proto message is
+// present with SuccessRateEjection unset, then Outlier Detection should be
+// turned on. The test setups and xDS system with xDS resources with Outlier
+// Detection present in the CDS update, but with SuccessRateEjection unset, and
+// asserts that Outlier Detection is turned on and ejects upstreams.
+func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Working backend 1.
+	backend1 := stubserver.StartTestService(t, nil)
+	port1 := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+
+	// Working backend 2.
+	backend2 := stubserver.StartTestService(t, nil)
+	port2 := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+
+	// Backend 3 that will always return an error and eventually ejected.
+	backend3 := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") },
+	})
+	port3 := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+
+	// Configure CDS resources with Outlier Detection set but
+	// EnforcingSuccessRate unset. This should cause Outlier Detection to be
+	// configured with SuccessRateEjection present in configuration, which will
+	// eventually be populated with its default values along with the knobs set
+	// as SuccessRate fields in the proto, and thus Outlier Detection should be
+	// on and actively eject upstreams.
+	const serviceName = "my-service-client-side-xds"
+	resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		SecLevel:   e2e.SecurityLevelNone,
+	}, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{
+		// Need to set knobs to trigger ejection within the test time frame.
+		Interval: &durationpb.Duration{Nanos: 50000000},
+		// EnforcingSuccessRateSet to nil, causes success rate algorithm to be
+		// turned on.
+		SuccessRateMinimumHosts:  &wrapperspb.UInt32Value{Value: 1},
+		SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 8},
+		SuccessRateStdevFactor:   &wrapperspb.UInt32Value{Value: 1},
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	fullAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+		{Addr: backend3.Address},
+	}
+	// At first, due to no statistics on each of the backends, the 3
+	// upstreams should all be round robined across.
+	if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+
+	// The addresses which don't return errors.
+	okAddresses := []resolver.Address{
+		{Addr: backend1.Address},
+		{Addr: backend2.Address},
+	}
+	// After calling the three upstreams, one of them constantly error
+	// and should eventually be ejected for a period of time. This
+	// period of time should cause the RPC's to be round robined only
+	// across the two that are healthy.
+	if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil {
+		t.Fatalf("error in expected round robin: %v", err)
+	}
+}

test/xds/xds_client_priority_locality_test.go

@@ -0,0 +1,211 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	rrutil "google.golang.org/grpc/internal/testutils/roundrobin"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// backendAddressesAndPorts extracts the address and port of each of the
+// StubServers passed in and returns them. Fails the test if any of the
+// StubServers passed have an invalid address.
+func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) {
+	addrs := make([]resolver.Address, len(servers))
+	ports := make([]uint32, len(servers))
+	for i := 0; i < len(servers); i++ {
+		addrs[i] = resolver.Address{Addr: servers[i].Address}
+		ports[i] = testutils.ParsePort(t, servers[i].Address)
+	}
+	return addrs, ports
+}
+
+// Tests scenarios involving localities moving between priorities.
+//   - The test starts off with a cluster that contains two priorities, one
+//     locality in each, and one endpoint in each. Verifies that traffic reaches
+//     the endpoint in the higher priority.
+//   - The test then moves the locality in the lower priority over to the higher
+//     priority. At that point, we would have a cluster with a single priority,
+//     but two localities, and one endpoint in each. Verifies that traffic is
+//     split between the endpoints.
+//   - The test then deletes the locality that was originally in the higher
+//     priority.Verifies that all traffic is now reaching the only remaining
+//     endpoint.
+func (s) TestClientSideXDS_LocalityChangesPriority(t *testing.T) {
+	// Spin up a management server and two test service backends.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+	backend0 := stubserver.StartTestService(t, nil)
+	defer backend0.Stop()
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	addrs, ports := backendAddressesAndPorts(t, []*stubserver.StubServer{backend0, backend1})
+
+	// Configure resources on the management server. We use default client side
+	// resources for listener, route configuration and cluster. For the
+	// endpoints resource though, we create one with two priorities, and one
+	// locality each, and one endpoint each.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointsName = "endpoints-" + serviceName
+	locality1 := e2e.LocalityID{Region: "my-region-1", Zone: "my-zone-1", SubZone: "my-subzone-1"}
+	locality2 := e2e.LocalityID{Region: "my-region-2", Zone: "my-zone-2", SubZone: "my-subzone-2"}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointsName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "my-locality-1",
+					Weight:   1000000,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{Ports: []uint32{ports[0]}}},
+					Locality: locality1,
+				},
+				{
+					Name:     "my-locality-2",
+					Weight:   1000000,
+					Priority: 1,
+					Backends: []e2e.BackendOptions{{Ports: []uint32{ports[1]}}},
+					Locality: locality2,
+				},
+			},
+		})},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	// // Ensure that RPCs get routed to the backend in the higher priority.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with two
+	// localities, and one endpoint each. The locality weights are equal at this
+	// point, and we expect RPCs to be round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the locality weights ever so slightly. We still expect RPCs to be
+	// round-robined across the two localities.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-1",
+				Weight:   499884,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend0.Address)}}},
+				Locality: locality1,
+			},
+			{
+				Name:     "my-locality-2",
+				Weight:   500115,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the endpoints resource to contain a single priority with one
+	// locality. The locality which was originally in the higher priority is now
+	// dropped.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+		ClusterName: endpointsName,
+		Host:        "localhost",
+		Localities: []e2e.LocalityOptions{
+			{
+				Name:     "my-locality-2",
+				Weight:   1000000,
+				Priority: 0,
+				Backends: []e2e.BackendOptions{{Ports: []uint32{testutils.ParsePort(t, backend1.Address)}}},
+				Locality: locality2,
+			},
+		},
+	})}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil {
+		t.Fatal(err)
+	}
+}

test/xds/xds_client_retry_test.go

@@ -0,0 +1,192 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestClientSideRetry(t *testing.T) {
+	// serverMu guards access to ctr and errs.
+	serverMu := sync.Mutex{}
+	ctr := 0
+	errs := []codes.Code{codes.ResourceExhausted}
+
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			serverMu.Lock()
+			defer func() {
+				ctr++
+				serverMu.Unlock()
+			}()
+			if ctr < len(errs) {
+				return nil, status.Errorf(errs[ctr], "this should be retried")
+			}
+			return &testpb.Empty{}, nil
+		},
+	})
+	defer server.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	defer cancel()
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.ResourceExhausted {
+		t.Fatalf("rpc EmptyCall() = _, %v; want _, ResourceExhausted", err)
+	}
+
+	testCases := []struct {
+		name        string
+		vhPolicy    *v3routepb.RetryPolicy
+		routePolicy *v3routepb.RetryPolicy
+		errs        []codes.Code // the errors returned by the server for each RPC
+		tryAgainErr codes.Code   // the error that would be returned if we are still using the old retry policies.
+		errWant     codes.Code
+	}{{
+		name: "virtualHost only, fail",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted,unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 1},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		routePolicy: nil,
+		tryAgainErr: codes.ResourceExhausted,
+		errWant:     codes.Unavailable,
+	}, {
+		name: "virtualHost only",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted, unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		routePolicy: nil,
+		tryAgainErr: codes.Unavailable,
+		errWant:     codes.OK,
+	}, {
+		name: "virtualHost+route, fail",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted,unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.ResourceExhausted, codes.Unavailable},
+		tryAgainErr: codes.OK,
+		errWant:     codes.Unavailable,
+	}, {
+		name: "virtualHost+route",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "resource-exhausted",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		errs:        []codes.Code{codes.Unavailable},
+		tryAgainErr: codes.Unavailable,
+		errWant:     codes.OK,
+	}, {
+		name: "virtualHost+route, not enough attempts",
+		vhPolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 2},
+		},
+		routePolicy: &v3routepb.RetryPolicy{
+			RetryOn:    "unavailable",
+			NumRetries: &wrapperspb.UInt32Value{Value: 1},
+		},
+		errs:        []codes.Code{codes.Unavailable, codes.Unavailable},
+		tryAgainErr: codes.OK,
+		errWant:     codes.Unavailable,
+	}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			serverMu.Lock()
+			errs = tc.errs
+
+			// Confirm tryAgainErr is correct before updating resources.
+			ctr = 0
+			serverMu.Unlock()
+			_, err := client.EmptyCall(ctx, &testpb.Empty{})
+			if code := status.Code(err); code != tc.tryAgainErr {
+				t.Fatalf("with old retry policy: EmptyCall() = _, %v; want _, %v", err, tc.tryAgainErr)
+			}
+
+			resources.Routes[0].VirtualHosts[0].RetryPolicy = tc.vhPolicy
+			resources.Routes[0].VirtualHosts[0].Routes[0].GetRoute().RetryPolicy = tc.routePolicy
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			for {
+				serverMu.Lock()
+				ctr = 0
+				serverMu.Unlock()
+				_, err := client.EmptyCall(ctx, &testpb.Empty{})
+				if code := status.Code(err); code == tc.tryAgainErr {
+					continue
+				} else if code != tc.errWant {
+					t.Fatalf("rpc EmptyCall() = _, %v; want _, %v", err, tc.errWant)
+				}
+				break
+			}
+		})
+	}
+}

test/xds/xds_client_sni_test.go

@@ -0,0 +1,851 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// validSNI matches the wildcard SAN (*.test.example.com) in the test server
+// certificate.
+const validSNI = "x.test.example.com"
+
+// Tests the SNI and SAN validation logic by verifying that RPCs succeed when
+// AutoSNISANValidation is enabled and the SNI matches a server certificate DNS
+// SAN. Also verifies that RPCs fail with an 'Unavailable' status if the SNI is
+// present but does not match any DNS SAN in the certificate.
+func (s) TestClientSideXDS_SNISANValidation(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends for two clusters:
+	// - backend1 configured with TLS creds, represents cluster1 (valid SNI)
+	// - backend2 configured with TLS creds, represents cluster2 (invalid SNI)
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	// Route configuration:
+	// - "/grpc.testing.TestService/EmptyCall" --> cluster1 (valid SNI)
+	// - "/grpc.testing.TestService/UnaryCall" --> cluster2 (invalid SNI)
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// Configure cluster1 with valid SNI and AutoSniSanValidation set to true.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  validSNI,
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: e2e.ClientSideCertProviderInstance},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with invalid SNI and AutoSniSanValidation set to true.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  "wrong.sni.domain",
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC to cluster1 should succeed because auto_sni_san_validation is true
+	// and sni matches server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because even though auto_sni_san_validation
+	// is true, sni doesn't match server cert SAN.
+	const wantErr = "do not match the SNI"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when AutoHostSNI is enabled, the endpoint's hostname is used for
+// the ServerName even though SNI is specified. It verifies that the TLS
+// handshake and subsequent RPC succeed because the resolved SNI i.e. the
+// hostname matches the server's certificate SAN.
+func (s) TestClientSideXDS_AutoHostSNI(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backend
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointName = "endpoint-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	// Configure the cluster with AutoHostSni and AutoSniSanValidation set to
+	// true with a invalid SNI.
+	cluster := e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS)
+	cluster.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoHostSni:          true,
+				AutoSniSanValidation: true,
+				Sni:                  "wrong.sni.domain",
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Endpoints configuring Hostname to the defaultTestCertSAN to verify
+	// AutoHostSni usage.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{{
+				Weight: 1,
+				Backends: []e2e.BackendOptions{{
+					Ports:    []uint32{testutils.ParsePort(t, server.Address)},
+					Hostname: validSNI,
+				}},
+			}},
+		}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC should succeed because auto_host_sni sets SNI from the endpoint
+	// hostname and auto_sni_san_validation validates that the SNI matches
+	// server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+}
+
+// Tests the scenario where multiple endpoints are configured with distinct
+// hostnames. It verifies that the client uses the specific hostname associated
+// with each endpoint during the TLS handshake by simulating an EDS failover and
+// confirming that the SNI is dynamically updated to match the active endpoint.
+func (s) TestClientSideXDS_AutoHostSNI_MultipleEndpoints(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+	const endpointName = "endpoint-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	clusters := []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointName, e2e.SecurityLevelMTLS)}
+	clusters[0].TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoHostSni:          true,
+				AutoSniSanValidation: true,
+				Sni:                  "wrong.sni.domain",
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+						ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+							InstanceName:    e2e.ClientSideCertProviderInstance,
+							CertificateName: "root",
+						},
+					},
+					TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure only Priority 0 with an invalid hostname. RPC should fail due
+	// to SAN mismatch.
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "locality-0",
+					Weight:   1,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server1.Address)},
+						Hostname: "wrong.sni.domain",
+					}},
+				},
+			},
+		}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC should fail because hostname for endpoints in Priority 0 is invalid.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err == nil {
+		t.Fatalf("EmptyCall() unexpectedly succeeded with invalid SNI and no failover")
+	} else if !strings.Contains(err.Error(), "authentication handshake failed") {
+		t.Fatalf("EmptyCall() failed with %v, want error containing 'authentication handshake failed'", err)
+	}
+
+	// Update the same EDS resource to add Priority 1 with an endpoint with
+	// valid hostname.
+	endpoints = []*v3endpointpb.ClusterLoadAssignment{
+		e2e.EndpointResourceWithOptions(e2e.EndpointOptions{
+			ClusterName: endpointName,
+			Host:        "localhost",
+			Localities: []e2e.LocalityOptions{
+				{
+					Name:     "locality-0",
+					Weight:   1,
+					Priority: 0,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server1.Address)},
+						Hostname: "wrong.sni.domain",
+					}},
+				},
+				{
+					Name:     "locality-1",
+					Weight:   1,
+					Priority: 1,
+					Backends: []e2e.BackendOptions{{
+						Ports:    []uint32{testutils.ParsePort(t, server2.Address)},
+						Hostname: validSNI,
+					}},
+				},
+			},
+		}),
+	}
+	resources.Endpoints = endpoints
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// RPC should succeed because it automatically fails over to Priority 1. Id
+	// only one hostname would have been used per leaf cluster , this RPC should
+	// have failed due to invalid SNI in Priority 0.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo), grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed after adding Priority 1: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server2.Address; got != want {
+		t.Fatalf("EmptyCall() successes routed to %q, want %q", got, want)
+	}
+}
+
+// TestClientSideXDS_FallbackSANMatchers tests that when AutoSniSanValidation is
+// true, and no SNI is provided by the control plane and AutoHostSNI is not set,
+// the validation falls back to using the explicit SAN matchers specified in the
+// configuration. It verifies that RPCs succeed when the fallback matchers match
+// the server certificate SAN and fail when they do not.
+func (s) TestClientSideXDS_FallbackSANMatchers(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// Configure cluster1 with AutoSniSanValidation set to true and no SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which matches the server1's
+	// certificate SAN.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}},
+								},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with AutoSniSanValidation set to true and no SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which does not match the
+	// server2's certificate SAN.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC to cluster1 should succeed because fallback SAN mathchers are used
+	// and they match server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because fallback SAN matchers are used
+	// but they don't match server cert SAN.
+	const wantErr = "do not match any of the accepted SANs"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when the XDSSNIEnabled environment variable is set to false, SNI
+// is not used for validation even if AutoSniSanValidation is true and SNI is
+// set. It verifies that the system falls back to using explicit SAN matchers if
+// provided, and the TLS handshake succeeds when they match the server
+// certificate SAN.
+func (s) TestClientSideXDS_SNIEnvVarDisabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, false)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backends.
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server1 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server1.Stop()
+	server2 := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server2.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName1 = "cluster1-" + serviceName
+	const clusterName2 = "cluster2-" + serviceName
+	const endpointName1 = "endpoint1-" + serviceName
+	const endpointName2 = "endpoint2-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+
+	routes := []*v3routepb.RouteConfiguration{{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{serviceName},
+			Routes: []*v3routepb.Route{
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1}}},
+				},
+				{
+					Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
+					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2}}},
+				},
+			},
+		}},
+	}}
+
+	// cluster1 configuration with AutoSniSanValidation set to true and wrong SNI
+	// provided for the handshake. The validation falls back to using the explicit
+	// SAN matchers specified in the configuration which matches the server1's
+	// certificate SAN.
+	cluster1 := e2e.DefaultCluster(clusterName1, endpointName1, e2e.SecurityLevelMTLS)
+	cluster1.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  "incorrect.sni",
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{
+									{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "*.test.example.com"}},
+								},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	// Configure cluster2 with AutoSniSanValidation set to true and correct SNI
+	// provided for the handshake. The validation falls back to using the
+	// explicit SAN matchers specified in the configuration which does not match
+	// the server2's certificate SAN.
+	cluster2 := e2e.DefaultCluster(clusterName2, endpointName2, e2e.SecurityLevelMTLS)
+	cluster2.TransportSocket = &v3corepb.TransportSocket{
+		Name: "envoy.transport_sockets.tls",
+		ConfigType: &v3corepb.TransportSocket_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+				Sni:                  validSNI,
+				AutoSniSanValidation: true,
+				CommonTlsContext: &v3tlspb.CommonTlsContext{
+					ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{
+						CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{
+							DefaultValidationContext: &v3tlspb.CertificateValidationContext{
+								MatchSubjectAltNames: []*v3matcherpb.StringMatcher{{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "wrong.san.domain"}}},
+								CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+									InstanceName:    e2e.ClientSideCertProviderInstance,
+									CertificateName: "root",
+								},
+							},
+						},
+					},
+					TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+						InstanceName:    e2e.ClientSideCertProviderInstance,
+						CertificateName: "identity",
+					},
+				},
+			}),
+		},
+	}
+
+	clusters := []*v3clusterpb.Cluster{cluster1, cluster2}
+	endpoints := []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(endpointName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
+		e2e.DefaultEndpoint(endpointName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// RPC to cluster1 should succeed because fallback SAN mathchers are used
+	// and they match server cert SAN.
+	peerInfo := &peer.Peer{}
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peerInfo)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peerInfo.Addr.String(), server1.Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
+	}
+
+	// RPC to cluster2 should fail because fallback SAN matchers are used but they
+	// don't match server cert SAN.
+	const wantErr = "do not match any of the accepted SANs"
+	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("UnaryCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
+	}
+}
+
+// Tests that when AutoHostSNI is enabled for a Logical DNS cluster, the SNI is
+// resolved from the DNSHostName in the cluster configuration. It verifies that
+// the TLS handshake succeeds when the DNSHostName matches the server's
+// certificate SAN.
+func (s) TestClientSideXDS_AutoHostSNI_LogicalDNS(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSSNIEnabled, true)
+
+	// Spin up an xDS management server.
+	mgmtServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Create test backend
+	serverCreds := testutils.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
+	server := stubserver.StartTestService(t, nil, grpc.Creds(serverCreds))
+	defer server.Stop()
+
+	// Replace DNS resolver with a manual resolver.
+	dnsR := manual.NewBuilderWithScheme("dns")
+	originalDNS := resolver.Get("dns")
+	resolver.Register(dnsR)
+	t.Cleanup(func() { resolver.Register(originalDNS) })
+
+	dnsR.UpdateState(resolver.State{Endpoints: []resolver.Endpoint{{Addresses: []resolver.Address{{Addr: server.Address}}}}})
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	const routeConfigName = "route-" + serviceName
+	const clusterName = "cluster-" + serviceName
+
+	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
+	routes := []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}
+
+	// Cluster of Type LogicalDNS. with DNSHostName set to match the server's cert
+	// SAN.
+	cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		Type:          e2e.ClusterTypeLogicalDNS,
+		ClusterName:   clusterName,
+		DNSHostName:   validSNI,
+		DNSPort:       uint32(testutils.ParsePort(t, server.Address)),
+		SecurityLevel: e2e.SecurityLevelMTLS,
+	})
+	cluster.TransportSocket.ConfigType.(*v3corepb.TransportSocket_TypedConfig).TypedConfig = testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+		AutoHostSni:          true,
+		AutoSniSanValidation: true,
+		CommonTlsContext: &v3tlspb.CommonTlsContext{
+			ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
+				ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+					InstanceName: e2e.ClientSideCertProviderInstance,
+				},
+			},
+			TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
+				InstanceName: e2e.ClientSideCertProviderInstance,
+			},
+		},
+	})
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  []*v3clusterpb.Cluster{cluster},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	clientCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+	if err != nil {
+		t.Fatal(err)
+	}
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// RPC should succeed because DNSHostName matches the server's certificate
+	// SAN.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+}

test/xds/xds_resolver_e2e_test.go

@@ -0,0 +1,754 @@
+/*
+*
+* Copyright 2026 gRPC authors.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/serviceconfig"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+
+	iresolver "google.golang.org/grpc/internal/resolver"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// wrappingBuilder wraps a resolver.Builder and intercepts the ClientConn
+// passed to Build by wrapping it with an interceptingClientConn.
+type wrappingBuilder struct {
+	resolver.Builder
+	jsonCh chan string
+}
+
+// Build wraps the provided resolver.ClientConn with an interceptingClientConn
+// before delegating to the underlying resolver.Builder.
+func (ib *wrappingBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) {
+	icc := &interceptingClientConn{
+		ClientConn: cc,
+		jsonCh:     ib.jsonCh,
+	}
+	return ib.Builder.Build(target, icc, opts)
+}
+
+// interceptingClientConn wraps a resolver.ClientConn and intercepts calls to
+// ParseServiceConfig to send the raw JSON service config.
+type interceptingClientConn struct {
+	resolver.ClientConn
+	jsonCh chan string
+}
+
+func (icc *interceptingClientConn) ParseServiceConfig(js string) *serviceconfig.ParseResult {
+	select {
+	case icc.jsonCh <- js:
+	default:
+	}
+	return icc.ClientConn.ParseServiceConfig(js)
+}
+
+type dummyFilterCfg struct {
+	httpfilter.FilterConfig
+}
+
+type testFilterBuilder struct {
+	httpfilter.Builder
+	typeURL      string
+	blockChan    chan struct{}
+	enteredChan  chan struct{}
+	closeChan    chan struct{}
+	newStreamErr error
+}
+
+func (tb *testFilterBuilder) TypeURLs() []string { return []string{tb.typeURL} }
+
+func (*testFilterBuilder) ParseFilterConfig(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return dummyFilterCfg{}, nil
+}
+
+func (*testFilterBuilder) ParseFilterConfigOverride(proto.Message, httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return dummyFilterCfg{}, nil
+}
+
+func (*testFilterBuilder) IsTerminal() bool { return false }
+
+func (tb *testFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter {
+	return tb
+}
+
+func (*testFilterBuilder) Close() {}
+
+// ClientFilterBuilder is an optional interface for filters to implement. This
+// compile time check ensures the test filter implements it.
+var _ httpfilter.ClientFilterBuilder = &testFilterBuilder{}
+
+func (tb *testFilterBuilder) BuildClientInterceptor(httpfilter.FilterConfig, httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) {
+	return &testInterceptor{
+		blockChan:    tb.blockChan,
+		enteredChan:  tb.enteredChan,
+		closeChan:    tb.closeChan,
+		newStreamErr: tb.newStreamErr,
+	}, nil
+}
+
+type testInterceptor struct {
+	blockChan    chan struct{}
+	enteredChan  chan struct{}
+	closeChan    chan struct{}
+	newStreamErr error
+}
+
+func (i *testInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	// Signal that we have entered the filter
+	i.enteredChan <- struct{}{}
+
+	select {
+	case <-i.blockChan:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+
+	if i.newStreamErr != nil {
+		return nil, i.newStreamErr
+	}
+	return newStream(ctx, opts...)
+}
+
+func (i *testInterceptor) Close() {
+	i.closeChan <- struct{}{}
+}
+
+func newTestFilterBuilder(t testing.TB, typeURL string) *testFilterBuilder {
+	tb := &testFilterBuilder{
+		typeURL:     typeURL,
+		blockChan:   make(chan struct{}),
+		enteredChan: make(chan struct{}, 1),
+		closeChan:   make(chan struct{}, 3),
+	}
+	httpfilter.Register(tb)
+	t.Cleanup(func() {
+		httpfilter.UnregisterForTesting(typeURL)
+	})
+	return tb
+}
+
+// wantServiceConfig returns a JSON representation of a service config with
+// xds_cluster_manager_experimental LB policy with child policies of
+// cds_experimental for the provided cluster names.
+func wantServiceConfig(clusters ...string) string {
+	var children []string
+	for _, cluster := range clusters {
+		children = append(children, fmt.Sprintf(`"cluster:%s": {
+			"childPolicy": [{
+				"cds_experimental": {
+					"cluster": "%s"
+				}
+			}]
+		}`, cluster, cluster))
+	}
+	return fmt.Sprintf(`{
+		"loadBalancingConfig": [{
+			"xds_cluster_manager_experimental": {
+				"children": {
+					%s
+				}
+			}
+		}]
+	}`, strings.Join(children, ","))
+}
+
+// compareJSONConfigs parses gotJSON and wantJSON using the internal gRPC
+// Service Config parser and asserts that they represent the same configuration.
+func compareJSONConfigs(t *testing.T, gotJSON, wantJSON string) {
+	gotParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(gotJSON)
+	if gotParsed.Err != nil {
+		t.Fatalf("Failed to parse got service config %q: %v", gotJSON, gotParsed.Err)
+	}
+	wantParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(wantJSON)
+	if wantParsed.Err != nil {
+		t.Fatalf("Failed to parse want service config %q: %v", wantJSON, wantParsed.Err)
+	}
+	if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) {
+		t.Fatalf("Service config mismatch.\nGot:\n%s\nWant:\n%s", gotJSON, wantJSON)
+	}
+}
+
+func verifyServiceConfig(ctx context.Context, t *testing.T, jsonCh chan string, wantClusters ...string) {
+	t.Helper()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for resolver state update")
+	case js := <-jsonCh:
+		compareJSONConfigs(t, js, wantServiceConfig(wantClusters...))
+	}
+}
+
+// Test verifies that when RPCs are in flight holding references to an old
+// cluster, that cluster remains in the service config until all in-flight
+// RPCs finish.
+func (s) TestResolverDelayedClusterRemoval_MultipleInFlightRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	blockChan, enteredChan := tb.blockChan, tb.enteredChan
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create the xDS resolver builder.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Create an intercepting resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "delaying-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a gRPC client using the intercepting resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Trigger first RPC to cluster A. This RPC should reach our custom HTTP
+	// filter, and get blocked.
+	errCh := make(chan error, 2)
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	// Verify that first RPC has entered the HTTP filter's NewStream and is
+	// currently blocked.
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for first RPC to reach HTTP filter")
+	}
+
+	// Read the first state update from the intercepting resolver.
+	// This should contain cluster-A.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Trigger second RPC to cluster A. This RPC should reach our custom HTTP
+	// filter, and get blocked.
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	// Verify that second RPC has entered the HTTP filter's NewStream and is
+	// currently blocked.
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for second RPC to reach HTTP filter")
+	}
+
+	// Now update the route configuration on the management server to point to cluster B.
+	clusterSpec.Cluster = clusterB
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Unblock one of the RPCs and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for an RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Verify that because the other RPC to cluster-A is still in flight, the
+	// cluster reference count does not drop to 0 and no service config update
+	// is produced.
+	select {
+	case js := <-jsonCh:
+		t.Fatalf("Unexpected service config update received: %s", js)
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the remaining RPC and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for remaining RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("Remaining RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Once the second RPC finishes (refCount drops 1 -> 0), cluster-A should be
+	// removed from the service config.
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// Test verifies that if stream creation fails early inside an HTTP filter or
+// interceptor (before the stream is created), the OnFinish CallOption is still
+// executed. This decrements the cluster reference count to 0, allowing the
+// resolver to unsubscribe and prune the old cluster from the service config.
+func (s) TestResolverPrunesCluster_StreamCreationFailure(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	const wantErr = "blocking filter error"
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	tb.newStreamErr = status.Error(codes.Unavailable, wantErr)
+	close(tb.blockChan) // Close blockChan immediately so that the RPC fails without blocking
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create the underlying xDS resolver builder.
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	serverA := stubserver.StartTestService(t, nil)
+	defer serverA.Stop()
+	serverB := stubserver.StartTestService(t, nil)
+	defer serverB.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	clusterSpec := &v3routepb.RouteAction_Cluster{Cluster: clusterA}
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+							ClusterSpecifier: clusterSpec,
+						}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "blocking-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, serverA.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a gRPC client using the xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Read the first state update from the intercepting resolver.
+	// This should contain cluster-A.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Make an RPC. Since the filter immediately fails, this should fail with
+	// the expected error.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err = client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("RPC failed with error %v, want code %v with desc containing %q", err, codes.Unavailable, wantErr)
+	}
+
+	// Update the route configuration on the management server to point
+	// to cluster B.
+	clusterSpec.Cluster = clusterB
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, serverB.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Read the second state update from the intercepting resolver. This should
+	// contain both cluster-A and cluster-B.
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Read the third state update from the intercepting resolver. Cluster-A's
+	// reference count drops to 0, causing the resolver to unsubscribe and
+	// prune it.
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// TestResolverDelayedInterceptorClose_ActiveRPCs verifies that when a route
+// configuration is updated (causing the old config selector to stop), an HTTP
+// filter interceptor for a route/cluster with an active in-flight RPC is not
+// closed immediately. It remains active until all active RPCs to that cluster
+// in that route complete and its refcount drops to 0.
+func (s) TestResolverDelayedInterceptorClose_ActiveRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	blockChan, enteredChan, closeChan := tb.blockChan, tb.enteredChan, tb.closeChan
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "delaying-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	errCh := make(chan error, 1)
+	go func() {
+		_, err := client.EmptyCall(ctx, &testpb.Empty{})
+		errCh <- err
+	}()
+
+	select {
+	case <-enteredChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for RPC to reach HTTP filter")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Update route configuration on the management server to point to cluster B.
+	hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}}
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Verify that because an RPC is still in flight on cluster-A, the interceptor
+	// is NOT closed prematurely when the old config selector is stopped.
+	select {
+	case <-closeChan:
+		t.Fatal("Unexpected interceptor Close() called while RPC is still active")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the active RPC and verify it completes successfully.
+	blockChan <- struct{}{}
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for RPC to succeed")
+	case err := <-errCh:
+		if err != nil {
+			t.Fatalf("RPC failed with unexpected error: %v", err)
+		}
+	}
+
+	// Once the RPC finishes, its OnFinish CallOption decrements the refcount
+	// of cluster-A to 0. Verifies that interceptor.Close() is now called.
+	select {
+	case <-closeChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for interceptor Close() after RPC finished")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}
+
+// Test verifies that when a route configuration is updated and there are no
+// active in-flight RPCs on a route/cluster, the HTTP filter interceptor is
+// closed immediately when the old config selector is stopped.
+func (s) TestResolverImmediateInterceptorClose_NoActiveRPCs(t *testing.T) {
+	testFilterTypeURL := t.Name()
+	tb := newTestFilterBuilder(t, testFilterTypeURL)
+	closeChan := tb.closeChan
+	close(tb.blockChan) // Do not block RPCs in the filter
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	r, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bc)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Intercept resolver builder.
+	jsonCh := make(chan string, 1)
+	ib := &wrappingBuilder{
+		Builder: r,
+		jsonCh:  jsonCh,
+	}
+
+	// Start test backends.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const (
+		serviceName = "my-service-xds"
+		clusterA    = "cluster-A"
+		clusterB    = "cluster-B"
+	)
+	hcm := &v3httppb.HttpConnectionManager{
+		RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+			RouteConfig: &v3routepb.RouteConfiguration{
+				Name: "route-" + serviceName,
+				VirtualHosts: []*v3routepb.VirtualHost{{
+					Domains: []string{serviceName},
+					Routes: []*v3routepb.Route{{
+						Match:  &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}},
+						Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterA}}},
+					}},
+				}},
+			},
+		},
+		HttpFilters: []*v3httppb.HttpFilter{
+			{
+				Name: "closing-filter",
+				ConfigType: &v3httppb.HttpFilter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+						TypeUrl: testFilterTypeURL,
+					}),
+				},
+			},
+			e2e.RouterHTTPFilter,
+		},
+	}
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterA, clusterA, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterA, "localhost", []uint32{testutils.ParsePort(t, server.Address)})},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(ib))
+	if err != nil {
+		t.Fatalf("Failed to create a gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA)
+
+	// Update route configuration on the management server to point to cluster B.
+	hcm.GetRouteConfig().VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterB}}}
+	resources.Listeners = []*v3listenerpb.Listener{{Name: serviceName, ApiListener: &v3listenerpb.ApiListener{ApiListener: testutils.MarshalAny(t, hcm)}}}
+	resources.Clusters = append(resources.Clusters, e2e.DefaultCluster(clusterB, clusterB, e2e.SecurityLevelNone))
+	resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterB, "localhost", []uint32{testutils.ParsePort(t, server.Address)}))
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterA, clusterB)
+
+	// Because there are no active RPCs on cluster-A, interceptor.Close() should
+	// be invoked immediately when the old config selector is stopped.
+	select {
+	case <-closeChan:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for interceptor Close() when no active RPCs exist")
+	}
+
+	verifyServiceConfig(ctx, t, jsonCh, clusterB)
+}

test/xds/xds_rls_clusterspecifier_plugin_test.go

@@ -0,0 +1,168 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/rls"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/balancer/rls" // Register the RLS Load Balancing policy.
+)
+
+// defaultClientResourcesWithRLSCSP returns a set of resources (LDS, RDS, CDS, EDS) for a
+// client to connect to a server with a RLS Load Balancer as a child of Cluster Manager.
+func defaultClientResourcesWithRLSCSP(t *testing.T, lb e2e.LoadBalancingPolicy, params e2e.ResourceParams, rlsProto *rlspb.RouteLookupConfig) e2e.UpdateOptions {
+	routeConfigName := "route-" + params.DialTarget
+	clusterName := "cluster-" + params.DialTarget
+	endpointsName := "endpoints-" + params.DialTarget
+	return e2e.UpdateOptions{
+		NodeID:    params.NodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)},
+		Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{
+			RouteConfigName:            routeConfigName,
+			ListenerName:               params.DialTarget,
+			ClusterSpecifierType:       e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin,
+			ClusterSpecifierPluginName: "rls-csp",
+			ClusterSpecifierPluginConfig: testutils.MarshalAny(t, &rlspb.RouteLookupClusterSpecifier{
+				RouteLookupConfig: rlsProto,
+			}),
+		})},
+		Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+			ClusterName:   clusterName,
+			ServiceName:   endpointsName,
+			Policy:        lb,
+			SecurityLevel: params.SecLevel,
+		})},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})},
+	}
+}
+
+// TestRLSinxDS tests an xDS configured system with an RLS Balancer present.
+//
+// This test sets up the RLS Balancer using the RLS Cluster Specifier Plugin,
+// spins up a test service and has a fake RLS Server correctly respond with a
+// target corresponding to this test service. This test asserts an RPC proceeds
+// as normal with the RLS Balancer as part of system.
+func (s) TestRLSinxDS(t *testing.T) {
+	tests := []struct {
+		name     string
+		lbPolicy e2e.LoadBalancingPolicy
+	}{
+		{
+			name:     "roundrobin",
+			lbPolicy: e2e.LoadBalancingPolicyRoundRobin,
+		},
+		{
+			name:     "ringhash",
+			lbPolicy: e2e.LoadBalancingPolicyRingHash,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			testRLSinxDS(t, test.lbPolicy)
+		})
+	}
+}
+
+func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) {
+	// Set up all components and configuration necessary - management server,
+	// xDS resolver, fake RLS Server, and xDS configuration which specifies an
+	// RLS Balancer that communicates to this set up fake RLS Server.
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	lis := testutils.NewListenerWrapper(t, nil)
+	rlsServer, rlsRequestCh := rls.SetupFakeRLSServer(t, lis)
+	rlsProto := &rlspb.RouteLookupConfig{
+		GrpcKeybuilders:      []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}},
+		LookupService:        rlsServer.Address,
+		LookupServiceTimeout: durationpb.New(defaultTestTimeout),
+		CacheSizeBytes:       1024,
+	}
+
+	const serviceName = "my-service-client-side-xds"
+	resources := defaultClientResourcesWithRLSCSP(t, lbPolicy, e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	}, rlsProto)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the fake RLS Server to set the RLS Balancers child CDS
+	// Cluster's name as the target for the RPC to use.
+	rlsServer.SetResponseCallback(func(context.Context, *rlspb.RouteLookupRequest) *rls.RouteLookupResponse {
+		return &rls.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{"cluster-" + serviceName}}}
+	})
+
+	// Create a ClientConn and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to dial local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	// Successfully sending the RPC will require the RLS Load Balancer to
+	// communicate with the fake RLS Server for information about the target.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// These RLS Verifications makes sure the RLS Load Balancer is actually part
+	// of the xDS Configured system that correctly sends out RPC.
+
+	// Verify connection is established to RLS Server.
+	if _, err = lis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for RLS LB policy to create control channel")
+	}
+
+	// Verify an rls request is sent out to fake RLS Server.
+	select {
+	case <-ctx.Done():
+		t.Fatalf("Timeout when waiting for an RLS request to be sent out")
+	case <-rlsRequestCh:
+	}
+}

test/xds/xds_security_config_nack_test.go

@@ -0,0 +1,377 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/resolver"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+	"github.com/google/uuid"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) {
+	const (
+		serviceName                     = "my-service-client-side-xds"
+		missingIdentityProviderInstance = "missing-identity-provider-instance"
+		missingRootProviderInstance     = "missing-root-provider-instance"
+	)
+
+	tests := []struct {
+		name           string
+		securityConfig *v3corepb.TransportSocket
+		wantErr        bool
+	}{
+		{
+			name: "both identity and root providers are not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only identity provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ServerSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only root provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ServerSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "both identity and root providers are present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.DownstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ServerSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ServerSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelMTLS,
+			})
+
+			// Create an inbound xDS listener resource for the server side.
+			inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")
+			for _, fc := range inboundLis.GetFilterChains() {
+				fc.TransportSocket = test.securityConfig
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			// Make a context with a shorter timeout from the top level test
+			// context for cases where we expect failures.
+			timeout := defaultTestTimeout
+			if test.wantErr {
+				timeout = defaultTestShortTimeout
+			}
+			ctx, cancel = context.WithTimeout(ctx, timeout)
+			defer cancel()
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr {
+				t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}
+
+func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) {
+	const (
+		serviceName                     = "my-service-client-side-xds"
+		missingIdentityProviderInstance = "missing-identity-provider-instance"
+		missingRootProviderInstance     = "missing-root-provider-instance"
+	)
+
+	tests := []struct {
+		name           string
+		securityConfig *v3corepb.TransportSocket
+		wantErr        bool
+	}{
+		{
+			name: "both identity and root providers are not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only identity provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: missingIdentityProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only root provider is not present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: missingRootProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "both identity and root providers are present in bootstrap",
+			securityConfig: &v3corepb.TransportSocket{
+				Name: "envoy.transport_sockets.tls",
+				ConfigType: &v3corepb.TransportSocket_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
+						CommonTlsContext: &v3tlspb.CommonTlsContext{
+							TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+								InstanceName: e2e.ClientSideCertProviderInstance,
+							},
+							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{
+								ValidationContext: &v3tlspb.CertificateValidationContext{
+									CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{
+										InstanceName: e2e.ClientSideCertProviderInstance,
+									},
+								},
+							},
+						},
+					}),
+				},
+			},
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+			// Create bootstrap configuration pointing to the above management
+			// server with certificate provider configuration.
+			nodeID := uuid.New().String()
+			bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+			// Create an xDS resolver with the above bootstrap configuration.
+			xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+			if err != nil {
+				t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+			}
+
+			server := stubserver.StartTestService(t, nil)
+			defer server.Stop()
+
+			// This creates a `Cluster` resource with a security config which
+			// refers to `e2e.ClientSideCertProviderInstance` for both root and
+			// identity certs.
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       "localhost",
+				Port:       testutils.ParsePort(t, server.Address),
+				SecLevel:   e2e.SecurityLevelMTLS,
+			})
+			resources.Clusters[0].TransportSocket = test.securityConfig
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			// Make a context with a shorter timeout from the top level test
+			// context for cases where we expect failures.
+			timeout := defaultTestTimeout
+			if test.wantErr {
+				timeout = defaultTestShortTimeout
+			}
+			ctx2, cancel2 := context.WithTimeout(ctx, timeout)
+			defer cancel2()
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx2, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr {
+				t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}

test/xds/xds_server_filter_e2e_test.go

@@ -0,0 +1,402 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func makeUnaryRPC(ctx context.Context, client testgrpc.TestServiceClient) error {
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		return fmt.Errorf("EmptyCall() failed: %w", err)
+	}
+	return nil
+}
+
+func makeStreamingRPC(ctx context.Context, client testgrpc.TestServiceClient) error {
+	stream, err := client.FullDuplexCall(ctx)
+	if err != nil {
+		return fmt.Errorf("FullDuplexCall() failed: %w", err)
+	}
+	if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
+		return fmt.Errorf("stream.Send() failed: %w", err)
+	}
+	if err := stream.CloseSend(); err != nil {
+		return fmt.Errorf("stream.CloseSend() failed: %w", err)
+	}
+	if _, err := stream.Recv(); err != io.EOF {
+		return err
+	}
+	return nil
+}
+
+// setupXDSListenerAndClient starts an xDS management server and a gRPC server,
+// configures the management server with an inbound listener containing the
+// specified httpFilters, dials the server using xDS, waits for the server to
+// enter SERVING mode, and returns a TestServiceClient.
+func setupXDSListenerAndClient(t *testing.T, httpFilters ...*v3httppb.HttpFilter) testgrpc.TestServiceClient {
+	t.Helper()
+	const serviceName = "my-service"
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	t.Cleanup(stopServer)
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("Failed to retrieve host and port of server: %v", err)
+	}
+
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+
+	filters := append([]*v3httppb.HttpFilter{}, httpFilters...)
+	filters = append(filters, e2e.HTTPFilter("router", &v3routerpb.Router{}))
+
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: filters,
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatalf("managementServer.Update() failed: %v", err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	t.Cleanup(func() { cc.Close() })
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	return testgrpc.NewTestServiceClient(cc)
+}
+
+// Test verifies that the xDS server-side HTTP filter chain correctly executes
+// InterceptRPC and wraps grpc.ServerStream for both Unary and Streaming RPCs.
+func (s) TestServerSideXDSHTTPFilter_InterceptRPC(t *testing.T) {
+	dummyTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(dummyTypeURL)
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	dummyFilter := newHTTPFilter(t, "dummy-filter", dummyTypeURL, "")
+	client := setupXDSListenerAndClient(t, dummyFilter)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC execution.
+	if err := makeUnaryRPC(ctx, client); err != nil {
+		t.Fatalf("makeUnaryRPC() failed: %v", err)
+	}
+	if got := fb.interceptRPCCount.Load(); got != 1 {
+		t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got)
+	}
+	if got := fb.recvMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected recvMsgCount for unary RPC, got %d, want 1", got)
+	}
+	if got := fb.sendMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected sendMsgCount for unary RPC, got %d, want 1", got)
+	}
+
+	// Verify Streaming RPC execution.
+	if err := makeStreamingRPC(ctx, client); err != nil {
+		t.Fatalf("makeStreamingRPC() failed: %v", err)
+	}
+	if got := fb.interceptRPCCount.Load(); got != 2 {
+		t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got)
+	}
+	if got := fb.recvMsgCount.Load(); got != 2 {
+		t.Fatalf("Unexpected recvMsgCount for streaming RPC, got %d, want 2", got)
+	}
+	// Since the server handler did not call SendMsg for the streaming RPC, the
+	// sendMsg count for this RPC is zero, so the total count remains 1.
+	if got := fb.sendMsgCount.Load(); got != 1 {
+		t.Fatalf("Unexpected sendMsgCount for streaming RPC, got %d, want 1", got)
+	}
+}
+
+// Test verifies that when a server-side xDS HTTP filter returns an error from
+// InterceptRPC, the RPC is rejected early with the expected error code and
+// message for both Unary and Streaming RPCs.
+func (s) TestServerSideXDSHTTPFilter_InterceptRPCEarlyRejection(t *testing.T) {
+	tests := []struct {
+		name       string
+		errToRet   error
+		wantCode   codes.Code
+		wantErrMsg string
+	}{
+		{
+			name:       "InterceptRPC_FailsWithStatusError",
+			errToRet:   status.Error(codes.PermissionDenied, "access denied by xDS server filter"),
+			wantCode:   codes.PermissionDenied,
+			wantErrMsg: "access denied by xDS server filter",
+		},
+		{
+			name:       "InterceptRPC_FailsWithNonStatusError",
+			errToRet:   errors.New("non-status error from xDS filter"),
+			wantCode:   codes.Unknown,
+			wantErrMsg: "non-status error from xDS filter",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			typeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(typeURL)
+			fb.interceptRPCFunc = func(grpc.ServerStream) (grpc.ServerStream, error) {
+				return nil, test.errToRet
+			}
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			rejectFilter := newHTTPFilter(t, "reject-filter", typeURL, "")
+			client := setupXDSListenerAndClient(t, rejectFilter)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			// Verify Unary RPC rejection.
+			if err := makeUnaryRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) {
+				t.Fatalf("makeUnaryRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg)
+			}
+			if got := fb.interceptRPCCount.Load(); got != 1 {
+				t.Fatalf("Unexpected interceptRPCCount for unary RPC, got %d, want 1", got)
+			}
+
+			// Verify Streaming RPC rejection.
+			if err := makeStreamingRPC(ctx, client); status.Code(err) != test.wantCode || !strings.Contains(err.Error(), test.wantErrMsg) {
+				t.Fatalf("makeStreamingRPC() returned err = %v (code: %v), want code %v and msg %q", err, status.Code(err), test.wantCode, test.wantErrMsg)
+			}
+			if got := fb.interceptRPCCount.Load(); got != 2 {
+				t.Fatalf("Unexpected interceptRPCCount for streaming RPC, got %d, want 2", got)
+			}
+		})
+	}
+}
+
+// Test verifies that multiple server-side xDS HTTP filters in a chain execute
+// their InterceptRPC methods sequentially in the configured order for both
+// Unary and Streaming RPCs.
+func (s) TestServerSideXDS_InterceptRPCMultiFilterChaining(t *testing.T) {
+	typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name())
+	eventsCh := make(chan string, 2)
+	const filter1, filter2 = "intercepted-filter-1", "intercepted-filter-2"
+
+	fb1 := newTrackingHTTPFilterBuilder(typeURL1)
+	fb1.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		eventsCh <- filter1
+		return ss, nil
+	}
+	fb2 := newTrackingHTTPFilterBuilder(typeURL2)
+	fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		eventsCh <- filter2
+		return ss, nil
+	}
+
+	httpfilter.Register(fb1)
+	defer httpfilter.UnregisterForTesting(fb1.typeURL)
+	httpfilter.Register(fb2)
+	defer httpfilter.UnregisterForTesting(fb2.typeURL)
+
+	f1 := newHTTPFilter(t, "filter-1", typeURL1, "")
+	f2 := newHTTPFilter(t, "filter-2", typeURL2, "")
+
+	client := setupXDSListenerAndClient(t, f1, f2)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC filter chaining.
+	if err := makeUnaryRPC(ctx, client); err != nil {
+		t.Fatalf("makeUnaryRPC() failed: %v", err)
+	}
+	select {
+	case e := <-eventsCh:
+		if e != filter1 {
+			t.Fatalf("First intercept event = %q, want %q", e, filter1)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter1)
+	}
+
+	select {
+	case e := <-eventsCh:
+		if e != filter2 {
+			t.Fatalf("Second intercept event = %q, want %q", e, filter2)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter2)
+	}
+
+	// Verify Streaming RPC filter chaining.
+	if err := makeStreamingRPC(ctx, client); err != nil {
+		t.Fatalf("makeStreamingRPC() failed: %v", err)
+	}
+	select {
+	case e := <-eventsCh:
+		if e != filter1 {
+			t.Fatalf("First intercept event = %q, want %q", e, filter1)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter1)
+	}
+
+	select {
+	case e := <-eventsCh:
+		if e != filter2 {
+			t.Fatalf("Second intercept event = %q, want %q", e, filter2)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %q", filter2)
+	}
+}
+
+type errorWrappedStream struct {
+	grpc.ServerStream
+	recvErr string
+}
+
+func (w *errorWrappedStream) RecvMsg(m any) error {
+	if w.recvErr != "" {
+		return status.Error(codes.Internal, w.recvErr)
+	}
+	return w.ServerStream.RecvMsg(m)
+}
+
+// Test verifies that when multiple filters wrap the server stream and a
+// filter's overridden RecvMsg returns an error, that error is properly
+// propagated back to the client for both Unary and Streaming RPCs.
+func (s) TestServerSideXDS_InterceptRPCMultiFilterErrorPropagation(t *testing.T) {
+	typeURL1, typeURL2 := fmt.Sprintf("%s-1", t.Name()), fmt.Sprintf("%s-2", t.Name())
+
+	fb1 := newTrackingHTTPFilterBuilder(typeURL1)
+
+	const wantErr = "recvMsg failed in filter 2"
+	fb2 := newTrackingHTTPFilterBuilder(typeURL2)
+	fb2.interceptRPCFunc = func(ss grpc.ServerStream) (grpc.ServerStream, error) {
+		return &errorWrappedStream{ServerStream: ss, recvErr: wantErr}, nil
+	}
+
+	httpfilter.Register(fb1)
+	defer httpfilter.UnregisterForTesting(fb1.typeURL)
+	httpfilter.Register(fb2)
+	defer httpfilter.UnregisterForTesting(fb2.typeURL)
+
+	filter1 := newHTTPFilter(t, "filter-1", typeURL1, "")
+	filter2 := newHTTPFilter(t, "filter-2", typeURL2, "")
+
+	client := setupXDSListenerAndClient(t, filter1, filter2)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Verify Unary RPC error propagation.
+	if err := makeUnaryRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("makeUnaryRPC() returned error = %v, want error containing %q", err, wantErr)
+	}
+
+	// Verify Streaming RPC error propagation.
+	if err := makeStreamingRPC(ctx, client); err == nil || !strings.Contains(err.Error(), wantErr) {
+		t.Fatalf("makeStreamingRPC() returned error = %v, want error containing %q", err, wantErr)
+	}
+}

test/xds/xds_server_filter_override_test.go

@@ -0,0 +1,393 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/xds"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+)
+
+type overrideInfo struct {
+	disabled bool
+	path     string
+}
+
+func makeOverrideConfig(t *testing.T, typeURL string, info *overrideInfo) *anypb.Any {
+	if info == nil {
+		return nil
+	}
+	cfg := &v3routepb.FilterConfig{Disabled: info.disabled}
+	if info.path != "" {
+		cfg.Config = testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+			TypeUrl: typeURL,
+			Value: &structpb.Struct{
+				Fields: map[string]*structpb.Value{
+					"path": {Kind: &structpb.Value_StringValue{StringValue: info.path}},
+				},
+			},
+		})
+	}
+	return testutils.MarshalAny(t, cfg)
+}
+
+// TestServerSideXDS_FilterOverride_Disabled verifies that filters are NOT
+// created when disabled by either base config or overrides at either
+// VirtualHost or Route level.
+func (s) TestServerSideXDS_FilterOverride_Disabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true)
+
+	tests := []struct {
+		name         string
+		baseDisabled bool
+		vhost        *overrideInfo
+		route        *overrideInfo
+	}{
+		{
+			name:         "BaseDisabled_NoOverrides",
+			baseDisabled: true,
+		},
+		{
+			name:         "BaseEnabled_VHostDisables",
+			baseDisabled: false,
+			vhost:        &overrideInfo{disabled: true},
+		},
+		{
+			name:         "BaseEnabled_VHostEnabled_RouteDisables",
+			baseDisabled: false,
+			vhost:        &overrideInfo{disabled: false, path: "vhost-value"},
+			route:        &overrideInfo{disabled: true},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			testFilterTypeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			// Wait for the server to enter SERVING mode before making RPCs to avoid
+			// flakes due to the server closing connections.
+			servingCh := make(chan struct{})
+			opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+				if args.Mode == connectivity.ServingModeServing {
+					close(servingCh)
+				}
+			})
+			lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+			defer stopServer()
+
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("Failed to retrieve host and port of server: %v", err)
+			}
+			const serviceName = "my-service"
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelNone,
+			})
+
+			baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, "base-path")
+			baseFilter.Disabled = tc.baseDisabled
+
+			var vhOverrideConfig map[string]*anypb.Any
+			if tc.vhost != nil {
+				vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)}
+			}
+			var routeOverrideConfig map[string]*anypb.Any
+			if tc.route != nil {
+				routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)}
+			}
+
+			vhs := []*v3routepb.VirtualHost{{
+				Domains: []string{"*"},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action:               &v3routepb.Route_NonForwardingAction{},
+					TypedPerFilterConfig: routeOverrideConfig,
+				}},
+				TypedPerFilterConfig: vhOverrideConfig,
+			}}
+
+			networkFilters := []*v3listenerpb.Filter{{
+				Name: "hcm",
+				ConfigType: &v3listenerpb.Filter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+						HttpFilters: []*v3httppb.HttpFilter{
+							baseFilter,
+							e2e.HTTPFilter("router", &v3routerpb.Router{}),
+						},
+						RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+							RouteConfig: &v3routepb.RouteConfiguration{
+								Name:         "routeName",
+								VirtualHosts: vhs,
+							},
+						},
+					}),
+				},
+			}}
+
+			inboundLis := &v3listenerpb.Listener{
+				Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+				Address: &v3corepb.Address{
+					Address: &v3corepb.Address_SocketAddress{
+						SocketAddress: &v3corepb.SocketAddress{
+							Address: host,
+							PortSpecifier: &v3corepb.SocketAddress_PortValue{
+								PortValue: port,
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			select {
+			case <-servingCh:
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for server to enter SERVING mode")
+			}
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall() failed: %v", err)
+			}
+
+			// Verify no filters or interceptors were created.
+			if got := fb.filtersCreated.Load(); got != 0 {
+				t.Fatalf("filtersCreated = %d; want 0", got)
+			}
+			if got := fb.interceptorsCreated.Load(); got != 0 {
+				t.Fatalf("interceptorsCreated = %d; want 0", got)
+			}
+		})
+	}
+}
+
+// TestServerSideXDS_FilterOverride_Enabled verifies that filters ARE created
+// and receive the correct configuration when enabled by either base config or
+// overrides at either VirtualHost or Route level.
+func (s) TestServerSideXDS_FilterOverride_Enabled(t *testing.T) {
+	testutils.SetEnvConfig(t, &envconfig.XDSClientExtProcEnabled, true)
+
+	const basePath = "base-path"
+
+	tests := []struct {
+		name         string
+		baseDisabled bool
+		vhost        *overrideInfo
+		route        *overrideInfo
+		wantPath     string
+	}{
+		{
+			name:         "BaseEnabled_NoOverrides",
+			baseDisabled: false,
+			wantPath:     basePath,
+		},
+		{
+			name:         "VHostEnables",
+			baseDisabled: true,
+			vhost:        &overrideInfo{disabled: false, path: "vhost-value"},
+			wantPath:     "vhost-value",
+		},
+		{
+			name:         "RouteEnables",
+			baseDisabled: true,
+			vhost:        &overrideInfo{disabled: true},
+			route:        &overrideInfo{disabled: false, path: "route-value"},
+			wantPath:     "route-value",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			pathCh := make(chan string, 1)
+			testFilterTypeURL := t.Name()
+			fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+			fb.pathCh = pathCh
+			httpfilter.Register(fb)
+			defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+			servingCh := make(chan struct{})
+			opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+				if args.Mode == connectivity.ServingModeServing {
+					close(servingCh)
+				}
+			})
+			lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+			defer stopServer()
+
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("Failed to retrieve host and port of server: %v", err)
+			}
+			const serviceName = "my-service"
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   e2e.SecurityLevelNone,
+			})
+
+			baseFilter := newHTTPFilter(t, "test-filter", testFilterTypeURL, basePath)
+			baseFilter.Disabled = tc.baseDisabled
+
+			var vhOverrideConfig map[string]*anypb.Any
+			if tc.vhost != nil {
+				vhOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.vhost)}
+			}
+			var routeOverrideConfig map[string]*anypb.Any
+			if tc.route != nil {
+				routeOverrideConfig = map[string]*anypb.Any{"test-filter": makeOverrideConfig(t, testFilterTypeURL, tc.route)}
+			}
+
+			vhs := []*v3routepb.VirtualHost{{
+				Domains: []string{"*"},
+				Routes: []*v3routepb.Route{{
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action:               &v3routepb.Route_NonForwardingAction{},
+					TypedPerFilterConfig: routeOverrideConfig,
+				}},
+				TypedPerFilterConfig: vhOverrideConfig,
+			}}
+
+			networkFilters := []*v3listenerpb.Filter{{
+				Name: "hcm",
+				ConfigType: &v3listenerpb.Filter_TypedConfig{
+					TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+						HttpFilters: []*v3httppb.HttpFilter{
+							baseFilter,
+							e2e.HTTPFilter("router", &v3routerpb.Router{}),
+						},
+						RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+							RouteConfig: &v3routepb.RouteConfiguration{
+								Name:         "routeName",
+								VirtualHosts: vhs,
+							},
+						},
+					}),
+				},
+			}}
+
+			inboundLis := &v3listenerpb.Listener{
+				Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+				Address: &v3corepb.Address{
+					Address: &v3corepb.Address_SocketAddress{
+						SocketAddress: &v3corepb.SocketAddress{
+							Address: host,
+							PortSpecifier: &v3corepb.SocketAddress_PortValue{
+								PortValue: port,
+							},
+						},
+					},
+				},
+				DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+			}
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("grpc.NewClient() failed: %v", err)
+			}
+			defer cc.Close()
+
+			select {
+			case <-servingCh:
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for server to enter SERVING mode")
+			}
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+				t.Fatalf("EmptyCall() failed: %v", err)
+			}
+
+			// Verify invoked with expected path.
+			select {
+			case p := <-pathCh:
+				if p != tc.wantPath {
+					t.Fatalf("Unexpected path received by filter: got %q, want %q", p, tc.wantPath)
+				}
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for filter to be invoked")
+			}
+			if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+				t.Fatalf("Created %d filter instances, want: %d", got, want)
+			}
+			if got, want := fb.interceptorsCreated.Load(), int32(1); got != want {
+				t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+			}
+		})
+	}
+}

test/xds/xds_server_filter_state_retention_test.go

@@ -0,0 +1,699 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	iresolver "google.golang.org/grpc/internal/resolver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/xds"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const filterCfgPathFieldName = "path"
+
+// testFilterCfg is the internal representation of the filter config proto. It
+// is returned by filter's config parsing methods.
+type testFilterCfg struct {
+	httpfilter.FilterConfig
+	path string
+}
+
+// filterConfigFromProto parses filter config specified as a v3.TypedStruct into
+// a testFilterCfg.
+func filterConfigFromProto(cfg proto.Message) (httpfilter.FilterConfig, error) {
+	ts, ok := cfg.(*v3xdsxdstypepb.TypedStruct)
+	if !ok {
+		return nil, fmt.Errorf("unsupported filter config type: %T, want %T", cfg, &v3xdsxdstypepb.TypedStruct{})
+	}
+
+	if ts.GetValue() == nil {
+		return testFilterCfg{}, nil
+	}
+	ret := testFilterCfg{}
+	if v := ts.GetValue().GetFields()[filterCfgPathFieldName]; v != nil {
+		ret.path = v.GetStringValue()
+	}
+	return ret, nil
+}
+
+// trackingHTTPFilterBuilder is a test filter that allows counting and tracking
+// the lifecycle and execution of filter and interceptor instances.
+type trackingHTTPFilterBuilder struct {
+	httpfilter.Builder
+	filtersCreated        atomic.Int32
+	filtersDestroyed      atomic.Int32
+	interceptorsCreated   atomic.Int32
+	interceptorsDestroyed atomic.Int32
+	interceptRPCCount     atomic.Int32
+	recvMsgCount          atomic.Int32
+	sendMsgCount          atomic.Int32
+	closeSendCount        atomic.Int32
+	typeURL               string
+	pathCh                chan string
+	interceptRPCFunc      func(ss grpc.ServerStream) (grpc.ServerStream, error)
+}
+
+func newTrackingHTTPFilterBuilder(typeURL string) *trackingHTTPFilterBuilder {
+	return &trackingHTTPFilterBuilder{typeURL: typeURL}
+}
+
+func (t *trackingHTTPFilterBuilder) IsTerminal() bool { return false }
+
+func (t *trackingHTTPFilterBuilder) TypeURLs() []string { return []string{t.typeURL} }
+
+func (*trackingHTTPFilterBuilder) ParseFilterConfig(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return filterConfigFromProto(cfg)
+}
+
+func (*trackingHTTPFilterBuilder) ParseFilterConfigOverride(cfg proto.Message, _ httpfilter.ParseOptions) (httpfilter.FilterConfig, error) {
+	return filterConfigFromProto(cfg)
+}
+
+func (t *trackingHTTPFilterBuilder) BuildServerFilter() httpfilter.ServerFilter {
+	t.filtersCreated.Add(1)
+	return t
+}
+
+func (t *trackingHTTPFilterBuilder) Close() {
+	t.filtersDestroyed.Add(1)
+}
+
+var _ httpfilter.ClientFilterBuilder = &trackingHTTPFilterBuilder{}
+var _ httpfilter.ServerFilterBuilder = &trackingHTTPFilterBuilder{}
+
+func (t *trackingHTTPFilterBuilder) BuildServerInterceptor(config, override httpfilter.FilterConfig) (httpfilter.ServerInterceptor, error) {
+	t.interceptorsCreated.Add(1)
+
+	var effectiveCfg testFilterCfg
+	if override != nil {
+		effectiveCfg = override.(testFilterCfg)
+	} else if config != nil {
+		effectiveCfg = config.(testFilterCfg)
+	} else {
+		return nil, fmt.Errorf("unexpected missing config")
+	}
+
+	interceptor := &trackingServerInterceptor{
+		parent:   t,
+		pathCh:   t.pathCh,
+		basePath: effectiveCfg.path,
+	}
+	return interceptor, nil
+}
+
+type trackingServerInterceptor struct {
+	parent   *trackingHTTPFilterBuilder
+	pathCh   chan string
+	basePath string
+}
+
+func (i *trackingServerInterceptor) InterceptRPC(ss grpc.ServerStream) (grpc.ServerStream, error) {
+	i.parent.interceptRPCCount.Add(1)
+	if i.pathCh != nil {
+		i.pathCh <- i.basePath
+	}
+	if i.parent.interceptRPCFunc != nil {
+		return i.parent.interceptRPCFunc(ss)
+	}
+	return &wrappedServerStream{
+		ServerStream: ss,
+		parent:       i.parent,
+	}, nil
+}
+
+func (i *trackingServerInterceptor) Close() {
+	i.parent.interceptorsDestroyed.Add(1)
+}
+
+// wrappedServerStream wraps grpc.ServerStream to intercept and count
+// RecvMsg and SendMsg invocations for testing.
+type wrappedServerStream struct {
+	grpc.ServerStream
+	parent *trackingHTTPFilterBuilder
+}
+
+func (w *wrappedServerStream) RecvMsg(m any) error {
+	if err := w.ServerStream.RecvMsg(m); err != nil {
+		return err
+	}
+	w.parent.recvMsgCount.Add(1)
+	return nil
+}
+
+func (w *wrappedServerStream) SendMsg(m any) error {
+	if err := w.ServerStream.SendMsg(m); err != nil {
+		return err
+	}
+	w.parent.sendMsgCount.Add(1)
+	return nil
+}
+
+func newHTTPFilter(t *testing.T, name, typeURL, path string) *v3httppb.HttpFilter {
+	return &v3httppb.HttpFilter{
+		Name: name,
+		ConfigType: &v3httppb.HttpFilter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3xdsxdstypepb.TypedStruct{
+				TypeUrl: typeURL,
+				Value: &structpb.Struct{
+					Fields: map[string]*structpb.Value{
+						filterCfgPathFieldName: {Kind: &structpb.Value_StringValue{StringValue: path}},
+					},
+				},
+			}),
+		},
+	}
+}
+
+func (t *trackingHTTPFilterBuilder) BuildClientFilter(httpfilter.ClientFilterOptions) httpfilter.ClientFilter {
+	t.filtersCreated.Add(1)
+	return t
+}
+
+func (t *trackingHTTPFilterBuilder) BuildClientInterceptor(_, _ httpfilter.FilterConfig) (httpfilter.ClientInterceptor, error) {
+	t.interceptorsCreated.Add(1)
+	return &trackingClientInterceptor{parent: t}, nil
+}
+
+type trackingClientInterceptor struct {
+	parent *trackingHTTPFilterBuilder
+}
+
+func (i *trackingClientInterceptor) NewStream(ctx context.Context, _ iresolver.RPCInfo, newStream func(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStream, error), opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	s, err := newStream(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &wrappedClientStream{
+		ClientStream: s,
+		parent:       i.parent,
+	}, nil
+}
+
+func (i *trackingClientInterceptor) Close() {
+	i.parent.interceptorsDestroyed.Add(1)
+}
+
+type wrappedClientStream struct {
+	grpc.ClientStream
+	parent *trackingHTTPFilterBuilder
+}
+
+func (w *wrappedClientStream) RecvMsg(m any) error {
+	w.parent.recvMsgCount.Add(1)
+	return w.ClientStream.RecvMsg(m)
+}
+
+func (w *wrappedClientStream) SendMsg(m any) error {
+	w.parent.sendMsgCount.Add(1)
+	return w.ClientStream.SendMsg(m)
+}
+
+func (w *wrappedClientStream) CloseSend() error {
+	w.parent.closeSendCount.Add(1)
+	return w.ClientStream.CloseSend()
+}
+
+// Tests the filter state retention behavior when filter configs in the existing
+// filter chain are updated, but there are no changes to the filter names. In
+// this case, the existing filter instance should be retained, while new
+// interceptor instances should be created with the updated config.
+func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterConfigChange(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "filter-1",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "v4-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+		},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Setup the management server with client and server-side resources.
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and two interceptors are created
+	// (one per filter chain).
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "initial-path"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Update the filter config in the listener resource.
+	networkFilters = []*v3listenerpb.Filter{{
+		Name: "filter-1",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "final-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis.FilterChains[0].Filters = networkFilters
+	inboundLis.FilterChains[1].Filters = networkFilters
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied on the gRPC server.
+WaitForUpdatedConfig:
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case cfg := <-pathCh:
+			if got, want := cfg, "final-path"; got == want {
+				break WaitForUpdatedConfig
+			}
+		case <-ctx.Done():
+			t.Fatalf("Timeout waiting for interceptor get updated config")
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the filter instance is retained, while the interceptor instances
+	// are replaced with the updated config.
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(4); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(2); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+
+	// Stop the server (which causes the listener to be closed) to trigger
+	// cleanup of filters and interceptors, and verify that all instances are
+	// cleaned up.
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}
+
+// Tests the filter state retention behavior when a new filter chain with a
+// filter of the existing type but different filter name is added to the
+// listener resource. In this case, a filter instance should be created because
+// the filter name is part of the key used to identify filter instances.
+func (s) TestServerSideXDS_FilterStateRetention_AcrossUpdates_FilterChainsChange(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "initial-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		DefaultFilterChain: &v3listenerpb.FilterChain{Filters: networkFilters},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	// Setup the management server with client and server-side resources.
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and one interceptor (for the
+	// default filter chain) is created.
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "initial-path"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Add a new filter chain to the listener resource that contains HTTP
+	// filters with a different filter name. This should result in a new filter
+	// instance being created because the filter name is part of the key used to
+	// identify filter instances.
+	networkFilters = []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker-new", testFilterTypeURL, "final-path"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{
+					RouteConfig: &v3routepb.RouteConfiguration{
+						Name:         "routeName",
+						VirtualHosts: vhs,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis.FilterChains = []*v3listenerpb.FilterChain{
+		{
+			Name: "v4-wildcard",
+			FilterChainMatch: &v3listenerpb.FilterChainMatch{
+				PrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "0.0.0.0",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+				SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				SourcePrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "0.0.0.0",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+			},
+			Filters: networkFilters,
+		},
+		{
+			Name: "v6-wildcard",
+			FilterChainMatch: &v3listenerpb.FilterChainMatch{
+				PrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "::",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+				SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+				SourcePrefixRanges: []*v3corepb.CidrRange{{
+					AddressPrefix: "::",
+					PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+				}},
+			},
+			Filters: networkFilters,
+		},
+	}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied on the gRPC server.
+WaitForUpdatedConfig:
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case cfg := <-pathCh:
+			if got, want := cfg, "final-path"; got == want {
+				break WaitForUpdatedConfig
+			}
+		case <-ctx.Done():
+			t.Fatalf("Timeout waiting for interceptor get updated config")
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the a new filter instance is created because of the new filter
+	// name. Three new interceptor instances should also be created (one for the
+	// default filter chain, and two for the newly added filter chains).
+	if got, want := fb.filtersCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(4); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+
+	// Stop the server (which causes the listener to be closed) to trigger
+	// cleanup of filters and interceptors, and verify that all instances are
+	// cleaned up.
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(2); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(4); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}

test/xds/xds_server_integration_test.go

@@ -0,0 +1,548 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	xdscreds "google.golang.org/grpc/credentials/xds"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/grpc/xds"
+
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+func testModeChangeServerOption(t *testing.T) grpc.ServerOption {
+	// Create a server option to get notified about serving mode changes. We don't
+	// do anything other than throwing a log entry here. But this is required,
+	// since the server code emits a log entry at the default level (which is
+	// ERROR) if no callback is registered for serving mode changes. Our
+	// testLogger fails the test if there is any log entry at ERROR level. It does
+	// provide an ExpectError()  method, but that takes a string and it would be
+	// painful to construct the exact error message expected here. Instead this
+	// works just fine.
+	return xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
+		t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
+	})
+}
+
+// acceptNotifyingListener wraps a listener and notifies users when a server
+// calls the Listener.Accept() method. This can be used to ensure that the
+// server is ready before requests are sent to it.
+type acceptNotifyingListener struct {
+	net.Listener
+	serverReady grpcsync.Event
+}
+
+func (l *acceptNotifyingListener) Accept() (net.Conn, error) {
+	l.serverReady.Fire()
+	return l.Listener.Accept()
+}
+
+// setupGRPCServer performs the following:
+//   - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
+//     register the test service on it
+//   - create a local TCP listener and start serving on it
+//
+// Returns the following:
+// - local listener on which the xDS-enabled gRPC server is serving on
+// - cleanup function to be invoked by the tests when done
+func setupGRPCServer(t *testing.T, bootstrapContents []byte, opts ...grpc.ServerOption) (net.Listener, func()) {
+	t.Helper()
+
+	// Configure xDS credentials to be used on the server-side.
+	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Initialize a test gRPC server, assign it to the stub server, and start
+	// the test service.
+	stub := &stubserver.StubServer{
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			return &testpb.Empty{}, nil
+		},
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{}, nil
+		},
+		FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+			for {
+				_, err := stream.Recv() // hangs here forever if stream doesn't shut down...doesn't receive EOF without any errors
+				if err == io.EOF {
+					return nil
+				}
+				if err != nil {
+					return err
+				}
+			}
+		},
+	}
+
+	opts = append([]grpc.ServerOption{
+		grpc.Creds(creds),
+		testModeChangeServerOption(t),
+		xds.BootstrapContentsForTesting(bootstrapContents),
+	}, opts...)
+	if stub.S, err = xds.NewGRPCServer(opts...); err != nil {
+		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
+	}
+
+	// Create a local listener and pass it to Serve().
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+
+	readyLis := &acceptNotifyingListener{
+		Listener:    lis,
+		serverReady: *grpcsync.NewEvent(),
+	}
+
+	stub.Listener = readyLis
+	stubserver.StartTestService(t, stub)
+
+	// Wait for the server to start running.
+	select {
+	case <-readyLis.serverReady.Done():
+	case <-time.After(defaultTestTimeout):
+		t.Fatalf("Timed out while waiting for the backend server to start serving")
+	}
+
+	return lis, func() {
+		stub.S.Stop()
+	}
+}
+
+func hostPortFromListener(lis net.Listener) (string, uint32, error) {
+	host, p, err := net.SplitHostPort(lis.Addr().String())
+	if err != nil {
+		return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
+	}
+	port, err := strconv.ParseInt(p, 10, 32)
+	if err != nil {
+		return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
+	}
+	return host, uint32(port), nil
+}
+
+// TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials
+// fallback functionality.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to not send any security configuration to both
+//     the client and the server. This results in both of them using the
+//     configured fallback credentials (which is insecure creds in this case).
+func (s) TestServerSideXDS_Fallback(t *testing.T) {
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+	defer cleanup2()
+
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resources.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service-fallback"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the server-side
+	// xdsCredentials to use fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	// Setup the management server with client and server-side resources.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn with the xds scheme and make a successful RPC.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Errorf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS
+// credentials with file watcher certificate provider.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to send security configuration to both the
+//     client and the server, pointing to the file watcher certificate provider.
+//     We verify both TLS and mTLS scenarios.
+func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
+	tests := []struct {
+		name     string
+		secLevel e2e.SecurityLevel
+	}{
+		{
+			name:     "tls",
+			secLevel: e2e.SecurityLevelTLS,
+		},
+		{
+			name:     "mtls",
+			secLevel: e2e.SecurityLevelMTLS,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			serviceName := "my-service-file-watcher-certs-" + test.name
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   test.secLevel,
+			})
+
+			// Create an inbound xDS listener resource for the server side that
+			// contains security configuration pointing to the file watcher
+			// plugin.
+			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName")
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			// Setup the management server with client and server resources.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+				FallbackCreds: insecure.NewCredentials(),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("failed to create a client for server: %v", err)
+			}
+			defer cc.Close()
+
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+				t.Fatalf("rpc EmptyCall() failed: %v", err)
+			}
+		})
+	}
+}
+
+// TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on
+// the server-side and xdsCredentials are configured for security. The control
+// plane initially does not any security configuration. This forces the
+// xdsCredentials to use fallback creds, which is this case is insecure creds.
+// We verify that a client connecting with TLS creds is not able to successfully
+// make an RPC. The control plane then sends a listener resource with security
+// configuration pointing to the use of the file_watcher plugin and we verify
+// that the same client is now able to successfully make an RPC.
+func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
+	managementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, managementServer.Address)
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	xdsResolver, err := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+	defer cleanup2()
+
+	// Grab the host and port of the server and create client side xDS resources
+	// corresponding to it. This contains default resources with no security
+	// configuration in the Cluster resource. This should force the xDS
+	// credentials on the client to use its fallback.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service-security-config-change"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Create an inbound xDS listener resource for the server side that does not
+	// contain any security configuration. This should force the xDS credentials
+	// on server to use its fallback.
+	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+
+	// Setup the management server with client and server-side resources.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create client-side xDS credentials with an insecure fallback.
+	xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+		FallbackCreds: insecure.NewCredentials(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a ClientConn with the xds scheme and make a successful RPC.
+	xdsCC, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer xdsCC.Close()
+
+	client := testgrpc.NewTestServiceClient(xdsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+
+	// Create a ClientConn with TLS creds. This should fail since the server is
+	// using fallback credentials which in this case in insecure creds.
+	tlsCreds := testutils.CreateClientTLSCredentials(t)
+	tlsCC, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds))
+	if err != nil {
+		t.Fatalf("failed to create a client for server: %v", err)
+	}
+	defer tlsCC.Close()
+
+	// We don't set 'waitForReady` here since we want this call to failfast.
+	client = testgrpc.NewTestServiceClient(tlsCC)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
+		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
+	}
+
+	// Switch server and client side resources with ones that contain required
+	// security configuration for mTLS with a file watcher certificate provider.
+	resources = e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelMTLS,
+	})
+	inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")
+	resources.Listeners = append(resources.Listeners, inboundLis)
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make another RPC with `waitForReady` set and expect this to succeed.
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// TestServerSideXDS_FileWatcherCertsSPIFFE is an e2e test which verifies xDS
+// credentials with file watcher certificate provider that is configured with a
+// SPIFFE Bundle Map for it's roots.
+//
+// The following sequence of events happen as part of this test:
+//   - An xDS-enabled gRPC server is created and xDS credentials are configured.
+//   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
+//     credentials are configured.
+//   - Control plane is configured to send security configuration to both the
+//     client and the server, pointing to the file watcher certificate provider.
+//     We verify both TLS and mTLS scenarios.
+func (s) TestServerSideXDS_FileWatcherCertsSPIFFE(t *testing.T) {
+	tests := []struct {
+		name     string
+		secLevel e2e.SecurityLevel
+	}{
+		{
+			name:     "tls",
+			secLevel: e2e.SecurityLevelTLS,
+		},
+		{
+			name:     "mtls",
+			secLevel: e2e.SecurityLevelMTLS,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolverWithSPIFFE(t)
+			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
+			defer cleanup2()
+
+			// Grab the host and port of the server and create client side xDS
+			// resources corresponding to it.
+			host, port, err := hostPortFromListener(lis)
+			if err != nil {
+				t.Fatalf("failed to retrieve host and port of server: %v", err)
+			}
+
+			// Create xDS resources to be consumed on the client side. This
+			// includes the listener, route configuration, cluster (with
+			// security configuration) and endpoint resources.
+			serviceName := "my-service-file-watcher-certs-" + test.name
+			resources := e2e.DefaultClientResources(e2e.ResourceParams{
+				DialTarget: serviceName,
+				NodeID:     nodeID,
+				Host:       host,
+				Port:       port,
+				SecLevel:   test.secLevel,
+			})
+
+			// Create an inbound xDS listener resource for the server side that
+			// contains security configuration pointing to the file watcher
+			// plugin.
+			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName")
+			resources.Listeners = append(resources.Listeners, inboundLis)
+
+			// Setup the management server with client and server resources.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := managementServer.Update(ctx, resources); err != nil {
+				t.Fatal(err)
+			}
+
+			// Create client-side xDS credentials with an insecure fallback.
+			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
+				FallbackCreds: insecure.NewCredentials(),
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// Create a ClientConn with the xds scheme and make an RPC.
+			cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(xdsResolver))
+			if err != nil {
+				t.Fatalf("failed to create a client for server: %v", err)
+			}
+			defer cc.Close()
+
+			peer := &peer.Peer{}
+			client := testgrpc.NewTestServiceClient(cc)
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
+				t.Fatalf("rpc EmptyCall() failed: %v", err)
+			}
+			verifySecurityInformationFromPeerSPIFFE(t, peer, test.secLevel, 1)
+		})
+	}
+}
+
+// Checks the AuthInfo available in the peer if it matches the expected security
+// level of the connection.
+func verifySecurityInformationFromPeerSPIFFE(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel, wantPeerChainLen int) {
+	// This is not a true helper in the Go sense, because it does not perform
+	// setup or cleanup tasks. Marking it a helper is to ensure that when the
+	// test fails, the line information of the caller is outputted instead of
+	// from here.
+	//
+	// And this function directly calls t.Fatalf() instead of returning an error
+	// and letting the caller decide what to do with it. This is also OK since
+	// all callers will simply end up calling t.Fatalf() with the returned
+	// error, and can't add any contextual information of value to the error
+	// message.
+	t.Helper()
+
+	authType := pr.AuthInfo.AuthType()
+	switch wantSecLevel {
+	case e2e.SecurityLevelNone:
+		if authType != "insecure" {
+			t.Fatalf("AuthType() is %s, want insecure", authType)
+		}
+	case e2e.SecurityLevelMTLS:
+		if authType != "tls" {
+			t.Fatalf("AuthType() is %s, want tls", authType)
+		}
+		ai, ok := pr.AuthInfo.(credentials.TLSInfo)
+		if !ok {
+			t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{})
+		}
+		if len(ai.State.PeerCertificates) != wantPeerChainLen {
+			t.Fatalf("Number of peer certificates is %d, want %d", len(ai.State.PeerCertificates), wantPeerChainLen)
+		}
+	}
+}

test/xds/xds_server_interceptor_leak_test.go

@@ -0,0 +1,274 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/internal/xds/httpfilter"
+	"google.golang.org/grpc/xds"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+)
+
+// Tests that interceptor instances are properly closed when a new RDS
+// configuration is received for the same route configuration name.
+func (s) TestServerSideXDS_InterceptorLeak_RDSUpdate(t *testing.T) {
+	// Register a custom httpFilter builder for the test.
+	pathCh := make(chan string, 1)
+	testFilterTypeURL := t.Name()
+	fb := newTrackingHTTPFilterBuilder(testFilterTypeURL)
+	fb.pathCh = pathCh
+	httpfilter.Register(fb)
+	defer httpfilter.UnregisterForTesting(fb.typeURL)
+
+	managementServer, nodeID, bootstrapContents, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	// Wait for the server to enter SERVING mode before making RPCs to avoid
+	// flakes due to the server closing connections.
+	servingCh := make(chan struct{})
+	opt := xds.ServingModeCallback(func(_ net.Addr, args xds.ServingModeChangeArgs) {
+		if args.Mode == connectivity.ServingModeServing {
+			close(servingCh)
+		}
+	})
+	lis, stopServer := setupGRPCServer(t, bootstrapContents, opt)
+	defer stopServer()
+
+	// Add xDS resources for the client.
+	host, port, err := hostPortFromListener(lis)
+	if err != nil {
+		t.Fatalf("failed to retrieve host and port of server: %v", err)
+	}
+	const serviceName = "my-service"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       host,
+		Port:       port,
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	// Add xDS resources for the server.
+	vhs := []*v3routepb.VirtualHost{{
+		Domains: []string{"*"},
+		Routes: []*v3routepb.Route{{
+			Match: &v3routepb.RouteMatch{
+				PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+			},
+			Action: &v3routepb.Route_NonForwardingAction{},
+		}},
+	}}
+	const routeConfigName = "routeName"
+	rds1 := &v3routepb.RouteConfiguration{
+		Name:         routeConfigName,
+		VirtualHosts: vhs,
+	}
+	networkFilters := []*v3listenerpb.Filter{{
+		Name: "hcm",
+		ConfigType: &v3listenerpb.Filter_TypedConfig{
+			TypedConfig: testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+				HttpFilters: []*v3httppb.HttpFilter{
+					newHTTPFilter(t, "tracker", testFilterTypeURL, "path1"),
+					e2e.HTTPFilter("router", &v3routerpb.Router{}),
+				},
+				RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
+					Rds: &v3httppb.Rds{
+						ConfigSource: &v3corepb.ConfigSource{
+							ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
+						},
+						RouteConfigName: routeConfigName,
+					},
+				},
+			}),
+		},
+	}}
+	inboundLis := &v3listenerpb.Listener{
+		Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))),
+		Address: &v3corepb.Address{
+			Address: &v3corepb.Address_SocketAddress{
+				SocketAddress: &v3corepb.SocketAddress{
+					Address: host,
+					PortSpecifier: &v3corepb.SocketAddress_PortValue{
+						PortValue: port,
+					},
+				},
+			},
+		},
+		FilterChains: []*v3listenerpb.FilterChain{
+			{
+				Name: "v4-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "0.0.0.0",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+			{
+				Name: "v6-wildcard",
+				FilterChainMatch: &v3listenerpb.FilterChainMatch{
+					PrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+					SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK,
+					SourcePrefixRanges: []*v3corepb.CidrRange{{
+						AddressPrefix: "::",
+						PrefixLen:     &wrapperspb.UInt32Value{Value: uint32(0)},
+					}},
+				},
+				Filters: networkFilters,
+			},
+		},
+	}
+	resources.Listeners = append(resources.Listeners, inboundLis)
+	resources.Routes = append(resources.Routes, rds1)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	select {
+	case <-servingCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for server to enter SERVING mode")
+	}
+
+	// Make an RPC and verify that one filter and two interceptors are created
+	// (one per filter chain).
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+
+	select {
+	case cfg := <-pathCh:
+		if got, want := cfg, "path1"; got != want {
+			t.Fatalf("Unexpected config sent to filter, got: %q, want: %q", got, want)
+		}
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for interceptor to be invoked")
+	}
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsCreated.Load(), int32(2); got != want {
+		t.Fatalf("Created %d interceptor instances, want: %d", got, want)
+	}
+
+	// Update the RDS configuration with a new route for the same route configuration
+	// name. This should result in a new interceptor instance being created for
+	// the new route.
+	rds2 := &v3routepb.RouteConfiguration{
+		Name: routeConfigName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{"*"},
+			Routes: []*v3routepb.Route{
+				{
+					Name: "EmptyCall",
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"},
+					},
+					Action: &v3routepb.Route_NonForwardingAction{},
+				},
+				{
+					Name: "Wildcard",
+					Match: &v3routepb.RouteMatch{
+						PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"},
+					},
+					Action: &v3routepb.Route_NonForwardingAction{},
+				},
+			},
+		}},
+	}
+	resources.Routes = []*v3routepb.RouteConfiguration{resources.Routes[0], rds2}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the updated config to be applied. Since the new configuration has
+	// two routes, we expect two interceptor instances to be created for each
+	// filter chain, for a total of 6 interceptor instances. We also wait for
+	// the old interceptors (one per filter chain, two total) to be destroyed,
+	// since destruction happens after the new interceptors are created within
+	// the same RDS update callback. Breaking only on interceptorsCreated==6
+	// would race with the subsequent interceptorsDestroyed increment.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+			t.Fatalf("EmptyCall() failed: %v", err)
+		}
+		select {
+		case <-pathCh:
+		default:
+		}
+		if fb.interceptorsCreated.Load() == 6 && fb.interceptorsDestroyed.Load() == 2 {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for updated config to be applied: %v", ctx.Err())
+	}
+
+	// Verify the filter instance is retained, while the interceptor instances
+	// are replaced with the updated config.
+	if got, want := fb.filtersCreated.Load(), int32(1); got != want {
+		t.Fatalf("Created %d filter instances, want: %d", got, want)
+	}
+
+	stopServer()
+	if got, want := fb.filtersDestroyed.Load(), int32(1); got != want {
+		t.Fatalf("Destroyed %d filter instances, want: %d", got, want)
+	}
+	if got, want := fb.interceptorsDestroyed.Load(), int32(6); got != want {
+		t.Fatalf("Destroyed %d interceptor instances, want: %d", got, want)
+	}
+}

test/xds/xds_telemetry_labels_test.go

@@ -0,0 +1,281 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package xds_test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/experimental/stats/telemetry"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/e2e/setup"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const serviceNameKey = "service_name"
+const serviceNameKeyCSM = "csm.service_name"
+const serviceNamespaceKey = "service_namespace"
+const serviceNamespaceKeyCSM = "csm.service_namespace_name"
+const serviceNameValue = "grpc-service"
+const serviceNamespaceValue = "grpc-service-namespace"
+const backendServiceKey = "grpc.lb.backend_service"
+const backendServiceValue = "cluster-my-service-client-side-xds"
+const localityKey = "grpc.lb.locality"
+const localityValue = `{region="region-1", zone="zone-1", sub_zone="subzone-1"}`
+
+// TestTelemetryLabels tests that telemetry labels from CDS make their way to
+// the stats handler. The stats handler sets the mutable context value that the
+// cluster impl picker will write telemetry labels to, and then the stats
+// handler asserts that subsequent HandleRPC calls from the RPC lifecycle
+// contain telemetry labels that it can see.
+func (s) TestTelemetryLabels(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	const xdsServiceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: xdsServiceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+
+	resources.Clusters[0].Metadata = &v3corepb.Metadata{
+		FilterMetadata: map[string]*structpb.Struct{
+			"com.google.csm.telemetry_labels": {
+				Fields: map[string]*structpb.Value{
+					serviceNameKey:      structpb.NewStringValue(serviceNameValue),
+					serviceNamespaceKey: structpb.NewStringValue(serviceNamespaceValue),
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	fsh := &fakeStatsHandler{
+		t: t,
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver), grpc.WithStatsHandler(fsh))
+	if err != nil {
+		t.Fatalf("failed to create a new client to local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
+		t.Fatalf("rpc EmptyCall() failed: %v", err)
+	}
+}
+
+// Tests that telemetry labels for an aggregate cluster hierarchy reflect the
+// active leaf cluster receiving traffic, and correctly switch labels when
+// failing over to a secondary leaf cluster.
+func (s) TestTelemetryLabels_AggregateCluster(t *testing.T) {
+	managementServer, nodeID, _, xdsResolver := setup.ManagementServerAndResolver(t)
+
+	const (
+		xdsServiceName = "my-service-client-side-xds"
+		cluster1Name   = "cluster-1"
+		cluster2Name   = "cluster-2"
+
+		csmName1 = "service-1"
+		csmNs1   = "namespace-1"
+		csmName2 = "service-2"
+		csmNs2   = "namespace-2"
+	)
+
+	servers := make([]*stubserver.StubServer, 2)
+	for i := range 2 {
+		servers[i] = stubserver.StartTestService(t, nil)
+		defer servers[i].Stop()
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(xdsServiceName, "route-"+xdsServiceName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig("route-"+xdsServiceName, xdsServiceName, xdsServiceName)},
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+				ClusterName: xdsServiceName,
+				Type:        e2e.ClusterTypeAggregate,
+				ChildNames:  []string{cluster1Name, cluster2Name},
+			}),
+			makeClusterResourceWithMetadata(cluster1Name, csmName1, csmNs1),
+			makeClusterResourceWithMetadata(cluster2Name, csmName2, csmNs2),
+		},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{
+			e2e.DefaultEndpoint(cluster1Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[0].Address))}),
+			e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}),
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", xdsServiceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+	if err != nil {
+		t.Fatalf("failed to create a new client to local test server: %v", err)
+	}
+	defer cc.Close()
+
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Make RPC to primary cluster and verify primary telemetry labels.
+	peer := &peer.Peer{}
+	var gotLabels map[string]string
+	callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+		gotLabels = l
+	})
+	if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), servers[0].Address; got != want {
+		t.Fatalf("EmptyCall() routed to %q, want %q", got, want)
+	}
+
+	wantLabels := map[string]string{
+		localityKey:       localityValue,
+		backendServiceKey: cluster1Name,
+	}
+	if diff := cmp.Diff(gotLabels, wantLabels); diff != "" {
+		t.Fatalf("Telemetry labels for primary cluster (-got +want): %v", diff)
+	}
+
+	// Trigger failover to secondary cluster by clearing primary endpoints.
+	resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{
+		e2e.DefaultEndpoint(cluster1Name, "localhost", nil),
+		e2e.DefaultEndpoint(cluster2Name, "localhost", []uint32{uint32(testutils.ParsePort(t, servers[1].Address))}),
+	}
+	if err := managementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Make RPCs until traffic switches to secondary cluster and capture
+	// secondary labels.
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		callCtx := telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+			gotLabels = l
+		})
+		if _, err := client.EmptyCall(callCtx, &testpb.Empty{}, grpc.Peer(peer)); err == nil && peer.Addr.String() == servers[1].Address {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout waiting for RPCs to switch to secondary cluster %q", servers[1].Address)
+	}
+
+	wantLabels = map[string]string{
+		localityKey:       localityValue,
+		backendServiceKey: cluster2Name,
+	}
+	if diff := cmp.Diff(gotLabels, wantLabels); diff != "" {
+		t.Fatalf("Telemetry labels after failover (-got +want): %v", diff)
+	}
+}
+
+type fakeStatsHandler struct {
+	labels map[string]string
+
+	t *testing.T
+}
+
+func (fsh *fakeStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+func (fsh *fakeStatsHandler) HandleConn(context.Context, stats.ConnStats) {}
+
+func (fsh *fakeStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context {
+	fsh.labels = make(map[string]string)
+	ctx = telemetry.NewContextWithLabelCallback(ctx, func(l map[string]string) {
+		for k, v := range l {
+			fsh.labels[k] = v
+		}
+	})
+	return ctx
+}
+
+func (fsh *fakeStatsHandler) HandleRPC(_ context.Context, rs stats.RPCStats) {
+	switch rs.(type) {
+	// stats.Begin is called before the picker runs, so it won't have telemetry
+	// labels.
+	// The following three stats callouts trigger OpenTelemetry metrics and are
+	// guaranteed to run after the picker has selected a subchannel. Therefore,
+	// they should have access to the desired telemetry labels.
+	case *stats.OutPayload, *stats.InPayload, *stats.End:
+		want := map[string]string{
+			serviceNameKeyCSM:      serviceNameValue,
+			serviceNamespaceKeyCSM: serviceNamespaceValue,
+			localityKey:            localityValue,
+			backendServiceKey:      backendServiceValue,
+		}
+		if diff := cmp.Diff(fsh.labels, want); diff != "" {
+			fsh.t.Fatalf("fsh.labels (-got +want): %v", diff)
+		}
+	default:
+		// Nothing to assert for the other stats.Handler callouts.
+
+	}
+}
+
+func makeClusterResourceWithMetadata(clusterName, serviceName, serviceNamespace string) *v3clusterpb.Cluster {
+	cluster := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{
+		ClusterName: clusterName,
+		Type:        e2e.ClusterTypeEDS,
+	})
+	cluster.Metadata = &v3corepb.Metadata{
+		FilterMetadata: map[string]*structpb.Struct{
+			"com.google.csm.telemetry_labels": {
+				Fields: map[string]*structpb.Value{
+					serviceNameKey:      structpb.NewStringValue(serviceName),
+					serviceNamespaceKey: structpb.NewStringValue(serviceNamespace),
+				},
+			},
+		},
+	}
+	return cluster
+}