snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/xdsclient/tests/ads_stream_ack_nack_test.go

@@ -0,0 +1,512 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// Creates an xDS client with the given bootstrap contents.
+func createXDSClient(t *testing.T, bootstrapContents []byte) xdsclient.XDSClient {
+	t.Helper()
+
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	t.Cleanup(close)
+	return client
+}
+
+// Tests simple ACK and NACK scenarios on the ADS stream:
+//  1. When a good response is received, i.e. once that is expected to be ACKed,
+//     the test verifies that an ACK is sent matching the version and nonce from
+//     the response.
+//  2. When a subsequent bad response is received, i.e. once is expected to be
+//     NACKed, the test verifies that a NACK is sent matching the previously
+//     ACKed version and current nonce from the response.
+//  3. When a subsequent good response is received, the test verifies that an
+//     ACK is sent matching the version and nonce from the current response.
+func (s) TestADS_ACK_NACK_Simple(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannel()
+	streamResponseCh := testutils.NewChannel()
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	client := createXDSClient(t, bc)
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "gRPC Go",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for a discovery response from the server")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantReq.VersionInfo = gotResp.GetVersionInfo()
+	wantReq.ResponseNonce = gotResp.GetNonce()
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantUpdate := listenerUpdateErrTuple{
+		update: &xdsresource.ListenerUpdate{
+			APIListener: &xdsresource.HTTPConnectionManagerConfig{
+				RouteConfigName: routeConfigName,
+				HTTPFilters:     []xdsresource.HTTPFilter{{Name: "router"}},
+			},
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the management server with a listener resource that contains an
+	// empty HTTP connection manager within the apiListener, which will cause
+	// the resource to be NACKed.
+	badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener)
+	badListener.ApiListener.ApiListener = nil
+	mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{badListener},
+		SkipValidation: true,
+	})
+
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for a discovery response from the server")
+	}
+	gotResp = r.(*v3discoverypb.DiscoveryResponse)
+
+	wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
+	if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the NACK contains the appropriate version, nonce and error.
+	// We expect the version to not change as this is a NACK.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for NACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
+		t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
+	}
+	if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
+		t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
+	}
+
+	// Update the management server to send a good resource again.
+	mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	})
+
+	// The envoy-go-control-plane management server keeps resending the same
+	// resource as long as we keep NACK'ing it. So, we will see the bad resource
+	// sent to us a few times here, before receiving the good resource.
+	var lastErr error
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr)
+		}
+
+		r, err = streamResponseCh.Receive(ctx)
+		if err != nil {
+			t.Fatal("Timeout when waiting for a discovery response from the server")
+		}
+		gotResp = r.(*v3discoverypb.DiscoveryResponse)
+
+		// Verify that the ACK contains the appropriate version and nonce.
+		r, err = streamRequestCh.Receive(ctx)
+		if err != nil {
+			t.Fatal("Timeout when waiting for ACK")
+		}
+		gotReq = r.(*v3discoverypb.DiscoveryRequest)
+		wantReq.VersionInfo = gotResp.GetVersionInfo()
+		wantReq.ResponseNonce = gotResp.GetNonce()
+		wantReq.ErrorDetail = nil
+		diff := cmp.Diff(gotReq, wantReq, protocmp.Transform())
+		if diff == "" {
+			lastErr = nil
+			break
+		}
+		lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) {
+		if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+			lastErr = err
+			continue
+		}
+		break
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr)
+	}
+}
+
+// Tests the case where the first response is invalid. The test verifies that
+// the NACK contains an empty version string.
+func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannel()
+	streamResponseCh := testutils.NewChannel()
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+	})
+
+	// Create a listener resource on the management server that is expected to
+	// be NACKed by the xDS client.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	listenerResource.ApiListener.ApiListener = nil
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	client := createXDSClient(t, bc)
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "gRPC Go",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the discovery response from client")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the error is propagated to the watcher.
+	var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
+	if err := verifyListenerUpdate(ctx, lw.updateCh, listenerUpdateErrTuple{err: wantNackErr}); err != nil {
+		t.Fatal(err)
+	}
+
+	// NACK should contain the appropriate error, nonce, but empty version.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion {
+		t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion)
+	}
+	if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
+		t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
+	}
+	if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
+		t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
+	}
+}
+
+// Tests the scenario where the xDS client is no longer interested in a
+// resource. The following sequence of events are tested:
+//  1. A resource is requested and a good response is received. The test verifies
+//     that an ACK is sent for this resource.
+//  2. The previously requested resource is no longer requested. The test
+//     verifies that the connection to the management server is closed.
+//  3. The same resource is requested again. The test verifies that a new
+//     request is sent with an empty version string, which corresponds to the
+//     first request on a new connection.
+func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannel()
+	streamResponseCh := testutils.NewChannel()
+	streamCloseCh := testutils.NewChannel()
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			streamCloseCh.SendContext(ctx, struct{}{})
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "gRPC Go",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the discovery response from client")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest)
+	wantACKReq.VersionInfo = gotResp.GetVersionInfo()
+	wantACKReq.ResponseNonce = gotResp.GetNonce()
+	if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantUpdate := listenerUpdateErrTuple{
+		update: &xdsresource.ListenerUpdate{
+			APIListener: &xdsresource.HTTPConnectionManagerConfig{
+				RouteConfigName: routeConfigName,
+				HTTPFilters:     []xdsresource.HTTPFilter{{Name: "router"}},
+			},
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cancel the watch on the listener resource. This should result in the
+	// existing connection to be management server getting closed.
+	ldsCancel()
+	if _, err := streamCloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when expecting existing connection to be closed: %v", err)
+	}
+
+	// There is a race between two events when the last watch on an xdsChannel
+	// is canceled:
+	// - an empty discovery request being sent out
+	// - the ADS stream being closed
+	// To handle this race, we drain the request channel here so that if an
+	// empty discovery request was received, it is pulled out of the request
+	// channel and thereby guaranteeing a clean slate for the next watch
+	// registered below.
+	streamRequestCh.Drain()
+
+	// Register a watch for the same listener resource.
+	lw = newListenerWatcher()
+	ldsCancel = xdsresource.WatchListener(client, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the discovery request is identical to the first one sent out
+	// to the management server.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for discovery request")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/xdsclient/tests/ads_stream_restart_test.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// Tests that an ADS stream is restarted after a connection failure. Also
+// verifies that if there were any watches registered before the connection
+// failed, those resources are re-requested after the stream is restarted.
+func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) {
+	// Create a restartable listener that can simulate a broken ADS stream.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server that uses a couple of channels to inform
+	// the test about the request and response messages being exchanged.
+	streamRequestCh := testutils.NewChannel()
+	streamResponseCh := testutils.NewChannel()
+	streamOpened := testutils.NewChannel()
+	streamClosed := testutils.NewChannel()
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			streamClosed.SendContext(ctx, nil)
+		},
+		OnStreamOpen: func(context.Context, int64, string) error {
+			streamOpened.SendContext(ctx, nil)
+			return nil
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create bootstrap configuration pointing to the above management server.
+	bootstrapContents := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := xdsresource.WatchListener(client, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that an ADS stream is opened and an LDS request with the above
+	// resource name is sent.
+	if _, err = streamOpened.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for ADS stream to open")
+	}
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "gRPC Go",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for a discovery response from the server")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantReq.VersionInfo = gotResp.GetVersionInfo()
+	wantReq.ResponseNonce = gotResp.GetNonce()
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantListenerUpdate := listenerUpdateErrTuple{
+		update: &xdsresource.ListenerUpdate{
+			APIListener: &xdsresource.HTTPConnectionManagerConfig{
+				RouteConfigName: routeConfigName,
+				HTTPFilters:     []xdsresource.HTTPFilter{{Name: "router"}},
+			},
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the restartable listener and wait for the stream to close.
+	lis.Stop()
+	if _, err = streamClosed.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for ADS stream to close")
+	}
+
+	// Restart the restartable listener and wait for the stream to open.
+	lis.Restart()
+	if _, err = streamOpened.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for ADS stream to open")
+	}
+
+	// Verify that the listener resource is requested again.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantReq.ResponseNonce = ""
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+}

internal/xds/xdsclient/tests/authority_test.go

@@ -0,0 +1,375 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	xdstestutils "google.golang.org/grpc/internal/xds/testutils"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+)
+
+const (
+	testAuthority1 = "test-authority1"
+	testAuthority2 = "test-authority2"
+	testAuthority3 = "test-authority3"
+)
+
+var (
+	// These two resources use `testAuthority1`, which contains an empty server
+	// config in the bootstrap file, and therefore will use the default
+	// management server.
+	authorityTestResourceName11 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"1", nil)
+	authorityTestResourceName12 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"2", nil)
+	// This resource uses `testAuthority2`, which contains an empty server
+	// config in the bootstrap file, and therefore will use the default
+	// management server.
+	authorityTestResourceName2 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority2, cdsName+"3", nil)
+	// This resource uses `testAuthority3`, which contains a non-empty server
+	// config in the bootstrap file, and therefore will use the non-default
+	// management server.
+	authorityTestResourceName3 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority3, cdsName+"3", nil)
+)
+
+// setupForAuthorityTests spins up two management servers, one to act as the
+// default and the other to act as the non-default. It also generates a
+// bootstrap configuration with three authorities (the first two pointing to the
+// default and the third one pointing to the non-default).
+//
+// Returns two listeners used by the default and non-default management servers
+// respectively, and the xDS client and its close function.
+func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, xdsclient.XDSClient, func()) {
+	// Create listener wrappers which notify on to a channel whenever a new
+	// connection is accepted. We use this to track the number of transports
+	// used by the xDS client.
+	lisDefault := testutils.NewListenerWrapper(t, nil)
+	lisNonDefault := testutils.NewListenerWrapper(t, nil)
+
+	// Start a management server to act as the default authority.
+	defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
+
+	// Start a management server to act as the non-default authority.
+	nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
+
+	// Create a bootstrap configuration with two non-default authorities which
+	// have empty server configs, and therefore end up using the default server
+	// config, which points to the above management server.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, defaultAuthorityServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			testAuthority1: []byte(`{}`),
+			testAuthority2: []byte(`{}`),
+			testAuthority3: []byte(fmt.Sprintf(`{
+				"xds_servers": [{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]}`, nonDefaultAuthorityServer.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name:               t.Name(),
+		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create an xDS client: %v", err)
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(authorityTestResourceName11, edsName, e2e.SecurityLevelNone),
+			e2e.DefaultCluster(authorityTestResourceName12, edsName, e2e.SecurityLevelNone),
+			e2e.DefaultCluster(authorityTestResourceName2, edsName, e2e.SecurityLevelNone),
+			e2e.DefaultCluster(authorityTestResourceName3, edsName, e2e.SecurityLevelNone),
+		},
+		SkipValidation: true,
+	}
+	if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+	return lisDefault, lisNonDefault, client, close
+}
+
+// Tests the xdsChannel sharing logic among authorities. The test verifies the
+// following scenarios:
+//   - A watch for a resource name with an authority matching an existing watch
+//     should not result in a new transport being created.
+//   - A watch for a resource name with different authority name but same
+//     authority config as an existing watch should not result in a new transport
+//     being created.
+func (s) TestAuthority_XDSChannelSharing(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	lis, _, client, close := setupForAuthorityTests(ctx, t)
+	defer close()
+
+	// Verify that no connection is established to the management server at this
+	// point. A transport is created only when a resource (which belongs to that
+	// authority) is requested.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Request the first resource. Verify that a new transport is created.
+	watcher := noopClusterWatcher{}
+	cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
+	defer cdsCancel1()
+	if _, err := lis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+
+	// Request the second resource. Verify that no new transport is created.
+	cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
+	defer cdsCancel2()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Request the third resource. Verify that no new transport is created.
+	cdsCancel3 := xdsresource.WatchCluster(client, authorityTestResourceName2, watcher)
+	defer cdsCancel3()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+}
+
+// Test the xdsChannel close logic. The test verifies that the xDS client
+// closes an xdsChannel immediately after the last watch is canceled.
+func (s) TestAuthority_XDSChannelClose(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	lis, _, client, close := setupForAuthorityTests(ctx, t)
+	defer close()
+
+	// Request the first resource. Verify that a new transport is created.
+	watcher := noopClusterWatcher{}
+	cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
+	val, err := lis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+	conn := val.(*testutils.ConnWrapper)
+
+	// Request the second resource. Verify that no new transport is created.
+	cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Cancel both watches, and verify that the connection to the management
+	// server is closed.
+	cdsCancel1()
+	cdsCancel2()
+	if _, err := conn.CloseCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for connection to management server to be closed")
+	}
+}
+
+// Tests the scenario where the primary management server is unavailable at
+// startup and the xDS client falls back to the secondary.  The test verifies
+// that the resource watcher is not notifified of the connectivity failure until
+// all servers have failed.
+func (s) TestAuthority_Fallback(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create primary and secondary management servers with restartable
+	// listeners.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	primaryLis := testutils.NewRestartableListener(l)
+	primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	l, err = testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
+	}
+	secondaryLis := testutils.NewRestartableListener(l)
+	secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
+
+	// Create bootstrap configuration with the above primary and fallback
+	// management servers, and an xDS client with that configuration.
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`
+		[
+			{
+				"server_uri": %q,
+				"channel_creds": [{"type": "insecure"}]
+			},
+			{
+				"server_uri": %q,
+				"channel_creds": [{"type": "insecure"}]
+			}
+		]`, primaryMgmtServer.Address, secondaryMgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	xdsC, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{Name: t.Name()})
+	if err != nil {
+		t.Fatalf("Failed to create an xDS client: %v", err)
+	}
+	defer close()
+
+	const clusterName = "cluster"
+	const edsPrimaryName = "eds-primary"
+	const edsSecondaryName = "eds-secondary"
+
+	// Create a Cluster resource on the primary.
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterName, edsPrimaryName, e2e.SecurityLevelNone),
+		},
+		SkipValidation: true,
+	}
+	if err := primaryMgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Create a Cluster resource on the secondary .
+	resources = e2e.UpdateOptions{
+		NodeID: nodeID,
+		Clusters: []*v3clusterpb.Cluster{
+			e2e.DefaultCluster(clusterName, edsSecondaryName, e2e.SecurityLevelNone),
+		},
+		SkipValidation: true,
+	}
+	if err := secondaryMgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Stop the primary.
+	primaryLis.Close()
+
+	// Register a watch.
+	watcher := newClusterWatcherV2()
+	cdsCancel := xdsresource.WatchCluster(xdsC, clusterName, watcher)
+	defer cdsCancel()
+
+	// Ensure that the connectivity error callback is not called.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if v, err := watcher.ambientErrCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("Error callback on the watcher with error:  %v", v.(error))
+	}
+
+	// Ensure that the resource update callback is invoked.
+	v, err := watcher.updateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Error when waiting for a resource update callback:  %v", err)
+	}
+	gotUpdate := v.(*xdsresource.ClusterUpdate)
+	wantUpdate := xdsresource.ClusterUpdate{
+		ClusterName:    clusterName,
+		EDSServiceName: edsSecondaryName,
+	}
+	cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy", "TelemetryLabels")}
+	if diff := cmp.Diff(wantUpdate, *gotUpdate, cmpOpts...); diff != "" {
+		t.Fatalf("Diff in the cluster resource update: (-want, got):\n%s", diff)
+	}
+
+	// Stop the secondary.
+	secondaryLis.Close()
+
+	// Ensure that the connectivity error callback is called.
+	if _, err := watcher.ambientErrCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for error callback on the watcher")
+	}
+}
+
+// TODO: Get rid of the clusterWatcher type in cds_watchers_test.go and use this
+// one instead. Also, rename this to clusterWatcher as part of that refactor.
+type clusterWatcherV2 struct {
+	updateCh      *testutils.Channel // Messages of type xdsresource.ClusterUpdate
+	ambientErrCh  *testutils.Channel // Messages of type ambient error
+	resourceErrCh *testutils.Channel // Messages of type resource error
+}
+
+func newClusterWatcherV2() *clusterWatcherV2 {
+	return &clusterWatcherV2{
+		updateCh:      testutils.NewChannel(),
+		ambientErrCh:  testutils.NewChannel(),
+		resourceErrCh: testutils.NewChannel(),
+	}
+}
+
+func (cw *clusterWatcherV2) ResourceChanged(update *xdsresource.ClusterUpdate, onDone func()) {
+	cw.updateCh.Send(update)
+	onDone()
+}
+
+func (cw *clusterWatcherV2) AmbientError(err error, onDone func()) {
+	// When used with a go-control-plane management server that continuously
+	// resends resources which are NACKed by the xDS client, using a `Replace()`
+	// here simplifies tests that want access to the most recently received
+	// error.
+	cw.ambientErrCh.Replace(err)
+	onDone()
+}
+
+func (cw *clusterWatcherV2) ResourceError(err error, onDone func()) {
+	// When used with a go-control-plane management server that continuously
+	// resends resources which are NACKed by the xDS client, using a `Replace()`
+	// here simplifies tests that want access to the most recently received
+	// error.
+	cw.resourceErrCh.Replace(err)
+	onDone()
+}

internal/xds/xdsclient/tests/client_custom_dialopts_test.go

@@ -0,0 +1,165 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	internalbootstrap "google.golang.org/grpc/internal/xds/bootstrap"
+	xci "google.golang.org/grpc/internal/xds/xdsclient/internal"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/xds/bootstrap"
+)
+
+// nopDialOption is a no-op grpc.DialOption with a name.
+type nopDialOption struct {
+	grpc.EmptyDialOption
+	name string
+}
+
+// testCredsBundle implements `credentials.Bundle` and `extraDialOptions`.
+type testCredsBundle struct {
+	credentials.Bundle
+	testDialOptNames []string
+}
+
+func (t *testCredsBundle) DialOptions() []grpc.DialOption {
+	var opts []grpc.DialOption
+	for _, name := range t.testDialOptNames {
+		opts = append(opts, &nopDialOption{name: name})
+	}
+	return opts
+}
+
+type testCredsBuilder struct {
+	testDialOptNames []string
+}
+
+func (t *testCredsBuilder) Build(json.RawMessage) (credentials.Bundle, func(), error) {
+	return &testCredsBundle{
+		Bundle:           insecure.NewBundle(),
+		testDialOptNames: t.testDialOptNames,
+	}, func() {}, nil
+}
+
+func (t *testCredsBuilder) Name() string {
+	return "test_dialer_creds"
+}
+
+func (s) TestClientCustomDialOptsFromCredentialsBundle(t *testing.T) {
+	// Create and register the credentials bundle builder.
+	credsBuilder := &testCredsBuilder{
+		testDialOptNames: []string{"opt1", "opt2", "opt3"},
+	}
+	bootstrap.RegisterChannelCredentials(credsBuilder)
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+	bc, err := internalbootstrap.NewContentsForTesting(internalbootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{
+				"type": %q,
+				"config": {"mgmt_server_address": %q}
+			}]
+		}]`, mgmtServer.Address, credsBuilder.Name(), mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS resolver with the above bootstrap configuration.
+	var resolverBuilder resolver.Builder
+	if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil {
+		resolverBuilder, err = newResolver.(func([]byte) (resolver.Builder, error))(bc)
+		if err != nil {
+			t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+		}
+	}
+
+	// Spin up a test backend.
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure client side xDS resources on the management server.
+	const serviceName = "my-service-client-side-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Intercept a grpc.NewClient call from the xds client to validate DialOptions.
+	xci.GRPCNewClient = func(target string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) {
+		got := map[string]int{}
+		for _, opt := range opts {
+			if mo, ok := opt.(*nopDialOption); ok {
+				got[mo.name]++
+			}
+		}
+		want := map[string]int{}
+		for _, name := range credsBuilder.testDialOptNames {
+			want[name]++
+		}
+		if !cmp.Equal(got, want) {
+			t.Errorf("grpc.NewClient() was called with unexpected DialOptions: got %v, want %v", got, want)
+		}
+		return grpc.NewClient(target, opts...)
+	}
+	defer func() { xci.GRPCNewClient = grpc.NewClient }()
+
+	// Create a ClientConn and make a successful RPC. The insecure transport
+	// credentials passed into the gRPC.NewClient is the credentials for the
+	// data plane communication with the test backend.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolverBuilder))
+	if err != nil {
+		t.Fatalf("Failed to dial local test server: %v", err)
+	}
+
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	cc.Close()
+}

internal/xds/xdsclient/tests/dump_test.go

@@ -0,0 +1,575 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+	"github.com/envoyproxy/go-control-plane/pkg/wellknown"
+)
+
+func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig {
+	return &v3statuspb.ClientConfig_GenericXdsConfig{
+		TypeUrl:      typeURL,
+		Name:         name,
+		VersionInfo:  version,
+		ClientStatus: status,
+		XdsConfig:    config,
+		ErrorState:   failure,
+	}
+}
+
+func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, pool *xdsclient.Pool) error {
+	cmpOpts := cmp.Options{
+		protocmp.Transform(),
+		protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"),
+		protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"),
+	}
+
+	var lastErr error
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		got := pool.DumpResources()
+		// Sort the client configs based on the `client_scope` field.
+		slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int {
+			return strings.Compare(a.ClientScope, b.ClientScope)
+		})
+		// Sort the resource configs based on the type_url and name fields.
+		for _, cc := range got.GetConfig() {
+			slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int {
+				if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 {
+					return strings.Compare(a.Name, b.Name)
+				}
+				return strings.Compare(a.TypeUrl, b.TypeUrl)
+			})
+		}
+		diff := cmp.Diff(want, got, cmpOpts)
+		if diff == "" {
+			return nil
+		}
+		lastErr = fmt.Errorf("received unexpected resource dump, diff (-want, +got):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want))
+	}
+	return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr)
+}
+
+// Tests the scenario where there are multiple xDS clients talking to the same
+// management server, and requesting the same set of resources. Verifies that
+// under all circumstances, both xDS clients receive the same configuration from
+// the server.
+func (s) TestDumpResources_ManyToOne(t *testing.T) {
+	// Initialize the xDS resources to be used in this test.
+	ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"}
+	rdsTargets := []string{"route-config-0", "route-config-1"}
+	cdsTargets := []string{"cluster-0", "cluster-1"}
+	edsTargets := []string{"endpoints-0", "endpoints-1"}
+	listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
+	listenerAnys := make([]*anypb.Any, len(ldsTargets))
+	for i := range ldsTargets {
+		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
+		listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
+	}
+	routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets))
+	routeAnys := make([]*anypb.Any, len(rdsTargets))
+	for i := range rdsTargets {
+		routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
+		routeAnys[i] = testutils.MarshalAny(t, routes[i])
+	}
+	clusters := make([]*v3clusterpb.Cluster, len(cdsTargets))
+	clusterAnys := make([]*anypb.Any, len(cdsTargets))
+	for i := range cdsTargets {
+		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
+		clusterAnys[i] = testutils.MarshalAny(t, clusters[i])
+	}
+	endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets))
+	endpointAnys := make([]*anypb.Any, len(edsTargets))
+	ips := []string{"0.0.0.0", "1.1.1.1"}
+	ports := []uint32{123, 456}
+	for i := range edsTargets {
+		endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1])
+		endpointAnys[i] = testutils.MarshalAny(t, endpoints[i])
+	}
+
+	// Spin up an xDS management server on a local port.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	// Create two xDS clients with the above bootstrap contents.
+	client1Name := t.Name() + "-1"
+	client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: client1Name,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close1()
+	client2Name := t.Name() + "-2"
+	client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: client2Name,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close2()
+
+	// Dump resources and expect empty configs.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantNode := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "gRPC Go",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+	}
+	wantResp := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:        wantNode,
+				ClientScope: client1Name,
+			},
+			{
+				Node:        wantNode,
+				ClientScope: client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register watches, dump resources and expect configs in requested state.
+	for _, xdsC := range []xdsclient.XDSClient{client1, client2} {
+		for _, target := range ldsTargets {
+			xdsresource.WatchListener(xdsC, target, noopListenerWatcher{})
+		}
+		for _, target := range rdsTargets {
+			xdsresource.WatchRouteConfig(xdsC, target, noopRouteConfigWatcher{})
+		}
+		for _, target := range cdsTargets {
+			xdsresource.WatchCluster(xdsC, target, noopClusterWatcher{})
+		}
+		for _, target := range edsTargets {
+			xdsresource.WatchEndpoints(xdsC, target, noopEndpointsWatcher{})
+		}
+	}
+	wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the resources on the management server.
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners,
+		Routes:    routes,
+		Clusters:  clusters,
+		Endpoints: endpoints,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Dump resources and expect ACK configs.
+	wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the first resource of each type in the management server to a
+	// value which is expected to be NACK'ed by the xDS client.
+	listeners[0] = func() *v3listenerpb.Listener {
+		hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{
+			HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})},
+		})
+		return &v3listenerpb.Listener{
+			Name:        ldsTargets[0],
+			ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
+			FilterChains: []*v3listenerpb.FilterChain{{
+				Name: "filter-chain-name",
+				Filters: []*v3listenerpb.Filter{{
+					Name:       wellknown.HTTPConnectionManager,
+					ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm},
+				}},
+			}},
+		}
+	}()
+	routes[0].VirtualHosts = []*v3routepb.VirtualHost{{Routes: []*v3routepb.Route{{}}}}
+	clusters[0].ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}
+	endpoints[0].Endpoints = []*v3endpointpb.LocalityLbEndpoints{{}}
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		Routes:         routes,
+		Clusters:       clusters,
+		Endpoints:      endpoints,
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, clusterAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, endpointAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, routeAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where there are multiple xDS client talking to different
+// management server, and requesting different set of resources.
+func (s) TestDumpResources_ManyToMany(t *testing.T) {
+	// Initialize the xDS resources to be used in this test:
+	// - The first xDS client watches old style resource names, and thereby
+	//   requests these resources from the top-level xDS server.
+	// - The second xDS client watches new style resource names with a non-empty
+	//   authority, and thereby requests these resources from the server
+	//   configuration for that authority.
+	authority := strings.Join(strings.Split(t.Name(), "/"), "")
+	ldsTargets := []string{
+		"lds.target.good:0000",
+		fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority),
+	}
+	rdsTargets := []string{
+		"route-config-0",
+		fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority),
+	}
+	cdsTargets := []string{
+		"cluster-0",
+		fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/cluster-1", authority),
+	}
+	edsTargets := []string{
+		"endpoints-0",
+		fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/endpoints-1", authority),
+	}
+	listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
+	listenerAnys := make([]*anypb.Any, len(ldsTargets))
+	for i := range ldsTargets {
+		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
+		listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
+	}
+	routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets))
+	routeAnys := make([]*anypb.Any, len(rdsTargets))
+	for i := range rdsTargets {
+		routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i])
+		routeAnys[i] = testutils.MarshalAny(t, routes[i])
+	}
+	clusters := make([]*v3clusterpb.Cluster, len(cdsTargets))
+	clusterAnys := make([]*anypb.Any, len(cdsTargets))
+	for i := range cdsTargets {
+		clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone)
+		clusterAnys[i] = testutils.MarshalAny(t, clusters[i])
+	}
+	endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets))
+	endpointAnys := make([]*anypb.Any, len(edsTargets))
+	ips := []string{"0.0.0.0", "1.1.1.1"}
+	ports := []uint32{123, 456}
+	for i := range edsTargets {
+		endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1])
+		endpointAnys[i] = testutils.MarshalAny(t, endpoints[i])
+	}
+
+	// Start two management servers.
+	mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// The first of the above management servers will be the top-level xDS
+	// server in the bootstrap configuration, and the second will be the xDS
+	// server corresponding to the test authority.
+	nodeID := uuid.New().String()
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer1.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			authority: []byte(fmt.Sprintf(`{
+				 "xds_servers": [{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]}`, mgmtServer2.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+
+	// Create two xDS clients with the above bootstrap contents.
+	client1Name := t.Name() + "-1"
+	client1, close1, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: client1Name,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close1()
+	client2Name := t.Name() + "-2"
+	client2, close2, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: client2Name,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close2()
+
+	// Check the resource dump before configuring resources on the management server.
+	// Dump resources and expect empty configs.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantNode := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "gRPC Go",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+	}
+	wantResp := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:        wantNode,
+				ClientScope: client1Name,
+			},
+			{
+				Node:        wantNode,
+				ClientScope: client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register watches, the first xDS client watches old style resource names,
+	// while the second xDS client watches new style resource names.
+	xdsresource.WatchListener(client1, ldsTargets[0], noopListenerWatcher{})
+	xdsresource.WatchRouteConfig(client1, rdsTargets[0], noopRouteConfigWatcher{})
+	xdsresource.WatchCluster(client1, cdsTargets[0], noopClusterWatcher{})
+	xdsresource.WatchEndpoints(client1, edsTargets[0], noopEndpointsWatcher{})
+	xdsresource.WatchListener(client2, ldsTargets[1], noopListenerWatcher{})
+	xdsresource.WatchRouteConfig(client2, rdsTargets[1], noopRouteConfigWatcher{})
+	xdsresource.WatchCluster(client2, cdsTargets[1], noopClusterWatcher{})
+	xdsresource.WatchEndpoints(client2, edsTargets[1], noopEndpointsWatcher{})
+
+	// Check the resource dump. Both clients should have all resources in
+	// REQUESTED state.
+	wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure resources on the first management server.
+	if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners[:1],
+		Routes:    routes[:1],
+		Clusters:  clusters[:1],
+		Endpoints: endpoints[:1],
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the resource dump. One client should have resources in ACKED state,
+	// while the other should still have resources in REQUESTED state.
+	wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[0], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure resources on the second management server.
+	if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: listeners[1:],
+		Routes:    routes[1:],
+		Clusters:  clusters[1:],
+		Endpoints: endpoints[1:],
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the resource dump. Both clients should have appropriate resources
+	// in REQUESTED state.
+	wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+				ClientScope:       client1Name,
+			},
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+				ClientScope:       client2Name,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, pool); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/xdsclient/tests/fallback/fallback_test.go

@@ -0,0 +1,931 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_fallback_test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/grpc/peer"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+
+	_ "google.golang.org/grpc/internal/xds/httpfilter/router" // Register the router filter.
+	_ "google.golang.org/grpc/xds"                            // To ensure internal.NewXDSResolverWithConfigForTesting is set.
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// Give the fallback tests additional time to complete because they need to
+// first identify failed connections before establishing new ones.
+const defaultFallbackTestTimeout = 20 * time.Second
+const defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen.
+
+func waitForRPCsToReachBackend(ctx context.Context, client testgrpc.TestServiceClient, backend string) error {
+	var lastErr error
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		var peer peer.Peer
+		if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
+			lastErr = err
+			continue
+		}
+		// Veirfy the peer when the RPC succeeds.
+		if peer.Addr.String() == backend {
+			break
+		}
+	}
+	if ctx.Err() != nil {
+		return fmt.Errorf("timeout when waiting for RPCs to reach expected backend. Last error: %v", lastErr)
+	}
+	return nil
+}
+
+// Tests fallback on startup where the xDS client is unable to establish a
+// connection to the primary server. The test verifies that the xDS client falls
+// back to the secondary server, and when the primary comes back up, it reverts
+// to it. The test also verifies that when all requested resources are cached
+// from the primary, fallback is not triggered when the connection goes down.
+func (s) TestFallback_OnStartup(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start two test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	const serviceName = "my-service-fallback-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, backend1.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := primaryManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	fallbackRouteConfigName := "fallback-route-" + serviceName
+	fallbackClusterName := "fallback-cluster-" + serviceName
+	fallbackEndpointsName := "fallback-endpoints-" + serviceName
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Shut both management servers down before starting the gRPC client to
+	// trigger fallback on startup.
+	primaryLis.Stop()
+	fallbackLis.Stop()
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	resolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that a connection is attempted to the primary.
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Ensure that a connection is attempted to the fallback.
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Make an RPC with a shortish deadline and expect it to fail.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	client := testgrpc.NewTestServiceClient(cc)
+	if _, err := client.EmptyCall(sCtx, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.DeadlineExceeded {
+		t.Fatalf("EmptyCall() = %v, want DeadlineExceeded", err)
+	}
+
+	// Start the fallback server. Ensure that an RPC can succeed, and that it
+	// reaches backend2.
+	fallbackLis.Restart()
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Start the primary server. It can take a while before the xDS client
+	// notices this, since the ADS stream implementation uses a backoff before
+	// retrying the stream.
+	primaryLis.Restart()
+
+	// Wait for the connection to the secondary to be closed and ensure that an
+	// RPC can succeed, and that it reaches backend1.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when retrieving the most recent connection to the fallback management server: %v", err)
+	}
+	conn := c.(*testutils.ConnWrapper)
+	if _, err := conn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the primary servers. Since all xDS resources were received from the
+	// primary (and RPCs were succeeding to the clusters returned by the
+	// primary), we will not trigger fallback.
+	primaryLis.Stop()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err == nil {
+		t.Fatalf("Fallback attempted when not expected to. There are no uncached resources from the primary server at this point.")
+	}
+
+	// Ensure that RPCs still succeed, and that they use the configuration
+	// received from the primary.
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests fallback when the primary management server fails during an update.
+func (s) TestFallback_MidUpdate(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// This boolean helps with triggering fallback mid update. When this boolean
+	// is set and the below defined cluster resource is requested, the primary
+	// management server shuts down the connection, forcing the client to
+	// fallback to the secondary server.
+	var closeConnOnMidUpdateClusterResource atomic.Bool
+	const (
+		serviceName              = "my-service-fallback-xds"
+		routeConfigName          = "route-" + serviceName
+		clusterName              = "cluster-" + serviceName
+		endpointsName            = "endpoints-" + serviceName
+		midUpdateRouteConfigName = "mid-update-route-" + serviceName
+		midUpdateClusterName     = "mid-update-cluster-" + serviceName
+		midUpdateEndpointsName   = "mid-update-endpoints-" + serviceName
+	)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: primaryLis,
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if closeConnOnMidUpdateClusterResource.Load() == false {
+				return nil
+			}
+			if req.GetTypeUrl() != version.V3ClusterURL {
+				return nil
+			}
+			for _, name := range req.GetResourceNames() {
+				if name == midUpdateClusterName {
+					primaryLis.Stop()
+					return fmt.Errorf("closing ADS stream because %q resource was requested", midUpdateClusterName)
+				}
+			}
+			return nil
+		},
+		AllowResourceSubset: true,
+	})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start three test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+	backend3 := stubserver.StartTestService(t, nil)
+	defer backend3.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	primaryResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, "localhost", []uint32{testutils.ParsePort(t, backend1.Address)})},
+	}
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	const (
+		fallbackRouteConfigName = "fallback-route-" + serviceName
+		fallbackClusterName     = "fallback-cluster-" + serviceName
+		fallbackEndpointsName   = "fallback-endpoints-" + serviceName
+	)
+	fallbackResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	resolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that RPCs reach the cluster specified by the primary server and
+	// that no connection is attempted to the fallback server.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := fallbackWrappedLis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("Connection attempt made to fallback server when none expected: %v", err)
+	}
+
+	// Instruct the primary server to close the connection if below defined
+	// cluster resource is requested.
+	closeConnOnMidUpdateClusterResource.Store(true)
+
+	// Update the listener resource on the primary server to point to a new
+	// route configuration that points to a new cluster that points to a new
+	// endpoints resource that contains backend3.
+	primaryResources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, midUpdateRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(midUpdateRouteConfigName, serviceName, midUpdateClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(midUpdateClusterName, midUpdateEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(midUpdateEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend3.Address)})},
+	}
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that a connection is attempted to the fallback (because both
+	// conditions mentioned for fallback in A71 are satisfied: connectivity
+	// failure and a watcher for an uncached resource), and that RPCs are
+	// routed to the cluster returned by the fallback server.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the fallback management server: %v", err)
+	}
+	fallbackConn := c.(*testutils.ConnWrapper)
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Set the primary management server to not close the connection anymore if
+	// the mid-update cluster resource is requested, and get it to start serving
+	// again.
+	closeConnOnMidUpdateClusterResource.Store(false)
+	primaryLis.Restart()
+
+	// A new snapshot, with the same resources, is pushed to the management
+	// server to get it to respond for already requested resource names.
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that RPCs reach the backend pointed to by the new cluster.
+	if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to the secondary to be closed since we have
+	// reverted back to the primary.
+	if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+}
+
+// Tests fallback when the primary management server fails during startup.
+func (s) TestFallback_MidStartup(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners and can also get notified when the listener
+	// receives a connection request.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+	fallbackWrappedLis := testutils.NewListenerWrapper(t, nil)
+	fallbackLis := testutils.NewRestartableListener(fallbackWrappedLis)
+
+	// This boolean helps with triggering fallback during startup. When this
+	// boolean is set and a cluster resource is requested, the primary
+	// management server shuts down the connection, forcing the client to
+	// fallback to the secondary server.
+	var closeConnOnClusterResource atomic.Bool
+	closeConnOnClusterResource.Store(true)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: primaryLis,
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if closeConnOnClusterResource.Load() == false {
+				return nil
+			}
+			if req.GetTypeUrl() != version.V3ClusterURL {
+				return nil
+			}
+			primaryLis.Stop()
+			return fmt.Errorf("closing ADS stream because cluster resource was requested")
+		},
+		AllowResourceSubset: true,
+	})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start two test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	const serviceName = "my-service-fallback-xds"
+	primaryResources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, backend1.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	fallbackRouteConfigName := "fallback-route-" + serviceName
+	fallbackClusterName := "fallback-cluster-" + serviceName
+	fallbackEndpointsName := "fallback-endpoints-" + serviceName
+	fallbackResources := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, fallbackResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	resolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+
+	// Ensure that a connection is attempted to the primary.
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the primary management server: %v", err)
+	}
+
+	// Ensure that a connection is attempted to the fallback.
+	c, err := fallbackWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Failure when waiting for a connection to be opened to the secondary management server: %v", err)
+	}
+	fallbackConn := c.(*testutils.ConnWrapper)
+
+	// Ensure that RPCs are routed to the cluster returned by the fallback
+	// management server.
+	client := testgrpc.NewTestServiceClient(cc)
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Get the primary management server to no longer close the connection when
+	// the cluster resource is requested.
+	closeConnOnClusterResource.Store(false)
+	primaryLis.Restart()
+
+	// A new snapshot, with the same resources, is pushed to the management
+	// server to get it to respond for already requested resource names.
+	if err := primaryManagementServer.Update(ctx, primaryResources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure that RPCs are routed to the cluster returned by the primary
+	// management server.
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the connection to the secondary to be closed since we have
+	// reverted back to the primary.
+	if _, err := fallbackConn.CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Connection to fallback server not closed once primary becomes ready: %v", err)
+	}
+}
+
+// Tests that RPCs succeed at startup when the primary management server is
+// down, but the secondary is available.
+func (s) TestFallback_OnStartup_RPCSuccess(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create two listeners for the two management servers. The test can
+	// start/stop these listeners.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create listener: %v", err)
+	}
+	primaryLis := testutils.NewRestartableListener(l)
+	l, err = testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create listener: %v", err)
+	}
+	fallbackLis := testutils.NewRestartableListener(l)
+
+	// Start two management servers, primary and fallback, with the above
+	// listeners.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	fallbackManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: fallbackLis})
+
+	// Start two test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	defer backend2.Stop()
+
+	// Configure xDS resource on the primary management server, with a cluster
+	// resource that contains an endpoint for backend1.
+	nodeID := uuid.New().String()
+	const serviceName = "my-service-fallback-xds"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, backend1.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	if err := primaryManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure xDS resource on the secondary management server, with a cluster
+	// resource that contains an endpoint for backend2. Only the listener
+	// resource has the same name on both servers.
+	fallbackRouteConfigName := "fallback-route-" + serviceName
+	fallbackClusterName := "fallback-cluster-" + serviceName
+	fallbackEndpointsName := "fallback-endpoints-" + serviceName
+	resources = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, fallbackRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(fallbackRouteConfigName, serviceName, fallbackClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(fallbackClusterName, fallbackEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(fallbackEndpointsName, "localhost", []uint32{testutils.ParsePort(t, backend2.Address)})},
+	}
+	if err := fallbackManagementServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Shutdown the primary management server before starting the gRPC client to
+	// trigger fallback on startup.
+	primaryLis.Stop()
+
+	// Generate bootstrap configuration with the above two servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		},
+		{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, primaryManagementServer.Address, fallbackManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	resolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+
+	// Make an RPC (without the `wait_for_ready` call option) and expect it to
+	// succeed since the fallback management server is up and running.
+	client := testgrpc.NewTestServiceClient(cc)
+	var peer peer.Peer
+	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil {
+		t.Fatalf("EmptyCall() failed: %v", err)
+	}
+	if got, want := peer.Addr.String(), backend2.Address; got != want {
+		t.Fatalf("Unexpected peer address: got %q, want %q", got, want)
+	}
+
+	// Start the primary server. It can take a while before the xDS client
+	// notices this, since the ADS stream implementation uses a backoff before
+	// retrying the stream.
+	primaryLis.Restart()
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Test verifies that when the primary management server is unavailable, the
+// system attempts to connect to the first fallback server, and if that is also
+// down, to the second fallback server. It also ensures that the system switches
+// back to the first fallback server once it becomes available again, and
+// eventually returns to the primary server when it comes back online, closing
+// connections to the fallback servers accordingly.
+func (s) TestFallback_ThreeServerPromotion(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTestTimeout)
+	defer cancel()
+
+	// Create three listener wrappers for three management servers.
+	primaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	primaryLis := testutils.NewRestartableListener(primaryWrappedLis)
+
+	secondaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	secondaryLis := testutils.NewRestartableListener(secondaryWrappedLis)
+
+	tertiaryWrappedLis := testutils.NewListenerWrapper(t, nil)
+	tertiaryLis := testutils.NewRestartableListener(tertiaryWrappedLis)
+
+	// Start the three management servers.
+	primaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	secondaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
+	tertiaryManagementServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: tertiaryLis})
+
+	// Start three test service backends.
+	backend1 := stubserver.StartTestService(t, nil)
+	backend1Port := testutils.ParsePort(t, backend1.Address)
+	defer backend1.Stop()
+	backend2 := stubserver.StartTestService(t, nil)
+	backend2Port := testutils.ParsePort(t, backend2.Address)
+	defer backend2.Stop()
+	backend3 := stubserver.StartTestService(t, nil)
+	backend3Port := testutils.ParsePort(t, backend3.Address)
+	defer backend3.Stop()
+
+	nodeID := uuid.New().String()
+	const (
+		serviceName              = "my-service-fallback-xds"
+		primaryRouteConfigName   = "primary-route-" + serviceName
+		secondaryRouteConfigName = "secondary-route-" + serviceName
+		tertiaryRouteConfigName  = "tertiary-route-" + serviceName
+		primaryClusterName       = "primary-cluster-" + serviceName
+		secondaryClusterName     = "secondary-cluster-" + serviceName
+		tertiaryClusterName      = "tertiary-cluster-" + serviceName
+		primaryEndpointsName     = "primary-endpoints-" + serviceName
+		secondaryEndpointsName   = "secondary-endpoints-" + serviceName
+		tertiaryEndpointsName    = "tertiary-endpoints-" + serviceName
+	)
+
+	// Configure partial resources on the primary and secondary
+	// management servers.
+	primaryManagementServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)},
+	})
+	secondaryManagementServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)},
+	})
+
+	// Configure full resources on tertiary management server.
+	updateOpts := e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, tertiaryRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(tertiaryRouteConfigName, serviceName, tertiaryClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(tertiaryClusterName, tertiaryEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(tertiaryEndpointsName, "localhost", []uint32{backend3Port})},
+	}
+	tertiaryManagementServer.Update(ctx, updateOpts)
+
+	// Create bootstrap configuration for all three management servers.
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[
+            {
+                "server_uri": %q,
+                "channel_creds": [{"type": "insecure"}]
+            },
+            {
+                "server_uri": %q,
+                "channel_creds": [{"type": "insecure"}]
+            },
+             {
+                "server_uri": %q,
+                "channel_creds": [{"type": "insecure"}]
+            }
+        ]`, primaryManagementServer.Address, secondaryManagementServer.Address, tertiaryManagementServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap file: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap configuration.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %v", err)
+	}
+	pool := xdsclient.NewPool(config)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	// Get the xDS resolver to use the above xDS client.
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	resolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	// Start a gRPC client that uses the above xDS resolver.
+	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver))
+	if err != nil {
+		t.Fatalf("Failed to create gRPC client: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	client := testgrpc.NewTestServiceClient(cc)
+
+	// Verify that connection attempts were made to primaryWrappedLis and
+	// secondaryWrappedLis, before using tertiaryWrappedLis to make
+	// successful RPCs to backend3.
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for connection to primary: %v", err)
+	}
+
+	// Stop primary, client should connect to secondary.
+	primaryLis.Stop()
+	if _, err := secondaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for connection to secondary after primary stopped: %v", err)
+	}
+
+	// Stop secondary, client should connect to tertiary.
+	secondaryLis.Stop()
+	tertiaryConn, err := tertiaryWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for connection to tertiary after secondary stopped: %v", err)
+	}
+
+	// Tertiary has all resources, RPCs should succeed to backend3.
+	if err := waitForRPCsToReachBackend(ctx, client, backend3.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Secondary1 becomes available, RPCs go to backend2.
+	secondaryLis.Restart()
+	updateOpts = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, secondaryRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(secondaryRouteConfigName, serviceName, secondaryClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(secondaryClusterName, secondaryEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(secondaryEndpointsName, "localhost", []uint32{backend2Port})},
+	}
+	secondaryManagementServer.Update(ctx, updateOpts)
+
+	secondaryConn, err := secondaryWrappedLis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for new connection to secondary: %v", err)
+	}
+	if _, err := tertiaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for connection to the tertiary to be closed after promotion to secondary: %v", err)
+	}
+	if err := waitForRPCsToReachBackend(ctx, client, backend2.Address); err != nil {
+		t.Fatal(err)
+	}
+
+	// Primary becomes available, RPCs go to backend1.
+	primaryLis.Restart()
+	updateOpts = e2e.UpdateOptions{
+		NodeID:    nodeID,
+		Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, primaryRouteConfigName)},
+		Routes:    []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(primaryRouteConfigName, serviceName, primaryClusterName)},
+		Clusters:  []*v3clusterpb.Cluster{e2e.DefaultCluster(primaryClusterName, primaryEndpointsName, e2e.SecurityLevelNone)},
+		Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(primaryEndpointsName, "localhost", []uint32{backend1Port})},
+	}
+	primaryManagementServer.Update(ctx, updateOpts)
+
+	if _, err := primaryWrappedLis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for new connection to primary: %v", err)
+	}
+	if _, err := secondaryConn.(*testutils.ConnWrapper).CloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for connection to the secondary to be closed after promotion to primary: %v", err)
+	}
+	if err := waitForRPCsToReachBackend(ctx, client, backend1.Address); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/xdsclient/tests/federation_watchers_test.go

@@ -0,0 +1,324 @@
+/*
+ *
+ * Copyright 2021 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/grpc/resolver"
+
+	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+)
+
+const testNonDefaultAuthority = "non-default-authority"
+
+// setupForFederationWatchersTest spins up two management servers, one for the
+// default (empty) authority and another for a non-default authority.
+//
+// Returns the management server associated with the non-default authority, the
+// nodeID to use, and the xDS client.
+func setupForFederationWatchersTest(t *testing.T) (*e2e.ManagementServer, string, xdsclient.XDSClient) {
+	// Start a management server as the default authority.
+	serverDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Start another management server as the other authority.
+	serverNonDefaultAuthority := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, serverDefaultAuthority.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			testNonDefaultAuthority: []byte(fmt.Sprintf(`{
+				"xds_servers": [{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]}`, serverNonDefaultAuthority.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bootstrapContents)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bootstrapContents), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	t.Cleanup(close)
+	return serverNonDefaultAuthority, nodeID, client
+}
+
+// TestFederation_ListenerResourceContextParamOrder covers the case of watching
+// a Listener resource with the new style resource name and context parameters.
+// The test registers watches for two resources which differ only in the order
+// of context parameters in their URI. The server is configured to respond with
+// a single resource with canonicalized context parameters. The test verifies
+// that both watchers are notified.
+func (s) TestFederation_ListenerResourceContextParamOrder(t *testing.T) {
+	serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
+
+	var (
+		// Two resource names only differ in context parameter order.
+		resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?a=1&b=2", testNonDefaultAuthority)
+		resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?b=2&a=1", testNonDefaultAuthority)
+	)
+
+	// Register two watches for listener resources with the same query string,
+	// but context parameters in different order.
+	lw1 := newListenerWatcher()
+	ldsCancel1 := xdsresource.WatchListener(client, resourceName1, lw1)
+	defer ldsCancel1()
+	lw2 := newListenerWatcher()
+	ldsCancel2 := xdsresource.WatchListener(client, resourceName2, lw2)
+	defer ldsCancel2()
+
+	// Configure the management server for the non-default authority to return a
+	// single listener resource, corresponding to the watches registered above.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName1, "rds-resource")},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	wantUpdate := listenerUpdateErrTuple{
+		update: &xdsresource.ListenerUpdate{
+			APIListener: &xdsresource.HTTPConnectionManagerConfig{
+				RouteConfigName: "rds-resource",
+				HTTPFilters:     []xdsresource.HTTPFilter{{Name: "router"}},
+			},
+		},
+	}
+	// Verify the contents of the received update.
+	if err := verifyListenerUpdate(ctx, lw1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyListenerUpdate(ctx, lw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestFederation_RouteConfigResourceContextParamOrder covers the case of
+// watching a RouteConfiguration resource with the new style resource name and
+// context parameters. The test registers watches for two resources which
+// differ only in the order of context parameters in their URI. The server is
+// configured to respond with a single resource with canonicalized context
+// parameters. The test verifies that both watchers are notified.
+func (s) TestFederation_RouteConfigResourceContextParamOrder(t *testing.T) {
+	serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
+
+	var (
+		// Two resource names only differ in context parameter order.
+		resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?a=1&b=2", testNonDefaultAuthority)
+		resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?b=2&a=1", testNonDefaultAuthority)
+	)
+
+	// Register two watches for route configuration resources with the same
+	// query string, but context parameters in different order.
+	rw1 := newRouteConfigWatcher()
+	rdsCancel1 := xdsresource.WatchRouteConfig(client, resourceName1, rw1)
+	defer rdsCancel1()
+	rw2 := newRouteConfigWatcher()
+	rdsCancel2 := xdsresource.WatchRouteConfig(client, resourceName2, rw2)
+	defer rdsCancel2()
+
+	// Configure the management server for the non-default authority to return a
+	// single route config resource, corresponding to the watches registered.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(resourceName1, "listener-resource", "cluster-resource")},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	wantUpdate := routeConfigUpdateErrTuple{
+		update: xdsresource.RouteConfigUpdate{
+			VirtualHosts: []*xdsresource.VirtualHost{
+				{
+					Domains: []string{"listener-resource"},
+					Routes: []*xdsresource.Route{
+						{
+							Prefix:           newStringP("/"),
+							ActionType:       xdsresource.RouteActionRoute,
+							WeightedClusters: []xdsresource.WeightedCluster{{Name: "cluster-resource", Weight: 100}},
+						},
+					},
+				},
+			},
+		},
+	}
+	// Verify the contents of the received update.
+	if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestFederation_ClusterResourceContextParamOrder covers the case of watching a
+// Cluster resource with the new style resource name and context parameters.
+// The test registers watches for two resources which differ only in the order
+// of context parameters in their URI. The server is configured to respond with
+// a single resource with canonicalized context parameters. The test verifies
+// that both watchers are notified.
+func (s) TestFederation_ClusterResourceContextParamOrder(t *testing.T) {
+	serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
+
+	var (
+		// Two resource names only differ in context parameter order.
+		resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", testNonDefaultAuthority)
+		resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?b=2&a=1", testNonDefaultAuthority)
+	)
+
+	// Register two watches for cluster resources with the same query string,
+	// but context parameters in different order.
+	cw1 := newClusterWatcher()
+	cdsCancel1 := xdsresource.WatchCluster(client, resourceName1, cw1)
+	defer cdsCancel1()
+	cw2 := newClusterWatcher()
+	cdsCancel2 := xdsresource.WatchCluster(client, resourceName2, cw2)
+	defer cdsCancel2()
+
+	// Configure the management server for the non-default authority to return a
+	// single cluster resource, corresponding to the watches registered.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Clusters:       []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, "eds-service-name", e2e.SecurityLevelNone)},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	wantUpdate := clusterUpdateErrTuple{
+		update: xdsresource.ClusterUpdate{
+			ClusterName:    "xdstp://non-default-authority/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2",
+			EDSServiceName: "eds-service-name",
+		},
+	}
+	// Verify the contents of the received update.
+	if err := verifyClusterUpdate(ctx, cw1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyClusterUpdate(ctx, cw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestFederation_EndpointsResourceContextParamOrder covers the case of watching
+// an Endpoints resource with the new style resource name and context parameters.
+// The test registers watches for two resources which differ only in the order
+// of context parameters in their URI. The server is configured to respond with
+// a single resource with canonicalized context parameters. The test verifies
+// that both watchers are notified.
+func (s) TestFederation_EndpointsResourceContextParamOrder(t *testing.T) {
+	serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t)
+
+	var (
+		// Two resource names only differ in context parameter order.
+		resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?a=1&b=2", testNonDefaultAuthority)
+		resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?b=2&a=1", testNonDefaultAuthority)
+	)
+
+	// Register two watches for endpoint resources with the same query string,
+	// but context parameters in different order.
+	ew1 := newEndpointsWatcher()
+	edsCancel1 := xdsresource.WatchEndpoints(client, resourceName1, ew1)
+	defer edsCancel1()
+	ew2 := newEndpointsWatcher()
+	edsCancel2 := xdsresource.WatchEndpoints(client, resourceName2, ew2)
+	defer edsCancel2()
+
+	// Configure the management server for the non-default authority to return a
+	// single endpoints resource, corresponding to the watches registered.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Endpoints:      []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(resourceName1, "localhost", []uint32{666})},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	wantUpdate := endpointsUpdateErrTuple{
+		update: xdsresource.EndpointsUpdate{
+			Localities: []xdsresource.Locality{
+				{
+					Endpoints: []xdsresource.Endpoint{{
+						ResolverEndpoint: resolver.Endpoint{
+							Addresses: []resolver.Address{{Addr: "localhost:666"}},
+						},
+						Weight: 1,
+					}},
+					Weight: 1,
+					ID: clients.Locality{
+						Region:  "region-1",
+						Zone:    "zone-1",
+						SubZone: "subzone-1",
+					},
+				},
+			},
+		},
+	}
+	// Verify the contents of the received update.
+	if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func newStringP(s string) *string {
+	return &s
+}

internal/xds/xdsclient/tests/helpers_test.go

@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestWatchExpiryTimeout = 500 * time.Millisecond
+	defaultTestTimeout            = 10 * time.Second
+	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
+
+	ldsName         = "xdsclient-test-lds-resource"
+	rdsName         = "xdsclient-test-rds-resource"
+	cdsName         = "xdsclient-test-cds-resource"
+	edsName         = "xdsclient-test-eds-resource"
+	ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource"
+	rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource"
+	cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource"
+	edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource"
+)
+
+func makeAuthorityName(name string) string {
+	segs := strings.Split(name, "/")
+	return strings.Join(segs, "")
+}
+
+func makeNewStyleLDSName(authority string) string {
+	return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority)
+}
+
+func makeNewStyleRDSName(authority string) string {
+	return fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource", authority)
+}
+
+func makeNewStyleCDSName(authority string) string {
+	return fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource", authority)
+}
+
+func makeNewStyleEDSName(authority string) string {
+	return fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource", authority)
+}

internal/xds/xdsclient/tests/loadreport_test.go

@@ -0,0 +1,566 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/testutils/xds/fakeserver"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+const (
+	testKey1 = "test-key1"
+	testKey2 = "test-key2"
+)
+
+var (
+	testLocality1     = clients.Locality{Region: "test-region1"}
+	testLocality2     = clients.Locality{Region: "test-region2"}
+	toleranceCmpOpt   = cmpopts.EquateApprox(0, 1e-5)
+	ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats",
+		cmpopts.SortSlices(func(a, b protocmp.Message) bool {
+			return a.String() < b.String()
+		}),
+	)
+)
+
+type wrappedListener struct {
+	net.Listener
+	newConnChan *testutils.Channel // Connection attempts are pushed here.
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	c, err := wl.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	wl.newConnChan.Send(struct{}{})
+	return c, err
+}
+
+// Tests a load reporting scenario where the xDS client is reporting loads to
+// multiple servers. Verifies the following:
+//   - calling the load reporting API with different server configuration
+//     results in connections being created to those corresponding servers
+//   - the same load.Store is not returned when the load reporting API called
+//     with different server configurations
+//   - canceling the load reporting from the client results in the LRS stream
+//     being canceled on the server
+func (s) TestReportLoad_ConnectionCreation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create two management servers that also serve LRS.
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create a local TCP listener: %v", err)
+	}
+	newConnChan1 := testutils.NewChannel()
+	lis1 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan1,
+	}
+	mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis1,
+		SupportLoadReportingService: true,
+	})
+	l, err = testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("Failed to create a local TCP listener: %v", err)
+	}
+	newConnChan2 := testutils.NewChannel()
+	lis2 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan2,
+	}
+	mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis2,
+		SupportLoadReportingService: true,
+	})
+
+	// Create an xDS client with a bootstrap configuration that contains both of
+	// the above two servers. The authority name is immaterial here since load
+	// reporting is per-server and not per-authority.
+	nodeID := uuid.New().String()
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer1.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			"test-authority": []byte(fmt.Sprintf(`{
+				"xds_servers": [{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]}`, mgmtServer2.Address)),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+	client := createXDSClient(t, bc)
+
+	serverCfg1, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer1.Address})
+	if err != nil {
+		t.Fatalf("Failed to create server config for testing: %v", err)
+	}
+	// Call the load reporting API to report load to the first management
+	// server, and ensure that a connection to the server is created.
+	store1, lrsCancel1 := client.ReportLoad(serverCfg1)
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	defer lrsCancel1(sCtx)
+	if _, err := newConnChan1.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting")
+	}
+	if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	serverCfg2, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer2.Address})
+	if err != nil {
+		t.Fatalf("Failed to create server config for testing: %v", err)
+	}
+	// Call the load reporting API to report load to the second management
+	// server, and ensure that a connection to the server is created.
+	store2, lrsCancel2 := client.ReportLoad(serverCfg2)
+	sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel2()
+	defer lrsCancel2(sCtx2)
+	if _, err := newConnChan2.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting")
+	}
+	if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	if store1 == store2 {
+		t.Fatalf("Got same store for different servers, want different")
+	}
+
+	// Push some loads on the received store.
+	store2.ReporterForCluster("cluster", "eds").CallDropped("test")
+
+	// Ensure the initial load reporting request is received at the server.
+	lrsServer := mgmtServer2.LRSServer
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "gRPC Go",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for LRS request with loads: %v", err)
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster",
+		ClusterServiceName:   "eds",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Cancel this load reporting stream, server should see error canceled.
+	sCtx2, sCancel2 = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel2()
+	lrsCancel2(sCtx2)
+
+	// Server should receive a stream canceled error. There may be additional
+	// load reports from the client in the channel.
+	for {
+		if ctx.Err() != nil {
+			t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server")
+		}
+		u, err := lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		// Ignore load reports sent before the stream was cancelled.
+		if u.(*fakeserver.Request).Err == nil {
+			continue
+		}
+		if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled {
+			t.Fatalf("Unexpected LRS request: %v, want error canceled", u)
+		}
+		break
+	}
+}
+
+// Tests a load reporting scenario where the load reporting API is called
+// multiple times for the same server. The test verifies the following:
+//   - calling the load reporting API the second time for the same server
+//     configuration does not create a new LRS stream
+//   - the LRS stream is closed *only* after all the API calls invoke their
+//     cancel functions
+//   - creating new streams after the previous one was closed works
+func (s) TestReportLoad_StreamCreation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	client := createXDSClient(t, bc)
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address})
+	if err != nil {
+		t.Fatalf("Failed to create server config for testing: %v", err)
+	}
+	store1, cancel1 := client.ReportLoad(serverConfig)
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	// Push some loads on the received store.
+	store1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+	store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1)
+	store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14)
+	store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718)
+	store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil)
+	store1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2)
+	store1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618)
+	store1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil)
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "gRPC Go",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster1",
+		ClusterServiceName:   "eds1",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{
+			{
+				Locality: &v3corepb.Locality{Region: "test-region1"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					// TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858
+					{MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+			{
+				Locality: &v3corepb.Locality{Region: "test-region2"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					{MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+		},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Make another call to the load reporting API, and ensure that a new LRS
+	// stream is not created.
+	store2, cancel2 := client.ReportLoad(serverConfig)
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("New LRS stream created when expected to use an existing one")
+	}
+
+	// Push more loads.
+	store2.ReporterForCluster("cluster2", "eds2").CallDropped("test")
+
+	// Ensure that loads are seen on the server. We need a loop here because
+	// there could have been some requests from the client in the time between
+	// us reading the first request and now. Those would have been queued in the
+	// request channel that we read out of.
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for new loads to be seen on the server")
+		}
+
+		req, err = lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+		if l := len(gotLoad); l != 1 {
+			continue
+		}
+		gotLoad[0].LoadReportInterval = nil
+		wantLoad := &v3endpointpb.ClusterStats{
+			ClusterName:          "cluster2",
+			ClusterServiceName:   "eds2",
+			TotalDroppedRequests: 1,
+			DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		}
+		if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
+			t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+			continue
+		}
+		break
+	}
+
+	// Cancel the first load reporting call, and ensure that the stream does not
+	// close (because we have another call open).
+	sCtx1, sCancel1 := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel1()
+	cancel1(sCtx1)
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("LRS stream closed when expected to stay open")
+	}
+
+	// Cancel the second load reporting call, and ensure the stream is closed.
+	sCtx2, sCancel2 := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel2()
+	cancel2(sCtx2)
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for LRS stream to close")
+	}
+
+	// Calling the load reporting API again should result in the creation of a
+	// new LRS stream. This ensures that creating and closing multiple streams
+	// works smoothly.
+	_, cancel3 := client.ReportLoad(serverConfig)
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+	sCtx3, sCancel3 := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel3()
+	cancel3(sCtx3)
+}
+
+// TestConcurrentReportLoad verifies that the client can safely handle concurrent
+// requests to initiate load reporting streams. It launches multiple goroutines
+// that all call client.ReportLoad simultaneously. We don't verify anything on
+// the server here, since that is done in other tests. This is just to ensure
+// that concurrent ReportLoad() calls work.
+func (s) TestConcurrentReportLoad(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+	client := createXDSClient(t, bc)
+
+	serverConfig, err := bootstrap.ServerConfigForTesting(bootstrap.ServerConfigTestingOptions{URI: mgmtServer.Address})
+	if err != nil {
+		t.Fatalf("Failed to create server config for testing: %v", err)
+	}
+
+	// Call ReportLoad() concurrently from multiple go routines.
+	const numGoroutines = 10
+	cancelStores := make([]func(context.Context), numGoroutines)
+	var wg sync.WaitGroup
+	wg.Add(numGoroutines)
+	for i := range numGoroutines {
+		go func(idx int) {
+			defer wg.Done()
+			_, cancelStores[idx] = client.ReportLoad(serverConfig)
+		}(i)
+	}
+	wg.Wait()
+
+	// Cancel all the load reporting streams. The last call to cancel is
+	// expected to block until a final load report with pending loads is sent.
+	// But the stream is currently blocked on a recv() call waiting for the LRS
+	// server to send the initial laod reporting response, which it never does.
+	// Hence we use a context with short timeout here.
+	for i, cancel := range cancelStores {
+		sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+		defer sCancel()
+		cancel(sCtx)
+		t.Logf("Cancelled LRS stream %d", i)
+	}
+}
+
+// TestConcurrentChannels verifies that we can create multiple gRPC channels
+// concurrently with a shared XDSClient, each of which will create a new LRS
+// stream without any race.
+func (s) TestConcurrentChannels(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true, SupportLoadReportingService: true})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	if internal.NewXDSResolverWithPoolForTesting == nil {
+		t.Fatalf("internal.NewXDSResolverWithConfigForTesting is nil")
+	}
+
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+
+	resolverBuilder := internal.NewXDSResolverWithPoolForTesting.(func(*xdsclient.Pool) (resolver.Builder, error))
+	xdsResolver, err := resolverBuilder(pool)
+	if err != nil {
+		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
+	}
+
+	server := stubserver.StartTestService(t, nil)
+	defer server.Stop()
+
+	// Configure the management server with resources that enable LRS.
+	const serviceName = "my-service-e2e-lrs-test"
+	resources := e2e.DefaultClientResources(e2e.ResourceParams{
+		DialTarget: serviceName,
+		NodeID:     nodeID,
+		Host:       "localhost",
+		Port:       testutils.ParsePort(t, server.Address),
+		SecLevel:   e2e.SecurityLevelNone,
+	})
+	resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{
+		ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{
+			Self: &v3corepb.SelfConfigSource{},
+		},
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	const (
+		numGoroutines = 10
+		numRPCs       = 10
+	)
+	for range numGoroutines {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for range numRPCs {
+				cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsResolver))
+				if err != nil {
+					t.Errorf("grpc.NewClient() failed: %v", err)
+					return
+				}
+				defer cc.Close()
+
+				testClient := testgrpc.NewTestServiceClient(cc)
+				if _, err := testClient.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+					t.Errorf("EmptyCall() failed: %v", err)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}

internal/xds/xdsclient/tests/rds_watchers_test.go

@@ -0,0 +1,996 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/internal/grpcsync"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/bootstrap"
+	"google.golang.org/grpc/internal/xds/xdsclient"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+type noopRouteConfigWatcher struct{}
+
+func (noopRouteConfigWatcher) ResourceChanged(_ *xdsresource.RouteConfigUpdate, onDone func()) {
+	onDone()
+}
+func (noopRouteConfigWatcher) ResourceError(_ error, onDone func()) {
+	onDone()
+}
+func (noopRouteConfigWatcher) AmbientError(_ error, onDone func()) {
+	onDone()
+}
+
+type routeConfigUpdateErrTuple struct {
+	update xdsresource.RouteConfigUpdate
+	err    error
+}
+
+type routeConfigWatcher struct {
+	updateCh *testutils.Channel
+}
+
+func newRouteConfigWatcher() *routeConfigWatcher {
+	return &routeConfigWatcher{updateCh: testutils.NewChannel()}
+}
+
+func (rw *routeConfigWatcher) ResourceChanged(update *xdsresource.RouteConfigUpdate, onDone func()) {
+	rw.updateCh.Send(routeConfigUpdateErrTuple{update: *update})
+	onDone()
+}
+
+func (rw *routeConfigWatcher) ResourceError(err error, onDone func()) {
+	// When used with a go-control-plane management server that continuously
+	// resends resources which are NACKed by the xDS client, using a `Replace()`
+	// here and in AmbientError() simplifies tests which will have
+	// access to the most recently received error.
+	rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err})
+	onDone()
+}
+
+func (rw *routeConfigWatcher) AmbientError(err error, onDone func()) {
+	rw.updateCh.Replace(routeConfigUpdateErrTuple{err: err})
+	onDone()
+}
+
+// badRouteConfigResource returns a RouteConfiguration resource for the given
+// routeName which contains a retry config with num_retries set to `0`. This is
+// expected to be NACK'ed by the xDS client.
+func badRouteConfigResource(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration {
+	return &v3routepb.RouteConfiguration{
+		Name: routeName,
+		VirtualHosts: []*v3routepb.VirtualHost{{
+			Domains: []string{ldsTarget},
+			Routes: []*v3routepb.Route{{
+				Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}},
+				Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
+					ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName},
+				}}}},
+			RetryPolicy: &v3routepb.RetryPolicy{
+				NumRetries: &wrapperspb.UInt32Value{Value: 0},
+			},
+		}},
+	}
+}
+
+// xdsClient is expected to produce an error containing this string when an
+// update is received containing a route configuration resource created using
+// `badRouteConfigResource`.
+const wantRouteConfigNACKErr = "received route is invalid: retry_policy.num_retries = 0; must be >= 1"
+
+// verifyRouteConfigUpdate waits for an update to be received on the provided
+// update channel and verifies that it matches the expected update.
+//
+// Returns an error if no update is received before the context deadline expires
+// or the received update does not match the expected one.
+func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate routeConfigUpdateErrTuple) error {
+	u, err := updateCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("timeout when waiting for a route configuration resource from the management server: %v", err)
+	}
+	got := u.(routeConfigUpdateErrTuple)
+	if wantUpdate.err != nil {
+		if got.err == nil || !strings.Contains(got.err.Error(), wantUpdate.err.Error()) {
+			return fmt.Errorf("update received with error: %v, want %q", got.err, wantUpdate.err)
+		}
+	}
+	cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw")}
+	if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" {
+		return fmt.Errorf("received unexpected diff in the route configuration resource update: (-want, got):\n%s", diff)
+	}
+	return nil
+}
+
+// verifyNoRouteConfigUpdate verifies that no route configuration update is
+// received on the provided update channel, and returns an error if an update is
+// received.
+//
+// A very short deadline is used while waiting for the update, as this function
+// is intended to be used when an update is not expected.
+func verifyNoRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel) error {
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded {
+		return fmt.Errorf("unexpected RouteConfigUpdate: %v", u)
+	}
+	return nil
+}
+
+// TestRDSWatch covers the case where a single watcher exists for a single route
+// configuration resource. The test verifies the following scenarios:
+//  1. An update from the management server containing the resource being
+//     watched should result in the invocation of the watch callback.
+//  2. An update from the management server containing a resource *not* being
+//     watched should not result in the invocation of the watch callback.
+//  3. After the watch is cancelled, an update from the management server
+//     containing the resource that was being watched should not result in the
+//     invocation of the watch callback.
+//
+// The test is run for old and new style names.
+func (s) TestRDSWatch(t *testing.T) {
+	tests := []struct {
+		desc                   string
+		resourceName           string
+		watchedResource        *v3routepb.RouteConfiguration // The resource being watched.
+		updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update.
+		notWatchedResource     *v3routepb.RouteConfiguration // A resource which is not being watched.
+		wantUpdate             routeConfigUpdateErrTuple
+	}{
+		{
+			desc:                   "old style resource",
+			resourceName:           rdsName,
+			watchedResource:        e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
+			updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"),
+			notWatchedResource:     e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsName, cdsName),
+			wantUpdate: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsName},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:                   "new style resource",
+			resourceName:           rdsNameNewStyle,
+			watchedResource:        e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle),
+			updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"),
+			notWatchedResource:     e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsNameNewStyle, cdsNameNewStyle),
+			wantUpdate: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsNameNewStyle},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+			nodeID := uuid.New().String()
+			bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+				Servers: []byte(fmt.Sprintf(`[{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]`, mgmtServer.Address)),
+				Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+				Authorities: map[string]json.RawMessage{
+					// Xdstp resource names used in this test do not specify an
+					// authority. These will end up looking up an entry with the
+					// empty key in the authorities map. Having an entry with an
+					// empty key and empty configuration, results in these
+					// resources also using the top-level configuration.
+					"": []byte(`{}`),
+				},
+			})
+			if err != nil {
+				t.Fatalf("Failed to create bootstrap configuration: %v", err)
+			}
+
+			// Create an xDS client with the above bootstrap contents.
+			config, err := bootstrap.NewConfigFromContents(bc)
+			if err != nil {
+				t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+			}
+			pool := xdsclient.NewPool(config)
+			client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+				Name: t.Name(),
+			})
+			if err != nil {
+				t.Fatalf("Failed to create xDS client: %v", err)
+			}
+			defer close()
+
+			// Register a watch for a route configuration resource and have the
+			// watch callback push the received update on to a channel.
+			rw := newRouteConfigWatcher()
+			rdsCancel := xdsresource.WatchRouteConfig(client, test.resourceName, rw)
+
+			// Configure the management server to return a single route
+			// configuration resource, corresponding to the one being watched.
+			resources := e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Routes:         []*v3routepb.RouteConfiguration{test.watchedResource},
+				SkipValidation: true,
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+
+			// Verify the contents of the received update.
+			if err := verifyRouteConfigUpdate(ctx, rw.updateCh, test.wantUpdate); err != nil {
+				t.Fatal(err)
+			}
+
+			// Configure the management server to return an additional route
+			// configuration resource, one that we are not interested in.
+			resources = e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Routes:         []*v3routepb.RouteConfiguration{test.watchedResource, test.notWatchedResource},
+				SkipValidation: true,
+			}
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+			if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
+				t.Fatal(err)
+			}
+
+			// Cancel the watch and update the resource corresponding to the original
+			// watch.  Ensure that the cancelled watch callback is not invoked.
+			rdsCancel()
+			resources = e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Routes:         []*v3routepb.RouteConfiguration{test.updatedWatchedResource, test.notWatchedResource},
+				SkipValidation: true,
+			}
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+			if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestRDSWatch_TwoWatchesForSameResourceName covers the case where two watchers
+// exist for a single route configuration resource.  The test verifies the
+// following scenarios:
+//  1. An update from the management server containing the resource being
+//     watched should result in the invocation of both watch callbacks.
+//  2. After one of the watches is cancelled, a redundant update from the
+//     management server should not result in the invocation of either of the
+//     watch callbacks.
+//  3. An update from the management server containing the resource being
+//     watched should result in the invocation of the un-cancelled watch
+//     callback.
+//
+// The test is run for old and new style names.
+func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) {
+	tests := []struct {
+		desc                   string
+		resourceName           string
+		watchedResource        *v3routepb.RouteConfiguration // The resource being watched.
+		updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update.
+		wantUpdateV1           routeConfigUpdateErrTuple
+		wantUpdateV2           routeConfigUpdateErrTuple
+	}{
+		{
+			desc:                   "old style resource",
+			resourceName:           rdsName,
+			watchedResource:        e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
+			updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"),
+			wantUpdateV1: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsName},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdateV2: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsName},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			desc:                   "new style resource",
+			resourceName:           rdsNameNewStyle,
+			watchedResource:        e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle),
+			updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"),
+			wantUpdateV1: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsNameNewStyle},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsNameNewStyle, Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+			wantUpdateV2: routeConfigUpdateErrTuple{
+				update: xdsresource.RouteConfigUpdate{
+					VirtualHosts: []*xdsresource.VirtualHost{
+						{
+							Domains: []string{ldsNameNewStyle},
+							Routes: []*xdsresource.Route{
+								{
+									Prefix:           newStringP("/"),
+									ActionType:       xdsresource.RouteActionRoute,
+									WeightedClusters: []xdsresource.WeightedCluster{{Name: "new-cds-resource", Weight: 100}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+			nodeID := uuid.New().String()
+			bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+				Servers: []byte(fmt.Sprintf(`[{
+					"server_uri": %q,
+					"channel_creds": [{"type": "insecure"}]
+				}]`, mgmtServer.Address)),
+				Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+				Authorities: map[string]json.RawMessage{
+					// Xdstp resource names used in this test do not specify an
+					// authority. These will end up looking up an entry with the
+					// empty key in the authorities map. Having an entry with an
+					// empty key and empty configuration, results in these
+					// resources also using the top-level configuration.
+					"": []byte(`{}`),
+				},
+			})
+			if err != nil {
+				t.Fatalf("Failed to create bootstrap configuration: %v", err)
+			}
+
+			// Create an xDS client with the above bootstrap contents.
+			config, err := bootstrap.NewConfigFromContents(bc)
+			if err != nil {
+				t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+			}
+			pool := xdsclient.NewPool(config)
+			client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+				Name: t.Name(),
+			})
+			if err != nil {
+				t.Fatalf("Failed to create xDS client: %v", err)
+			}
+			defer close()
+
+			// Register two watches for the same route configuration resource
+			// and have the callbacks push the received updates on to a channel.
+			rw1 := newRouteConfigWatcher()
+			rdsCancel1 := xdsresource.WatchRouteConfig(client, test.resourceName, rw1)
+			defer rdsCancel1()
+			rw2 := newRouteConfigWatcher()
+			rdsCancel2 := xdsresource.WatchRouteConfig(client, test.resourceName, rw2)
+
+			// Configure the management server to return a single route
+			// configuration resource, corresponding to the one being watched.
+			resources := e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Routes:         []*v3routepb.RouteConfiguration{test.watchedResource},
+				SkipValidation: true,
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+
+			// Verify the contents of the received update.
+			if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV1); err != nil {
+				t.Fatal(err)
+			}
+			if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, test.wantUpdateV1); err != nil {
+				t.Fatal(err)
+			}
+
+			// Cancel the second watch and force the management server to push a
+			// redundant update for the resource being watched. Neither of the
+			// two watch callbacks should be invoked.
+			rdsCancel2()
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+			if err := verifyNoRouteConfigUpdate(ctx, rw1.updateCh); err != nil {
+				t.Fatal(err)
+			}
+			if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil {
+				t.Fatal(err)
+			}
+
+			// Update to the resource being watched. The un-cancelled callback
+			// should be invoked while the cancelled one should not be.
+			resources = e2e.UpdateOptions{
+				NodeID:         nodeID,
+				Routes:         []*v3routepb.RouteConfiguration{test.updatedWatchedResource},
+				SkipValidation: true,
+			}
+			if err := mgmtServer.Update(ctx, resources); err != nil {
+				t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+			}
+			if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, test.wantUpdateV2); err != nil {
+				t.Fatal(err)
+			}
+			if err := verifyNoRouteConfigUpdate(ctx, rw2.updateCh); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// TestRDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three
+// watchers (two watchers for one resource, and the third watcher for another
+// resource), exist across two route configuration resources.  The test verifies
+// that an update from the management server containing both resources results
+// in the invocation of all watch callbacks.
+//
+// The test is run with both old and new style names.
+func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	authority := makeAuthorityName(t.Name())
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			// Xdstp style resource names used in this test use a slash removed
+			// version of t.Name as their authority, and the empty config
+			// results in the top-level xds server configuration being used for
+			// this authority.
+			authority: []byte(`{}`),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register two watches for the same route configuration resource
+	// and have the callbacks push the received updates on to a channel.
+	rw1 := newRouteConfigWatcher()
+	rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1)
+	defer rdsCancel1()
+	rw2 := newRouteConfigWatcher()
+	rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2)
+	defer rdsCancel2()
+
+	// Register the third watch for a different route configuration resource.
+	rdsNameNewStyle := makeNewStyleRDSName(authority)
+	rw3 := newRouteConfigWatcher()
+	rdsCancel3 := xdsresource.WatchRouteConfig(client, rdsNameNewStyle, rw3)
+	defer rdsCancel3()
+
+	// Configure the management server to return two route configuration
+	// resources, corresponding to the registered watches.
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Routes: []*v3routepb.RouteConfiguration{
+			e2e.DefaultRouteConfig(rdsName, ldsName, cdsName),
+			e2e.DefaultRouteConfig(rdsNameNewStyle, ldsName, cdsName),
+		},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify the contents of the received update for the all watchers. The two
+	// resources returned differ only in the resource name. Therefore the
+	// expected update is the same for all the watchers.
+	wantUpdate := routeConfigUpdateErrTuple{
+		update: xdsresource.RouteConfigUpdate{
+			VirtualHosts: []*xdsresource.VirtualHost{
+				{
+					Domains: []string{ldsName},
+					Routes: []*xdsresource.Route{
+						{
+							Prefix:           newStringP("/"),
+							ActionType:       xdsresource.RouteActionRoute,
+							WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw3.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestRDSWatch_ResourceCaching covers the case where a watch is registered for
+// a resource which is already present in the cache.  The test verifies that the
+// watch callback is invoked with the contents from the cache, instead of a
+// request being sent to the management server.
+func (s) TestRDSWatch_ResourceCaching(t *testing.T) {
+	firstRequestReceived := false
+	firstAckReceived := grpcsync.NewEvent()
+	secondRequestReceived := grpcsync.NewEvent()
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			// The first request has an empty version string.
+			if !firstRequestReceived && req.GetVersionInfo() == "" {
+				firstRequestReceived = true
+				return nil
+			}
+			// The first ack has a non-empty version string.
+			if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" {
+				firstAckReceived.Fire()
+				return nil
+			}
+			// Any requests after the first request and ack, are not expected.
+			secondRequestReceived.Fire()
+			return nil
+		},
+	})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a route configuration resource and have the watch
+	// callback push the received update on to a channel.
+	rw1 := newRouteConfigWatcher()
+	rdsCancel1 := xdsresource.WatchRouteConfig(client, rdsName, rw1)
+	defer rdsCancel1()
+
+	// Configure the management server to return a single route configuration
+	// resource, corresponding to the one we registered a watch for.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify the contents of the received update.
+	wantUpdate := routeConfigUpdateErrTuple{
+		update: xdsresource.RouteConfigUpdate{
+			VirtualHosts: []*xdsresource.VirtualHost{
+				{
+					Domains: []string{ldsName},
+					Routes: []*xdsresource.Route{
+						{
+							Prefix:           newStringP("/"),
+							ActionType:       xdsresource.RouteActionRoute,
+							WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw1.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-ctx.Done():
+		t.Fatal("timeout when waiting for receipt of ACK at the management server")
+	case <-firstAckReceived.Done():
+	}
+
+	// Register another watch for the same resource. This should get the update
+	// from the cache.
+	rw2 := newRouteConfigWatcher()
+	rdsCancel2 := xdsresource.WatchRouteConfig(client, rdsName, rw2)
+	defer rdsCancel2()
+	if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+	// No request should get sent out as part of this watch.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	select {
+	case <-sCtx.Done():
+	case <-secondRequestReceived.Done():
+		t.Fatal("xdsClient sent out request instead of using update from cache")
+	}
+}
+
+// TestRDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client
+// does not receive an RDS response for the request that it sends. The test
+// verifies that the watch callback is invoked with an error once the
+// watchExpiryTimer fires.
+func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client talking to the above management server.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name:               t.Name(),
+		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create an xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a resource which is expected to fail with an error
+	// after the watch expiry timer fires.
+	rw := newRouteConfigWatcher()
+	rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
+	defer rdsCancel()
+
+	// Wait for the watch expiry timer to fire.
+	<-time.After(defaultTestWatchExpiryTimeout)
+
+	// Verify that an empty update with the expected error is received.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantErr := xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "")
+	if err := verifyRouteConfigUpdate(ctx, rw.updateCh, routeConfigUpdateErrTuple{err: wantErr}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the
+// client receives a valid RDS response for the request that it sends. The test
+// verifies that the behavior associated with the expiry timer (i.e, callback
+// invocation with error) does not take place.
+func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client talking to the above management server.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name:               t.Name(),
+		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
+	})
+	if err != nil {
+		t.Fatalf("Failed to create an xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a route configuration resource and have the watch
+	// callback push the received update on to a channel.
+	rw := newRouteConfigWatcher()
+	rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
+	defer rdsCancel()
+
+	// Configure the management server to return a single route configuration
+	// resource, corresponding to the one we registered a watch for.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Routes:         []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify the contents of the received update.
+	wantUpdate := routeConfigUpdateErrTuple{
+		update: xdsresource.RouteConfigUpdate{
+			VirtualHosts: []*xdsresource.VirtualHost{
+				{
+					Domains: []string{ldsName},
+					Routes: []*xdsresource.Route{
+						{
+							Prefix:           newStringP("/"),
+							ActionType:       xdsresource.RouteActionRoute,
+							WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for the watch expiry timer to fire, and verify that the callback is
+	// not invoked.
+	<-time.After(defaultTestWatchExpiryTimeout)
+	if err := verifyNoRouteConfigUpdate(ctx, rw.updateCh); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestRDSWatch_NACKError covers the case where an update from the management
+// server is NACK'ed by the xdsclient. The test verifies that the error is
+// propagated to the watcher.
+func (s) TestRDSWatch_NACKError(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	bc := e2e.DefaultBootstrapContents(t, nodeID, mgmtServer.Address)
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register a watch for a route configuration resource and have the watch
+	// callback push the received update on to a channel.
+	rw := newRouteConfigWatcher()
+	rdsCancel := xdsresource.WatchRouteConfig(client, rdsName, rw)
+	defer rdsCancel()
+
+	// Configure the management server to return a single route configuration
+	// resource which is expected to be NACKed by the client.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Routes:         []*v3routepb.RouteConfiguration{badRouteConfigResource(rdsName, ldsName, cdsName)},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify that the expected error is propagated to the watcher.
+	u, err := rw.updateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err)
+	}
+	gotErr := u.(routeConfigUpdateErrTuple).err
+	if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) {
+		t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr)
+	}
+}
+
+// TestRDSWatch_PartialValid covers the case where a response from the
+// management server contains both valid and invalid resources and is expected
+// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding
+// to the valid resource receive the update, while watchers corresponding to the
+// invalid resource receive an error.
+func (s) TestRDSWatch_PartialValid(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	authority := makeAuthorityName(t.Name())
+	bc, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
+		Servers: []byte(fmt.Sprintf(`[{
+			"server_uri": %q,
+			"channel_creds": [{"type": "insecure"}]
+		}]`, mgmtServer.Address)),
+		Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
+		Authorities: map[string]json.RawMessage{
+			// Xdstp style resource names used in this test use a slash removed
+			// version of t.Name as their authority, and the empty config
+			// results in the top-level xds server configuration being used for
+			// this authority.
+			authority: []byte(`{}`),
+		},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create bootstrap configuration: %v", err)
+	}
+
+	// Create an xDS client with the above bootstrap contents.
+	config, err := bootstrap.NewConfigFromContents(bc)
+	if err != nil {
+		t.Fatalf("Failed to parse bootstrap contents: %s, %v", string(bc), err)
+	}
+	pool := xdsclient.NewPool(config)
+	client, close, err := pool.NewClientForTesting(xdsclient.OptionsForTesting{
+		Name: t.Name(),
+	})
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer close()
+
+	// Register two watches for route configuration resources. The first watch
+	// is expected to receive an error because the received resource is NACKed.
+	// The second watch is expected to get a good update.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	badResourceName := rdsName
+	rw1 := newRouteConfigWatcher()
+	rdsCancel1 := xdsresource.WatchRouteConfig(client, badResourceName, rw1)
+	defer rdsCancel1()
+	goodResourceName := makeNewStyleRDSName(authority)
+	rw2 := newRouteConfigWatcher()
+	rdsCancel2 := xdsresource.WatchRouteConfig(client, goodResourceName, rw2)
+	defer rdsCancel2()
+
+	// Configure the management server to return two route configuration
+	// resources, corresponding to the registered watches.
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Routes: []*v3routepb.RouteConfiguration{
+			badRouteConfigResource(badResourceName, ldsName, cdsName),
+			e2e.DefaultRouteConfig(goodResourceName, ldsName, cdsName),
+		},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify that the expected error is propagated to the watcher which
+	// requested for the bad resource.
+	u, err := rw1.updateCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err)
+	}
+	gotErr := u.(routeConfigUpdateErrTuple).err
+	if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) {
+		t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr)
+	}
+
+	// Verify that the watcher watching the good resource receives a good
+	// update.
+	wantUpdate := routeConfigUpdateErrTuple{
+		update: xdsresource.RouteConfigUpdate{
+			VirtualHosts: []*xdsresource.VirtualHost{
+				{
+					Domains: []string{ldsName},
+					Routes: []*xdsresource.Route{
+						{
+							Prefix:           newStringP("/"),
+							ActionType:       xdsresource.RouteActionRoute,
+							WeightedClusters: []xdsresource.WeightedCluster{{Name: cdsName, Weight: 100}},
+						},
+					},
+				},
+			},
+		},
+	}
+	if err := verifyRouteConfigUpdate(ctx, rw2.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}