snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/clients/xdsclient/test/ads_stream_ack_nack_test.go

@@ -0,0 +1,671 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// Creates an xDS client with the given management server address, node ID
+// and transport builder.
+func createXDSClient(t *testing.T, mgmtServerAddress string, nodeID string, transportBuilder clients.TransportBuilder) *xdsclient.XDSClient {
+	t.Helper()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServerAddress,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: transportBuilder,
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	t.Cleanup(func() { client.Close() })
+	return client
+}
+
+// Tests simple ACK and NACK scenarios on the ADS stream:
+//  1. When a good response is received, i.e. once that is expected to be ACKed,
+//     the test verifies that an ACK is sent matching the version and nonce from
+//     the response.
+//  2. When a subsequent bad response is received, i.e. once is expected to be
+//     NACKed, the test verifies that a NACK is sent matching the previously
+//     ACKed version and current nonce from the response.
+//  3. When a subsequent good response is received, the test verifies that an
+//     ACK is sent matching the version and nonce from the current response.
+func (s) TestADS_ACK_NACK_Simple(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannelWithSize(1)
+	streamResponseCh := testutils.NewChannelWithSize(1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client pointing to the above server.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "user-agent",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for a discovery response from the server")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantReq.VersionInfo = gotResp.GetVersionInfo()
+	wantReq.ResponseNonce = gotResp.GetNonce()
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantUpdate := listenerUpdateErrTuple{
+		update: listenerUpdate{RouteConfigName: routeConfigName},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the management server with a listener resource that contains an
+	// empty HTTP connection manager within the apiListener, which will cause
+	// the resource to be NACKed.
+	badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener)
+	badListener.ApiListener.ApiListener = nil
+	mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{badListener},
+		SkipValidation: true,
+	})
+
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for a discovery response from the server")
+	}
+	gotResp = r.(*v3discoverypb.DiscoveryResponse)
+
+	wantNackErr := xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
+	if err := verifyListenerUpdate(ctx, lw.ambientErrCh, listenerUpdateErrTuple{ambientErr: wantNackErr}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the NACK contains the appropriate version, nonce and error.
+	// We expect the version to not change as this is a NACK.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for NACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
+		t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
+	}
+	if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
+		t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
+	}
+
+	// Update the management server to send a good resource again.
+	mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	})
+
+	// The envoy-go-control-plane management server keeps resending the same
+	// resource as long as we keep NACK'ing it. So, we will see the bad resource
+	// sent to us a few times here, before receiving the good resource.
+	var lastErr error
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for an ACK from the xDS client. Last seen error: %v", lastErr)
+		}
+
+		r, err = streamResponseCh.Receive(ctx)
+		if err != nil {
+			t.Fatal("Timeout when waiting for a discovery response from the server")
+		}
+		gotResp = r.(*v3discoverypb.DiscoveryResponse)
+
+		// Verify that the ACK contains the appropriate version and nonce.
+		r, err = streamRequestCh.Receive(ctx)
+		if err != nil {
+			t.Fatal("Timeout when waiting for ACK")
+		}
+		gotReq = r.(*v3discoverypb.DiscoveryRequest)
+		wantReq.VersionInfo = gotResp.GetVersionInfo()
+		wantReq.ResponseNonce = gotResp.GetNonce()
+		wantReq.ErrorDetail = nil
+		diff := cmp.Diff(gotReq, wantReq, protocmp.Transform())
+		if diff == "" {
+			lastErr = nil
+			break
+		}
+		lastErr = fmt.Errorf("unexpected diff in discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) {
+		if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+			lastErr = err
+			continue
+		}
+		break
+	}
+	if ctx.Err() != nil {
+		t.Fatalf("Timeout when waiting for listener update. Last seen error: %v", lastErr)
+	}
+}
+
+// Tests the case where the first response is invalid. The test verifies that
+// the NACK contains an empty version string.
+func (s) TestADS_NACK_InvalidFirstResponse(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannelWithSize(1)
+	streamResponseCh := testutils.NewChannelWithSize(1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+	})
+
+	// Create a listener resource on the management server that is expected to
+	// be NACKed by the xDS client.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	listenerResource.ApiListener.ApiListener = nil
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client pointing to the above server.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "user-agent",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the discovery response from client")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the error is propagated to the watcher.
+	var wantNackErr = xdsresource.NewError(xdsresource.ErrorTypeNACKed, "unexpected http connection manager resource type")
+	if err := verifyListenerUpdate(ctx, lw.resourceErrCh, listenerUpdateErrTuple{resourceErr: wantNackErr}); err != nil {
+		t.Fatal(err)
+	}
+
+	// NACK should contain the appropriate error, nonce, but empty version.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if gotVersion, wantVersion := gotReq.GetVersionInfo(), ""; gotVersion != wantVersion {
+		t.Errorf("Unexpected version in discovery request, got: %v, want: %v", gotVersion, wantVersion)
+	}
+	if gotNonce, wantNonce := gotReq.GetResponseNonce(), gotResp.GetNonce(); gotNonce != wantNonce {
+		t.Errorf("Unexpected nonce in discovery request, got: %v, want: %v", gotNonce, wantNonce)
+	}
+	if gotErr := gotReq.GetErrorDetail(); gotErr == nil || !strings.Contains(gotErr.GetMessage(), wantNackErr.Error()) {
+		t.Fatalf("Unexpected error in discovery request, got: %v, want: %v", gotErr.GetMessage(), wantNackErr)
+	}
+}
+
+// Tests the scenario where the xDS client is no longer interested in a
+// resource. The following sequence of events are tested:
+//  1. A resource is requested and a good response is received. The test verifies
+//     that an ACK is sent for this resource.
+//  2. The previously requested resource is no longer requested. The test
+//     verifies that the connection to the management server is closed.
+//  3. The same resource is requested again. The test verifies that a new
+//     request is sent with an empty version string, which corresponds to the
+//     first request on a new connection.
+func (s) TestADS_ACK_NACK_ResourceIsNotRequestedAnymore(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port. Configure the
+	// request and response handlers to push on channels that are inspected by
+	// the test goroutine to verify ACK version and nonce.
+	streamRequestCh := testutils.NewChannelWithSize(1)
+	streamResponseCh := testutils.NewChannelWithSize(1)
+	streamCloseCh := testutils.NewChannelWithSize(1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			streamRequestCh.SendContext(ctx, req)
+			return nil
+		},
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			streamResponseCh.SendContext(ctx, resp)
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			streamCloseCh.SendContext(ctx, struct{}{})
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	listenerResource := e2e.DefaultClientListener(listenerName, routeConfigName)
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client pointing to the above server.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	r, err := streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the initial discovery request")
+	}
+	gotReq := r.(*v3discoverypb.DiscoveryRequest)
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "user-agent",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	r, err = streamResponseCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for the discovery response from client")
+	}
+	gotResp := r.(*v3discoverypb.DiscoveryResponse)
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for ACK")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	wantACKReq := proto.Clone(wantReq).(*v3discoverypb.DiscoveryRequest)
+	wantACKReq.VersionInfo = gotResp.GetVersionInfo()
+	wantACKReq.ResponseNonce = gotResp.GetNonce()
+	if diff := cmp.Diff(gotReq, wantACKReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantUpdate := listenerUpdateErrTuple{
+		update: listenerUpdate{RouteConfigName: routeConfigName},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cancel the watch on the listener resource. This should result in the
+	// existing connection to be management server getting closed.
+	ldsCancel()
+	if _, err := streamCloseCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when expecting existing connection to be closed: %v", err)
+	}
+
+	// There is a race between two events when the last watch on an xdsChannel
+	// is canceled:
+	// - an empty discovery request being sent out
+	// - the ADS stream being closed
+	// To handle this race, we drain the request channel here so that if an
+	// empty discovery request was received, it is pulled out of the request
+	// channel and thereby guaranteeing a clean slate for the next watch
+	// registered below.
+	streamRequestCh.Drain()
+
+	// Register a watch for the same listener resource.
+	lw = newListenerWatcher()
+	ldsCancel = client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the discovery request is identical to the first one sent out
+	// to the management server.
+	r, err = streamRequestCh.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for discovery request")
+	}
+	gotReq = r.(*v3discoverypb.DiscoveryRequest)
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestADS_NACKError_DuplicateSuppression verifies that when a duplicate invalid
+// LDS resource is sent by the server, the client suppresses the duplicate error
+// notification to the watcher.
+func (s) TestADS_NACKError_DuplicateSuppression(t *testing.T) {
+	// The first NACK is expected upon receiving the invalid resource. The
+	// second NACK is a duplicate, sent because the management server resends
+	// the invalid resource in a loop as long as the resource update has not
+	// changed to a valid one.  The third NACK is waited for to verify that the
+	// duplicated error is still suppressed as expected.
+	const wantNACKCount = 3
+
+	nacksReceivedCh := make(chan struct{})
+	nackCount := 0
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if req.GetTypeUrl() != xdsresource.V3ListenerURL {
+				return nil
+			}
+			if req.GetErrorDetail() != nil {
+				nackCount++
+				if nackCount == wantNACKCount {
+					close(nacksReceivedCh)
+				}
+			}
+			return nil
+		},
+	})
+
+	const listenerName = "listener"
+	nodeID := uuid.New().String()
+
+	// Create an xDS client pointing to the above server.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	watcher := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
+	defer ldsCancel()
+
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{badListenerResource(t, listenerName)},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Verify that the watcher receives the initial error.
+	v, err := watcher.resourceErrCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for an endpoints resource error from the management server: %v", err)
+	}
+	const wantNackErr = "no RouteSpecifier"
+	if gotErr := v.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
+		t.Fatalf("Update received with error: %v, want %q", gotErr, wantNackErr)
+	}
+
+	// Wait for the management server to receive at least wantNACKCount NACKs,
+	// indicating that the NACK resend loop has run multiple times.
+	select {
+	case <-nacksReceivedCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for %d NACKs to be received by the server: %v", wantNACKCount, ctx.Err())
+	}
+
+	// Verify that no duplicate error update was written to the error channel,
+	// proving that subsequent duplicate error updates were suppressed.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if v, err := watcher.resourceErrCh.Receive(sCtx); err == nil {
+		t.Fatalf("Unexpected duplicate error received by watcher: %v", v)
+	}
+}
+
+// TestADS_NACKError_DuplicateSuppression_MultipleResources verifies that
+// when multiple invalid resources cause errors, and a subsequent update carries
+// a duplicate error for only a subset of resources, the client still suppresses
+// the duplicate error notification to the watcher of the affected resource(s).
+func (s) TestADS_NACKError_DuplicateSuppression_MultipleResources(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	const listenerName1 = "listener-1"
+	const listenerName2 = "listener-2"
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	watcher1 := newListenerWatcher()
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, watcher1)
+	defer ldsCancel1()
+
+	watcher2 := newListenerWatcher()
+	ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, watcher2)
+	defer ldsCancel2()
+
+	// Update 1: Management server sends two invalid listener resources.
+	resources1 := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			badListenerResource(t, listenerName1),
+			badListenerResource(t, listenerName2),
+		},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	if err := mgmtServer.Update(ctx, resources1); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources1, err)
+	}
+
+	// Verify that watcher1 receives the initial error for listener 1.
+	v1, err := watcher1.resourceErrCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout waiting for listener 1 error from management server: %v", err)
+	}
+	const wantNackErr = "no RouteSpecifier"
+	if gotErr := v1.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
+		t.Fatalf("Update 1 listener 1 received with error: %v, want %q", gotErr, wantNackErr)
+	}
+
+	// Verify that watcher2 receives the initial error for listener 2.
+	v2, err := watcher2.resourceErrCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout waiting for listener 2 error from management server: %v", err)
+	}
+
+	if gotErr := v2.(listenerUpdateErrTuple).resourceErr; !strings.Contains(gotErr.Error(), wantNackErr) {
+		t.Fatalf("Update 1 listener 2 received with error: %v, want %q", gotErr, wantNackErr)
+	}
+
+	// Update 2: Management server sends listener 1 still invalid, but listener
+	// 2 now valid.
+	resources2 := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			badListenerResource(t, listenerName1),
+			e2e.DefaultClientListener(listenerName2, "route-config-2"),
+		},
+		SkipValidation: true,
+	}
+
+	if err := mgmtServer.Update(ctx, resources2); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources2, err)
+	}
+
+	// Wait for watcher2 to receive the valid resource update for listener 2.
+	if _, err := watcher2.updateCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout waiting for listener 2 resource update: %v", err)
+	}
+
+	// Verify that no duplicate error update was written to watcher 1.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if v, err := watcher1.resourceErrCh.Receive(sCtx); err == nil {
+		t.Fatalf("Unexpected duplicate error received by watcher 1: %v", v)
+	}
+}

internal/xds/clients/xdsclient/test/ads_stream_backoff_test.go

@@ -0,0 +1,444 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+	"google.golang.org/protobuf/testing/protocmp"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+)
+
+func overrideStreamBackOff(t *testing.T, streamBackOff func(int) time.Duration) {
+	originalStreamBackoff := xdsclientinternal.StreamBackoff
+	xdsclientinternal.StreamBackoff = streamBackOff
+	t.Cleanup(func() { xdsclientinternal.StreamBackoff = originalStreamBackoff })
+}
+
+// Creates an xDS client with the given management server address, nodeID and backoff function.
+func createXDSClientWithBackoff(t *testing.T, mgmtServerAddress string, nodeID string, streamBackoff func(int) time.Duration) *xdsclient.XDSClient {
+	t.Helper()
+	overrideStreamBackOff(t, streamBackoff)
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	return createXDSClient(t, mgmtServerAddress, nodeID, grpctransport.NewBuilder(configs))
+}
+
+// Tests the case where the management server returns an error in the ADS
+// streaming RPC. Verifies that the ADS stream is restarted after a backoff
+// period, and that the previously requested resources are re-requested on the
+// new stream.
+func (s) TestADS_BackoffAfterStreamFailure(t *testing.T) {
+	// Channels used for verifying different events in the test.
+	streamCloseCh := make(chan struct{}, 1)  // ADS stream is closed.
+	ldsResourcesCh := make(chan []string, 1) // Listener resource names in the discovery request.
+	backoffCh := make(chan struct{}, 1)      // Backoff after stream failure.
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server that returns RPC errors.
+	streamErr := errors.New("ADS stream error")
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			// Push the requested resource names on to a channel.
+			if req.GetTypeUrl() == version.V3ListenerURL {
+				t.Logf("Received LDS request for resources: %v", req.GetResourceNames())
+				select {
+				case ldsResourcesCh <- req.GetResourceNames():
+				case <-ctx.Done():
+				}
+			}
+			// Return an error everytime a request is sent on the stream. This
+			// should cause the transport to backoff before attempting to
+			// recreate the stream.
+			return streamErr
+		},
+		// Push on a channel whenever the stream is closed.
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			select {
+			case streamCloseCh <- struct{}{}:
+			case <-ctx.Done():
+			}
+		},
+	})
+
+	// Override the backoff implementation to push on a channel that is read by
+	// the test goroutine.
+	backoffCtx, backoffCancel := context.WithCancel(ctx)
+	streamBackoff := func(int) time.Duration {
+		select {
+		case backoffCh <- struct{}{}:
+		case <-backoffCtx.Done():
+		}
+		return 0
+	}
+	defer backoffCancel()
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	nodeID := uuid.New().String()
+	client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
+
+	// Register a watch for a listener resource.
+	const listenerName = "listener"
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that an ADS stream is created and an LDS request with the above
+	// resource name is sent.
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the received stream error is reported to the watcher.
+	if err := verifyListenerResourceError(ctx, lw.resourceErrCh, streamErr.Error(), nodeID); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the stream is closed.
+	select {
+	case <-streamCloseCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for stream to be closed after an error")
+	}
+
+	// Verify that the ADS stream backs off before recreating the stream.
+	select {
+	case <-backoffCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for ADS stream to backoff after stream failure")
+	}
+
+	// Verify that the same resource name is re-requested on the new stream.
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
+		t.Fatal(err)
+	}
+
+	// To prevent indefinite blocking during xDS client close, which is caused
+	// by a blocking backoff channel write, cancel the backoff context early
+	// given that the test is complete.
+	backoffCancel()
+
+}
+
+// Tests the case where a stream breaks because the server goes down. Verifies
+// that when the server comes back up, the same resources are re-requested, this
+// time with the previously acked version and an empty nonce.
+func (s) TestADS_RetriesAfterBrokenStream(t *testing.T) {
+	// Channels used for verifying different events in the test.
+	streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1)   // Discovery request is received.
+	streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) // Discovery response is received.
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server listening on a local port.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		// Push the received request on to a channel for the test goroutine to
+		// verify that it matches expectations.
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			select {
+			case streamRequestCh <- req:
+			case <-ctx.Done():
+			}
+			return nil
+		},
+		// Push the response that the management server is about to send on to a
+		// channel. The test goroutine to uses this to extract the version and
+		// nonce, expected on subsequent requests.
+		OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) {
+			select {
+			case streamResponseCh <- resp:
+			case <-ctx.Done():
+			}
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Override the backoff implementation to always return 0, to reduce test
+	// run time. Instead control when the backoff returns by blocking on a
+	// channel, that the test closes.
+	backoffCh := make(chan struct{})
+	streamBackoff := func(int) time.Duration {
+		select {
+		case backoffCh <- struct{}{}:
+		case <-ctx.Done():
+		}
+		return 0
+	}
+
+	// Create an xDS client pointing to the above server.
+	client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that the initial discovery request matches expectation.
+	var gotReq *v3discoverypb.DiscoveryRequest
+	select {
+	case gotReq = <-streamRequestCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for discovery request on the stream")
+	}
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "user-agent",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Capture the version and nonce from the response.
+	var gotResp *v3discoverypb.DiscoveryResponse
+	select {
+	case gotResp = <-streamResponseCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for discovery response on the stream")
+	}
+	version := gotResp.GetVersionInfo()
+	nonce := gotResp.GetNonce()
+
+	// Verify that the ACK contains the appropriate version and nonce.
+	wantReq.VersionInfo = version
+	wantReq.ResponseNonce = nonce
+	select {
+	case gotReq = <-streamRequestCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for the discovery request ACK on the stream")
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+
+	// Verify the update received by the watcher.
+	wantUpdate := listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: routeConfigName},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Bring down the management server to simulate a broken stream.
+	lis.Stop()
+
+	// Verify that the error callback on the watcher is not invoked.
+	verifyNoListenerUpdate(ctx, lw.updateCh)
+
+	// Wait for backoff to kick in, and unblock the first backoff attempt.
+	select {
+	case <-backoffCh:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for stream backoff")
+	}
+
+	// Bring up the management server. The test does not have prcecise control
+	// over when new streams to the management server will start succeeding. The
+	// ADS stream implementation will backoff as many times as required before
+	// it can successfully create a new stream. Therefore, we need to receive on
+	// the backoffCh as many times as required, and unblock the backoff
+	// implementation.
+	lis.Restart()
+	go func() {
+		for {
+			select {
+			case <-backoffCh:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	// Verify that the transport creates a new stream and sends out a new
+	// request which contains the previously acked version, but an empty nonce.
+	wantReq.ResponseNonce = ""
+	select {
+	case gotReq = <-streamRequestCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for the discovery request ACK on the stream")
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+}
+
+// Tests the case where a resource is requested before the a valid ADS stream
+// exists. Verifies that the a discovery request is sent out for the previously
+// requested resource once a valid stream is created.
+func (s) TestADS_ResourceRequestedBeforeStreamCreation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Channels used for verifying different events in the test.
+	streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received.
+
+	// Create an xDS management server listening on a local port.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+	streamErr := errors.New("ADS stream error")
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+
+		// Return an error everytime a request is sent on the stream. This
+		// should cause the transport to backoff before attempting to recreate
+		// the stream.
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			select {
+			case streamRequestCh <- req:
+			default:
+			}
+			return streamErr
+		},
+	})
+
+	// Bring down the management server before creating the transport. This
+	// allows us to test the case where SendRequest() is called when there is no
+	// stream to the management server.
+	lis.Stop()
+
+	// Override the backoff implementation to always return 0, to reduce test
+	// run time. Instead control when the backoff returns by blocking on a
+	// channel, that the test closes.
+	backoffCh := make(chan struct{}, 1)
+	unblockBackoffCh := make(chan struct{})
+	streamBackoff := func(int) time.Duration {
+		select {
+		case backoffCh <- struct{}{}:
+		default:
+		}
+		<-unblockBackoffCh
+		return 0
+	}
+
+	// Create an xDS client with bootstrap pointing to the above server.
+	nodeID := uuid.New().String()
+	client := createXDSClientWithBackoff(t, mgmtServer.Address, nodeID, streamBackoff)
+
+	// Register a watch for a listener resource.
+	const listenerName = "listener"
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// The above watch results in an attempt to create a new stream, which will
+	// fail, and will result in backoff. Wait for backoff to kick in.
+	select {
+	case <-backoffCh:
+	case <-ctx.Done():
+		t.Fatal("Timeout waiting for stream backoff")
+	}
+
+	// Bring up the connection to the management server, and unblock the backoff
+	// implementation.
+	lis.Restart()
+	close(unblockBackoffCh)
+
+	// Verify that the initial discovery request matches expectation.
+	var gotReq *v3discoverypb.DiscoveryRequest
+	select {
+	case gotReq = <-streamRequestCh:
+	case <-ctx.Done():
+		t.Fatalf("Timeout waiting for discovery request on the stream")
+	}
+	wantReq := &v3discoverypb.DiscoveryRequest{
+		VersionInfo: "",
+		Node: &v3corepb.Node{
+			Id:                   nodeID,
+			UserAgentName:        "user-agent",
+			UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+			ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+		},
+		ResourceNames: []string{listenerName},
+		TypeUrl:       "type.googleapis.com/envoy.config.listener.v3.Listener",
+		ResponseNonce: "",
+	}
+	if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff)
+	}
+}
+
+// waitForResourceNames waits for the wantNames to be received on namesCh.
+// Returns a non-nil error if the context expires before that.
+func waitForResourceNames(ctx context.Context, t *testing.T, namesCh chan []string, wantNames []string) error {
+	t.Helper()
+
+	var lastRequestedNames []string
+	for ; ; <-time.After(defaultTestShortTimeout) {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("timeout waiting for resources %v to be requested from the management server. Last requested resources: %v", wantNames, lastRequestedNames)
+		case gotNames := <-namesCh:
+			if cmp.Equal(gotNames, wantNames, cmpopts.EquateEmpty(), cmpopts.SortSlices(func(s1, s2 string) bool { return s1 < s2 })) {
+				return nil
+			}
+			lastRequestedNames = gotNames
+		}
+	}
+}

internal/xds/clients/xdsclient/test/ads_stream_flow_control_test.go

@@ -0,0 +1,625 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"sort"
+	"testing"
+	"time"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+)
+
+// blockingListenerWatcher implements xdsresource.ListenerWatcher. It writes to
+// a channel when it receives a callback from the watch. It also makes the
+// DoneNotifier passed to the callback available to the test, thereby enabling
+// the test to block this watcher for as long as required.
+type blockingListenerWatcher struct {
+	doneNotifierCh chan func()   // DoneNotifier passed to the callback.
+	updateCh       chan struct{} // Written to when an update is received.
+	ambientErrCh   chan struct{} // Written to when an ambient error is received.
+	resourceErrCh  chan struct{} // Written to when a resource error is received.
+}
+
+func newBLockingListenerWatcher() *blockingListenerWatcher {
+	return &blockingListenerWatcher{
+		doneNotifierCh: make(chan func(), 1),
+		updateCh:       make(chan struct{}, 1),
+		ambientErrCh:   make(chan struct{}, 1),
+		resourceErrCh:  make(chan struct{}, 1),
+	}
+}
+
+func (lw *blockingListenerWatcher) ResourceChanged(_ xdsclient.ResourceData, done func()) {
+	// Notify receipt of the update.
+	select {
+	case lw.updateCh <- struct{}{}:
+	default:
+	}
+
+	select {
+	case lw.doneNotifierCh <- done:
+	default:
+	}
+}
+
+func (lw *blockingListenerWatcher) ResourceError(_ error, done func()) {
+	// Notify receipt of an error.
+	select {
+	case lw.resourceErrCh <- struct{}{}:
+	default:
+	}
+
+	select {
+	case lw.doneNotifierCh <- done:
+	default:
+	}
+}
+
+func (lw *blockingListenerWatcher) AmbientError(_ error, done func()) {
+	// Notify receipt of an error.
+	select {
+	case lw.ambientErrCh <- struct{}{}:
+	default:
+	}
+
+	select {
+	case lw.doneNotifierCh <- done:
+	default:
+	}
+}
+
+type transportBuilder struct {
+	adsStreamCh chan *stream
+}
+
+func (b *transportBuilder) Build(si clients.ServerIdentifier) (clients.Transport, error) {
+	cc, err := grpc.NewClient(si.ServerURI, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.ForceCodec(&byteCodec{})))
+	if err != nil {
+		return nil, err
+	}
+
+	return &transport{cc: cc, adsStreamCh: b.adsStreamCh}, nil
+}
+
+type transport struct {
+	cc          *grpc.ClientConn
+	adsStreamCh chan *stream
+}
+
+func (t *transport) NewStream(ctx context.Context, method string) (clients.Stream, error) {
+	s, err := t.cc.NewStream(ctx, &grpc.StreamDesc{ClientStreams: true, ServerStreams: true}, method)
+	if err != nil {
+		return nil, err
+	}
+
+	stream := &stream{
+		stream: s,
+		recvCh: make(chan struct{}, 1),
+	}
+	t.adsStreamCh <- stream
+
+	return stream, nil
+}
+
+func (t *transport) Close() {
+	t.cc.Close()
+}
+
+type stream struct {
+	stream grpc.ClientStream
+	recvCh chan struct{}
+}
+
+func (s *stream) Send(msg []byte) error {
+	return s.stream.SendMsg(msg)
+}
+
+func (s *stream) Recv() ([]byte, error) {
+	select {
+	case s.recvCh <- struct{}{}:
+	case <-s.stream.Context().Done():
+		// Unblock the recv() once the stream is done.
+	}
+
+	var typedRes []byte
+	if err := s.stream.RecvMsg(&typedRes); err != nil {
+		return nil, err
+	}
+	return typedRes, nil
+}
+
+type byteCodec struct{}
+
+func (c *byteCodec) Marshal(v any) ([]byte, error) {
+	if b, ok := v.([]byte); ok {
+		return b, nil
+	}
+	return nil, fmt.Errorf("transport: message is %T, but must be a []byte", v)
+}
+
+func (c *byteCodec) Unmarshal(data []byte, v any) error {
+	if b, ok := v.(*[]byte); ok {
+		*b = data
+		return nil
+	}
+	return fmt.Errorf("transport: target is %T, but must be *[]byte", v)
+}
+
+func (c *byteCodec) Name() string {
+	return "transport.byteCodec"
+}
+
+// Tests ADS stream level flow control with a single resource. The test does the
+// following:
+//   - Starts a management server and configures a listener resource on it.
+//   - Creates an xDS client to the above management server, starts a couple of
+//     listener watchers for the above resource, and verifies that the update
+//     reaches these watchers.
+//   - These watchers don't invoke the onDone callback until explicitly
+//     triggered by the test. This allows the test to verify that the next
+//     Recv() call on the ADS stream does not happen until both watchers have
+//     completely processed the update, i.e invoke the onDone callback.
+//   - Resource is updated on the management server, and the test verifies that
+//     the update reaches the watchers.
+func (s) TestADSFlowControl_ResourceUpdates_SingleResource(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+
+	// Create an xDS client pointing to the above server with a test transport
+	// that allow monitoring the underlying stream through adsStreamCh.
+	adsStreamCh := make(chan *stream, 1)
+	client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
+
+	// Configure two watchers for the same listener resource.
+	const listenerResourceName = "test-listener-resource"
+	const routeConfigurationName = "test-route-configuration-resource"
+	watcher1 := newBLockingListenerWatcher()
+	cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher1)
+	defer cancel1()
+	watcher2 := newBLockingListenerWatcher()
+	cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher2)
+	defer cancel2()
+
+	// Wait for the ADS stream to be created.
+	var adsStream *stream
+	select {
+	case adsStream = <-adsStreamCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be created")
+	}
+
+	// Configure the listener resource on the management server.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be read from")
+	}
+
+	// Wait for the update to reach the watchers.
+	select {
+	case <-watcher1.updateCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for update to reach watcher 1")
+	}
+	select {
+	case <-watcher2.updateCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for update to reach watcher 2")
+	}
+
+	// Update the listener resource on the management server to point to a new
+	// route configuration resource.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, "new-route")},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Unblock one watcher.
+	onDone := <-watcher1.doneNotifierCh
+	onDone()
+
+	// Wait for a short duration and ensure that there is no read on the stream.
+	select {
+	case <-adsStream.recvCh:
+		t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the second watcher.
+	onDone = <-watcher2.doneNotifierCh
+	onDone()
+
+	// Ensure that there is a read on the stream, now that the previous update
+	// has been consumed by all watchers.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
+	}
+
+	// Wait for the new update to reach the watchers.
+	select {
+	case <-watcher1.updateCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for update to reach watcher 1")
+	}
+	select {
+	case <-watcher2.updateCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for update to reach watcher 2")
+	}
+
+	// At this point, the xDS client is shut down (and the associated transport
+	// is closed) without the watchers invoking their respective onDone
+	// callbacks. This verifies that the closing a transport that has pending
+	// watchers does not block.
+}
+
+// Tests ADS stream level flow control with a multiple resources. The test does
+// the following:
+//   - Starts a management server and configures two listener resources on it.
+//   - Creates an xDS client to the above management server, starts a couple of
+//     listener watchers for the two resources, and verifies that the update
+//     reaches these watchers.
+//   - These watchers don't invoke the onDone callback until explicitly
+//     triggered by the test. This allows the test to verify that the next
+//     Recv() call on the ADS stream does not happen until both watchers have
+//     completely processed the update, i.e invoke the onDone callback.
+func (s) TestADSFlowControl_ResourceUpdates_MultipleResources(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server.
+	const listenerResourceName1 = "test-listener-resource-1"
+	const listenerResourceName2 = "test-listener-resource-2"
+	wantResourceNames := []string{listenerResourceName1, listenerResourceName2}
+	requestCh := make(chan struct{}, 1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			if req.GetTypeUrl() != version.V3ListenerURL {
+				return nil
+			}
+			gotResourceNames := req.GetResourceNames()
+			sort.Slice(gotResourceNames, func(i, j int) bool { return req.ResourceNames[i] < req.ResourceNames[j] })
+			if slices.Equal(gotResourceNames, wantResourceNames) {
+				// The two resource names will be part of the initial request
+				// and also the ACK. Hence, we need to make this write
+				// non-blocking.
+				select {
+				case requestCh <- struct{}{}:
+				default:
+				}
+			}
+			return nil
+		},
+	})
+
+	nodeID := uuid.New().String()
+
+	// Create an xDS client pointing to the above server with a test transport
+	// that allow monitoring the underlying stream through adsStreamCh.
+	adsStreamCh := make(chan *stream, 1)
+	client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
+
+	// Configure two watchers for two different listener resources.
+	const routeConfigurationName1 = "test-route-configuration-resource-1"
+	watcher1 := newBLockingListenerWatcher()
+	cancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName1, watcher1)
+	defer cancel1()
+	const routeConfigurationName2 = "test-route-configuration-resource-2"
+	watcher2 := newBLockingListenerWatcher()
+	cancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName2, watcher2)
+	defer cancel2()
+
+	// Wait for the wrapped ADS stream to be created.
+	var adsStream *stream
+	select {
+	case adsStream = <-adsStreamCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be created")
+	}
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be read from")
+	}
+
+	// Wait for both resource names to be requested.
+	select {
+	case <-requestCh:
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for both resource names to be requested")
+	}
+
+	// Configure the listener resources on the management server.
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			e2e.DefaultClientListener(listenerResourceName1, routeConfigurationName1),
+			e2e.DefaultClientListener(listenerResourceName2, routeConfigurationName2),
+		},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// At this point, we expect the management server to send both resources in
+	// the same response. So, both watchers would be notified at the same time,
+	// and no more Recv() calls should happen until both of them have invoked
+	// their respective onDone() callbacks.
+
+	// The order of callback invocations among the two watchers is not
+	// guaranteed. So, we select on both of them and unblock the first watcher
+	// whose callback is invoked.
+	var otherWatcherUpdateCh chan struct{}
+	var otherWatcherDoneCh chan func()
+	select {
+	case <-watcher1.updateCh:
+		onDone := <-watcher1.doneNotifierCh
+		onDone()
+		otherWatcherUpdateCh = watcher2.updateCh
+		otherWatcherDoneCh = watcher2.doneNotifierCh
+	case <-watcher2.updateCh:
+		onDone := <-watcher2.doneNotifierCh
+		onDone()
+		otherWatcherUpdateCh = watcher1.updateCh
+		otherWatcherDoneCh = watcher1.doneNotifierCh
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for update to reach first watchers")
+	}
+
+	// Wait for a short duration and ensure that there is no read on the stream.
+	select {
+	case <-adsStream.recvCh:
+		t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Wait for the update on the second watcher and unblock it.
+	select {
+	case <-otherWatcherUpdateCh:
+		onDone := <-otherWatcherDoneCh
+		onDone()
+	case <-ctx.Done():
+		t.Fatal("Timed out waiting for update to reach second watcher")
+	}
+
+	// Ensure that there is a read on the stream, now that the previous update
+	// has been consumed by all watchers.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
+	}
+}
+
+// Test ADS stream flow control with a single resource that is expected to be
+// NACKed by the xDS client and the watcher's ResourceError() callback is
+// expected to be invoked because resource is not cached. Verifies that no
+// further reads are attempted until the error is completely processed by the
+// watcher.
+func (s) TestADSFlowControl_ResourceErrors(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+
+	// Create an xDS client pointing to the above server with a test transport
+	// that allow monitoring the underlying stream through adsStreamCh.
+	adsStreamCh := make(chan *stream, 1)
+	client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
+
+	// Configure a watcher for a listener resource.
+	const listenerResourceName = "test-listener-resource"
+	watcher := newBLockingListenerWatcher()
+	cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher)
+	defer cancel()
+
+	// Wait for the stream to be created.
+	var adsStream *stream
+	select {
+	case adsStream = <-adsStreamCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be created")
+	}
+
+	// Configure the management server to return a single listener resource
+	// which is expected to be NACKed by the client.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{badListenerResource(t, listenerResourceName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be read from")
+	}
+
+	// Wait for the resource error to reach the watcher.
+	select {
+	case <-watcher.resourceErrCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for error to reach watcher")
+	}
+
+	// Wait for a short duration and ensure that there is no read on the stream.
+	select {
+	case <-adsStream.recvCh:
+		t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock one watcher.
+	onDone := <-watcher.doneNotifierCh
+	onDone()
+
+	// Ensure that there is a read on the stream, now that the previous error
+	// has been consumed by the watcher.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream after all watchers have processed the previous update")
+	}
+}
+
+// Test ADS stream flow control with a single resource that is deleted from the
+// management server and therefore the watcher's ResourceError()
+// callback is expected to be invoked. Verifies that no further reads are
+// attempted until the callback is completely handled by the watcher.
+func (s) TestADSFlowControl_ResourceDoesNotExist(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+
+	// Create an xDS client pointing to the above server with a test transport
+	// that allow monitoring the underlying stream through adsStreamCh.
+	adsStreamCh := make(chan *stream, 1)
+	client := createXDSClient(t, mgmtServer.Address, nodeID, &transportBuilder{adsStreamCh: adsStreamCh})
+
+	// Configure a watcher for a listener resource.
+	const listenerResourceName = "test-listener-resource"
+	const routeConfigurationName = "test-route-configuration-resource"
+	watcher := newBLockingListenerWatcher()
+	cancel = client.WatchResource(xdsresource.V3ListenerURL, listenerResourceName, watcher)
+	defer cancel()
+
+	// Wait for the ADS stream to be created.
+	var adsStream *stream
+	select {
+	case adsStream = <-adsStreamCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for ADS stream to be created")
+	}
+
+	// Configure the listener resource on the management server.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
+	}
+
+	// Wait for the update to reach the watcher and unblock it.
+	select {
+	case <-watcher.updateCh:
+		onDone := <-watcher.doneNotifierCh
+		onDone()
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for update to reach watcher 1")
+	}
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
+	}
+
+	// Remove the listener resource on the management server.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Wait for the resource not found callback to be invoked.
+	select {
+	case <-watcher.resourceErrCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for resource not found callback to be invoked on the watcher")
+	}
+
+	// Wait for a short duration and ensure that there is no read on the stream.
+	select {
+	case <-adsStream.recvCh:
+		t.Fatal("Recv() called on the ADS stream before all watchers have processed the previous update")
+	case <-time.After(defaultTestShortTimeout):
+	}
+
+	// Unblock the watcher.
+	onDone := <-watcher.doneNotifierCh
+	onDone()
+
+	// Ensure that there is a read on the stream.
+	select {
+	case <-adsStream.recvCh:
+	case <-ctx.Done():
+		t.Fatalf("Timed out waiting for Recv() to be called on the ADS stream")
+	}
+}

internal/xds/clients/xdsclient/test/ads_stream_restart_test.go

@@ -0,0 +1,202 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/grpc/internal/xds/xdsclient/xdsresource/version"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// Tests that an ADS stream is restarted after a connection failure. Also
+// verifies that if there were any watches registered before the connection
+// failed, those resources are re-requested after the stream is restarted.
+func (s) TestADS_ResourcesAreRequestedAfterStreamRestart(t *testing.T) {
+	// Create a restartable listener that can simulate a broken ADS stream.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server that uses a couple of channels to inform
+	// the test about the specific LDS and CDS resource names being requested.
+	ldsResourcesCh := make(chan []string, 2)
+	streamOpened := make(chan struct{}, 1)
+	streamClosed := make(chan struct{}, 1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error {
+			t.Logf("Received request for resources: %v of type %s", req.GetResourceNames(), req.GetTypeUrl())
+
+			// Drain the resource name channels before writing to them to ensure
+			// that the most recently requested names are made available to the
+			// test.
+			if req.GetTypeUrl() == version.V3ListenerURL {
+				select {
+				case <-ldsResourcesCh:
+				default:
+				}
+				ldsResourcesCh <- req.GetResourceNames()
+			}
+			return nil
+		},
+		OnStreamClosed: func(int64, *v3corepb.Node) {
+			select {
+			case streamClosed <- struct{}{}:
+			default:
+			}
+
+		},
+		OnStreamOpen: func(context.Context, int64, string) error {
+			select {
+			case streamOpened <- struct{}{}:
+			default:
+			}
+			return nil
+		},
+	})
+
+	// Create a listener resource on the management server.
+	const listenerName = "listener"
+	const routeConfigName = "route-config"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an xDS client pointing to the above server.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Register a watch for a listener resource.
+	lw := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, lw)
+	defer ldsCancel()
+
+	// Verify that an ADS stream is opened and an LDS request with the above
+	// resource name is sent.
+	select {
+	case <-streamOpened:
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to open")
+	}
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the update received by the watcher.
+	wantListenerUpdate := listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: routeConfigName,
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantListenerUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create another listener resource on the management server, in addition
+	// to the existing listener resource.
+	const listenerName2 = "listener2"
+	const routeConfigName2 = "route-config2"
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName), e2e.DefaultClientListener(listenerName2, routeConfigName2)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register a watch for another listener resource, and verify that a LDS request
+	// with the both listener resource names are sent.
+	lw2 := newListenerWatcher()
+	ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, lw2)
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName, listenerName2}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify the update received by the watcher.
+	wantListenerUpdate = listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: routeConfigName2,
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw2.updateCh, wantListenerUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cancel the watch for the second listener resource, and verify that an LDS
+	// request with only first listener resource names is sent.
+	ldsCancel2()
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the restartable listener and wait for the stream to close.
+	lis.Stop()
+	select {
+	case <-streamClosed:
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to close")
+	}
+
+	// Restart the restartable listener and wait for the stream to open.
+	lis.Restart()
+	select {
+	case <-streamOpened:
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to open")
+	}
+
+	// Verify that the first listener resource is requested again.
+	if err := waitForResourceNames(ctx, t, ldsResourcesCh, []string{listenerName}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Wait for a short duration and verify that no LDS request is sent, since
+	// there are no resources being watched.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	select {
+	case <-sCtx.Done():
+	case names := <-ldsResourcesCh:
+		t.Fatalf("LDS request sent for resource names %v, when expecting no request", names)
+	}
+}

internal/xds/clients/xdsclient/test/ads_stream_watch_test.go

@@ -0,0 +1,222 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils/xds/e2e"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	xdsclientinternal "google.golang.org/grpc/internal/xds/clients/xdsclient/internal"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+)
+
+func waitForResourceWatchState(ctx context.Context, client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error {
+	var lastErr error
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		err := verifyResourceWatchState(client, resourceName, wantState, wantTimer)
+		if err == nil {
+			break
+		}
+		lastErr = err
+	}
+	if ctx.Err() != nil {
+		return fmt.Errorf("timeout when waiting for expected watch state for resource %q: %v", resourceName, lastErr)
+	}
+	return nil
+}
+
+func verifyResourceWatchState(client *xdsclient.XDSClient, resourceName string, wantState xdsresource.WatchState, wantTimer bool) error {
+	resourceWatchStateForTesting := xdsclientinternal.ResourceWatchStateForTesting.(func(*xdsclient.XDSClient, xdsclient.ResourceType, string) (xdsresource.ResourceWatchState, error))
+	gotState, err := resourceWatchStateForTesting(client, listenerType, resourceName)
+	if err != nil {
+		return fmt.Errorf("failed to get watch state for resource %q: %v", resourceName, err)
+	}
+	if gotState.State != wantState {
+		return fmt.Errorf("watch state for resource %q is %v, want %v", resourceName, gotState.State, wantState)
+	}
+	if (gotState.ExpiryTimer != nil) != wantTimer {
+		return fmt.Errorf("expiry timer for resource %q is %t, want %t", resourceName, gotState.ExpiryTimer != nil, wantTimer)
+	}
+	return nil
+}
+
+// Tests the state transitions of the resource specific watch state within the
+// ADS stream, specifically when the stream breaks (for both resources that have
+// been previously received and for resources that are yet to be received).
+func (s) TestADS_WatchState_StreamBreaks(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create an xDS management server with a restartable listener.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lis})
+
+	// Create an xDS client pointing to the above server.
+	nodeID := uuid.New().String()
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Create a watch for the first listener resource and verify that the timer
+	// is running and the watch state is `requested`.
+	const listenerName1 = "listener1"
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName1, noopListenerWatcher{})
+	defer ldsCancel1()
+	if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateRequested, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the first resource on the management server. This should result
+	// in the resource being pushed to the xDS client and should result in the
+	// timer getting stopped and the watch state moving to `received`.
+	const routeConfigName = "route-config"
+	listenerResource1 := e2e.DefaultClientListener(listenerName1, routeConfigName)
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource1},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForResourceWatchState(ctx, client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a watch for the second listener resource and verify that the timer
+	// is running and the watch state is `requested`.
+	const listenerName2 = "listener2"
+	ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, listenerName2, noopListenerWatcher{})
+	defer ldsCancel2()
+	if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the server to break the ADS stream. Since the first resource was
+	// already received, this should not change anything for it. But for the
+	// second resource, it should result in the timer getting stopped and the
+	// watch state moving to `started`.
+	lis.Stop()
+	if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateStarted, false); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Restart the server and verify that the timer is running and the watch
+	// state is `requested`, for the second resource. For the first resource,
+	// nothing should change.
+	lis.Restart()
+	if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateRequested, true); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyResourceWatchState(client, listenerName1, xdsresource.ResourceWatchStateReceived, false); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the second resource on the management server. This should result
+	// in the resource being pushed to the xDS client and should result in the
+	// timer getting stopped and the watch state moving to `received`.
+	listenerResource2 := e2e.DefaultClientListener(listenerName2, routeConfigName)
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{listenerResource1, listenerResource2},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatal(err)
+	}
+	if err := waitForResourceWatchState(ctx, client, listenerName2, xdsresource.ResourceWatchStateReceived, false); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the behavior of the xDS client when a resource watch timer expires and
+// verifies the resource watch state transitions as expected.
+func (s) TestADS_WatchState_TimerFires(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Start an xDS management server.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create an xDS client with bootstrap pointing to the above server, and a
+	// short resource expiry timeout.
+	nodeID := uuid.New().String()
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	t.Cleanup(func() { client.Close() })
+	// Create a watch for the first listener resource and verify that the timer
+	// is running and the watch state is `requested`.
+	const listenerName = "listener"
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, listenerName, noopListenerWatcher{})
+	defer ldsCancel1()
+	if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateRequested, true); err != nil {
+		t.Fatal(err)
+	}
+
+	// Since the resource is not configured on the management server, the watch
+	// expiry timer is expected to fire, and the watch state should move to
+	// `timeout`.
+	if err := waitForResourceWatchState(ctx, client, listenerName, xdsresource.ResourceWatchStateTimeout, false); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/clients/xdsclient/test/authority_test.go

@@ -0,0 +1,334 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+)
+
+const (
+	testAuthority1 = "test-authority1"
+	testAuthority2 = "test-authority2"
+	testAuthority3 = "test-authority3"
+)
+
+var (
+	// These two resources use `testAuthority1`, which contains an empty server
+	// config and therefore will use the default management server.
+	authorityTestResourceName11 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName, nil)
+	authorityTestResourceName12 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName+"2", nil)
+	// This resource uses `testAuthority2`, which contains an empty server
+	// config and therefore will use the default management server.
+	authorityTestResourceName2 = buildResourceName(listenerResourceTypeName, testAuthority2, ldsName+"3", nil)
+	// This resource uses `testAuthority3`, which contains a non-empty server
+	// config, and therefore will use the non-default management server.
+	authorityTestResourceName3 = buildResourceName(listenerResourceTypeName, testAuthority3, ldsName+"3", nil)
+)
+
+// setupForAuthorityTests spins up two management servers, one to act as the
+// default and the other to act as the non-default. It also creates a
+// xDS client configuration with three authorities (the first two pointing to
+// the default and the third one pointing to the non-default).
+//
+// Returns two listeners used by the default and non-default management servers
+// respectively, and the xDS client.
+func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, *xdsclient.XDSClient) {
+	// Create listener wrappers which notify on to a channel whenever a new
+	// connection is accepted. We use this to track the number of transports
+	// used by the xDS client.
+	lisDefault := testutils.NewListenerWrapper(t, nil)
+	lisNonDefault := testutils.NewListenerWrapper(t, nil)
+
+	// Start a management server to act as the default authority.
+	defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
+
+	// Start a management server to act as the non-default authority.
+	nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
+
+	// Create a bootstrap configuration with two non-default authorities which
+	// have empty server configs, and therefore end up using the default server
+	// config, which points to the above management server.
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	ext := grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}
+	siDefault := clients.ServerIdentifier{
+		ServerURI:  defaultAuthorityServer.Address,
+		Extensions: ext,
+	}
+	siNonDefault := clients.ServerIdentifier{
+		ServerURI:  nonDefaultAuthorityServer.Address,
+		Extensions: ext,
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: siDefault}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp style resource names used in this test use a slash removed
+		// version of t.Name as their authority, and the empty config
+		// results in the top-level xds server configuration being used for
+		// this authority.
+		Authorities: map[string]xdsclient.Authority{
+			testAuthority1: {XDSServers: []xdsclient.ServerConfig{}},
+			testAuthority2: {XDSServers: []xdsclient.ServerConfig{}},
+			testAuthority3: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: siNonDefault}}},
+		},
+		WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			e2e.DefaultClientListener(authorityTestResourceName11, rdsName),
+			e2e.DefaultClientListener(authorityTestResourceName12, rdsName),
+			e2e.DefaultClientListener(authorityTestResourceName2, rdsName),
+			e2e.DefaultClientListener(authorityTestResourceName3, rdsName),
+		},
+		SkipValidation: true,
+	}
+	if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+	return lisDefault, lisNonDefault, client
+}
+
+// Tests the xdsChannel sharing logic among authorities. The test verifies the
+// following scenarios:
+//   - A watch for a resource name with an authority matching an existing watch
+//     should not result in a new transport being created.
+//   - A watch for a resource name with different authority name but same
+//     authority config as an existing watch should not result in a new transport
+//     being created.
+func (s) TestAuthority_XDSChannelSharing(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	lis, _, client := setupForAuthorityTests(ctx, t)
+	defer client.Close()
+
+	// Verify that no connection is established to the management server at this
+	// point. A transport is created only when a resource (which belongs to that
+	// authority) is requested.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Request the first resource. Verify that a new transport is created.
+	watcher := noopListenerWatcher{}
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
+	defer ldsCancel1()
+	if _, err := lis.NewConnCh.Receive(ctx); err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+
+	// Request the second resource. Verify that no new transport is created.
+	ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
+	defer ldsCancel2()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Request the third resource. Verify that no new transport is created.
+	ldsCancel3 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName2, watcher)
+	defer ldsCancel3()
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+}
+
+// Test the xdsChannel close logic. The test verifies that the xDS client
+// closes an xdsChannel immediately after the last watch is canceled.
+func (s) TestAuthority_XDSChannelClose(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	lis, _, client := setupForAuthorityTests(ctx, t)
+	defer client.Close()
+
+	// Request the first resource. Verify that a new transport is created.
+	watcher := noopListenerWatcher{}
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
+	val, err := lis.NewConnCh.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
+	}
+	conn := val.(*testutils.ConnWrapper)
+
+	// Request the second resource. Verify that no new transport is created.
+	ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("Unexpected new transport created to management server")
+	}
+
+	// Cancel both watches, and verify that the connection to the management
+	// server is closed.
+	ldsCancel1()
+	ldsCancel2()
+	if _, err := conn.CloseCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for connection to management server to be closed")
+	}
+}
+
+// Tests the scenario where the primary management server is unavailable at
+// startup and the xDS client falls back to the secondary.  The test verifies
+// that the resource watcher is not notified of the connectivity failure until
+// all servers have failed.
+func (s) TestAuthority_Fallback(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create primary and secondary management servers with restartable
+	// listeners.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	primaryLis := testutils.NewRestartableListener(l)
+	primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
+	l, err = net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	secondaryLis := testutils.NewRestartableListener(l)
+	secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
+
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	psi := clients.ServerIdentifier{
+		ServerURI:  primaryMgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	ssi := clients.ServerIdentifier{
+		ServerURI:  secondaryMgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	// Create config with the above primary and fallback management servers,
+	// and an xDS client with that configuration.
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: psi}, {ServerIdentifier: ssi}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	const listenerName = "listener"
+	const rdsPrimaryName = "rds-primary"
+	const rdsSecondaryName = "rds-secondary"
+
+	// Create a Cluster resource on the primary.
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsPrimaryName)},
+		SkipValidation: true,
+	}
+	if err := primaryMgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Create a Cluster resource on the secondary .
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsSecondaryName)},
+		SkipValidation: true,
+	}
+	if err := secondaryMgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Stop the primary.
+	primaryLis.Close()
+
+	// Register a watch.
+	watcher := newListenerWatcher()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
+	defer ldsCancel()
+
+	// Ensure that the connectivity error callback is not called. Since, this
+	// is the first watch without cached resource, it checks for resourceErrCh
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if v, err := watcher.resourceErrCh.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatalf("Resource error callback on the watcher with error:  %v", v.(error))
+	}
+
+	// Ensure that the resource update callback is invoked.
+	wantUpdate := listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: rdsSecondaryName,
+		},
+	}
+	if err := verifyListenerUpdate(ctx, watcher.updateCh, wantUpdate); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the secondary.
+	secondaryLis.Close()
+
+	// Ensure that the connectivity error callback is called as ambient error
+	// since cached resource exist.
+	if _, err := watcher.ambientErrCh.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for ambient error callback on the watcher")
+	}
+}

internal/xds/clients/xdsclient/test/custom_resource_watch_test.go

@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
+
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// customTestResourceType is a fake resource type used for testing.
+var customTestResourceType = xdsclient.ResourceType{
+	TypeURL:                    "type.googleapis.com/google.protobuf.StringValue",
+	TypeName:                   "CustomResource",
+	AllResourcesRequiredInSotW: false,
+	Decoder:                    customTestDecoder{},
+}
+
+type customTestDecoder struct{}
+
+func (customTestDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
+	any := resource.ToAny()
+	if any.GetTypeUrl() != customTestResourceType.TypeURL {
+		return nil, fmt.Errorf("unexpected resource type: %q", any.GetTypeUrl())
+	}
+	var val wrapperspb.StringValue
+	if err := proto.Unmarshal(any.GetValue(), &val); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal resource: %v", err)
+	}
+
+	// We expect the value to be "name|content". This allows us to extract the
+	// name which is required by the xDS client to match the watch.
+	str := val.GetValue()
+	name, content, found := strings.Cut(str, "|")
+	if !found {
+		return nil, fmt.Errorf("invalid format: expected 'name|content', got %q", str)
+	}
+
+	return &xdsclient.DecodeResult{
+		Name:     name,
+		Resource: &customTestResourceData{val: content},
+	}, nil
+}
+
+type customTestResourceData struct {
+	val string
+}
+
+func (c *customTestResourceData) Equal(other xdsclient.ResourceData) bool {
+	if c == nil && other == nil {
+		return true
+	}
+	if c == nil || other == nil {
+		return false
+	}
+	o, ok := other.(*customTestResourceData)
+	if !ok {
+		return false
+	}
+	return c.val == o.val
+}
+
+func (c *customTestResourceData) Bytes() []byte {
+	return []byte(c.val)
+}
+
+// customTestWatcher is a watcher for the custom resource type.
+type customTestWatcher struct {
+	updateCh chan xdsclient.ResourceData
+	errCh    chan error
+}
+
+func newCustomTestWatcher() *customTestWatcher {
+	return &customTestWatcher{
+		updateCh: make(chan xdsclient.ResourceData, 1),
+		errCh:    make(chan error, 1),
+	}
+}
+
+func (w *customTestWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) {
+	w.updateCh <- update
+	onDone()
+}
+
+func (w *customTestWatcher) ResourceError(err error, onDone func()) {
+	w.errCh <- fmt.Errorf("ResourceError: %v", err)
+	onDone()
+}
+
+func (w *customTestWatcher) AmbientError(err error, onDone func()) {
+	w.errCh <- fmt.Errorf("AmbientError: %v", err)
+	onDone()
+}
+
+// Tests that the xDS client can watch a custom resource type that is injected
+// via the config.
+func (s) TestCustomResourceWatch(t *testing.T) {
+	const authority = "my-authority"
+	resourceName := "xdstp://" + authority + "/" + customTestResourceType.TypeURL + "/my-resource"
+
+	tests := []struct {
+		name          string
+		resourceValue string
+		wantUpdate    string
+		wantNACK      string
+	}{
+		{
+			name:          "valid_resource",
+			resourceValue: resourceName + "|hello world",
+			wantUpdate:    "hello world",
+		},
+		{
+			name:          "decode_error",
+			resourceValue: "malformed-value",
+			wantNACK:      "invalid format: expected 'name|content'",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// Start a fake xDS management server.
+			mgmtServer, cleanup, err := fakeserver.StartServer(nil)
+			if err != nil {
+				t.Fatalf("Failed to start fake xDS server: %v", err)
+			}
+			defer cleanup()
+
+			resourceTypes := map[string]xdsclient.ResourceType{
+				customTestResourceType.TypeURL: customTestResourceType,
+			}
+			si := clients.ServerIdentifier{
+				ServerURI:  mgmtServer.Address,
+				Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+			}
+
+			configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+			nodeID := uuid.New().String()
+			xdsClientConfig := xdsclient.Config{
+				Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+				Node:             clients.Node{ID: nodeID},
+				TransportBuilder: grpctransport.NewBuilder(configs),
+				ResourceTypes:    resourceTypes,
+				Authorities: map[string]xdsclient.Authority{
+					authority: {XDSServers: []xdsclient.ServerConfig{}},
+				},
+			}
+
+			client, err := xdsclient.New(xdsClientConfig)
+			if err != nil {
+				t.Fatalf("Failed to create xDS client: %v", err)
+			}
+			defer client.Close()
+
+			watcher := newCustomTestWatcher()
+			cancelWatch := client.WatchResource(customTestResourceType.TypeURL, resourceName, watcher)
+			defer cancelWatch()
+
+			// Wait for the discovery request.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+
+			// Verify the request type URL.
+			v, err := mgmtServer.XDSRequestChan.Receive(ctx)
+			if err != nil {
+				t.Fatalf("Timeout when waiting for a DiscoveryRequest message: %v", err)
+			}
+			req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
+			if req.GetTypeUrl() != customTestResourceType.TypeURL {
+				t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL)
+			}
+			if len(req.GetResourceNames()) != 1 || req.GetResourceNames()[0] != resourceName {
+				t.Fatalf("DiscoveryRequest ResourceNames = %v, want [%v]", req.GetResourceNames(), resourceName)
+			}
+
+			// Send a response with the custom resource.
+			resource, err := anypb.New(&wrapperspb.StringValue{Value: test.resourceValue})
+			if err != nil {
+				t.Fatalf("Failed to marshal resource: %v", err)
+			}
+			mgmtServer.XDSResponseChan <- &fakeserver.Response{
+				Resp: &v3discoverypb.DiscoveryResponse{
+					TypeUrl:     customTestResourceType.TypeURL,
+					VersionInfo: "1",
+					Resources:   []*anypb.Any{resource},
+				},
+			}
+
+			if test.wantNACK != "" {
+				// Verify the NACK.
+				// We expect a new DiscoveryRequest with ErrorDetail set.
+				v, err = mgmtServer.XDSRequestChan.Receive(ctx)
+				if err != nil {
+					t.Fatalf("Timeout when waiting for NACK DiscoveryRequest message: %v", err)
+				}
+				req = v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
+				if req.GetTypeUrl() != customTestResourceType.TypeURL {
+					t.Fatalf("DiscoveryRequest TypeUrl = %v, want %v", req.GetTypeUrl(), customTestResourceType.TypeURL)
+				}
+				if req.GetErrorDetail() == nil {
+					t.Fatalf("DiscoveryRequest ErrorDetail is nil, want non-nil")
+				}
+				if !strings.Contains(req.GetErrorDetail().GetMessage(), test.wantNACK) {
+					t.Fatalf("DiscoveryRequest ErrorDetail = %v, want substring %q", req.GetErrorDetail().GetMessage(), test.wantNACK)
+				}
+				return
+			}
+
+			// Verify the update.
+			select {
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for resource update")
+			case err := <-watcher.errCh:
+				t.Fatalf("Received unexpected error: %v", err)
+			case got := <-watcher.updateCh:
+				gotData, ok := got.(*customTestResourceData)
+				if !ok {
+					t.Fatalf("Received unexpected data type: %T", got)
+				}
+				if gotData.val != test.wantUpdate {
+					t.Fatalf("Received resource value %q, want %q", gotData.val, test.wantUpdate)
+				}
+			}
+		})
+	}
+}

internal/xds/clients/xdsclient/test/dump_test.go

@@ -0,0 +1,557 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/pretty"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3"
+)
+
+func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any, failure *v3adminpb.UpdateFailureState) *v3statuspb.ClientConfig_GenericXdsConfig {
+	return &v3statuspb.ClientConfig_GenericXdsConfig{
+		TypeUrl:      typeURL,
+		Name:         name,
+		VersionInfo:  version,
+		ClientStatus: status,
+		XdsConfig:    config,
+		ErrorState:   failure,
+	}
+}
+
+func checkResourceDump(ctx context.Context, want *v3statuspb.ClientStatusResponse, client *xdsclient.XDSClient) error {
+	var cmpOpts = cmp.Options{
+		protocmp.Transform(),
+		protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"),
+		protocmp.IgnoreFields((*v3statuspb.ClientConfig)(nil), "client_scope"),
+		protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"),
+	}
+
+	var lastErr error
+	for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) {
+		b, err := client.DumpResources()
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		got := &v3statuspb.ClientStatusResponse{}
+		if err := proto.Unmarshal(b, got); err != nil {
+			lastErr = err
+			continue
+		}
+		// Sort the client configs based on the `client_scope` field.
+		slices.SortFunc(got.GetConfig(), func(a, b *v3statuspb.ClientConfig) int {
+			return strings.Compare(a.ClientScope, b.ClientScope)
+		})
+		// Sort the resource configs based on the type_url and name fields.
+		for _, cc := range got.GetConfig() {
+			slices.SortFunc(cc.GetGenericXdsConfigs(), func(a, b *v3statuspb.ClientConfig_GenericXdsConfig) int {
+				if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 {
+					return strings.Compare(a.Name, b.Name)
+				}
+				return strings.Compare(a.TypeUrl, b.TypeUrl)
+			})
+		}
+		diff := cmp.Diff(want, got, cmpOpts)
+		if diff == "" {
+			return nil
+		}
+		lastErr = fmt.Errorf("received unexpected resource dump, diff (-got, +want):\n%s, got: %s\n want:%s", diff, pretty.ToJSON(got), pretty.ToJSON(want))
+	}
+	return fmt.Errorf("timeout when waiting for resource dump to reach expected state: %v", lastErr)
+}
+
+// Tests the scenario where there are multiple xDS clients talking to the same
+// management server, and requesting the same set of resources. Verifies that
+// under all circumstances, both xDS clients receive the same configuration from
+// the server.
+func (s) TestDumpResources_ManyToOne(t *testing.T) {
+	// Initialize the xDS resources to be used in this test.
+	ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"}
+	rdsTargets := []string{"route-config-0", "route-config-1"}
+	listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
+	listenerAnys := make([]*anypb.Any, len(ldsTargets))
+	for i := range ldsTargets {
+		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
+		listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
+	}
+
+	// Spin up an xDS management server on a local port.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create two xDS clients with the above config.
+	client1, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client1.Close()
+	client2, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client2.Close()
+
+	// Dump resources and expect empty configs.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantNode := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+	}
+	wantResp := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node: wantNode,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register watches, dump resources and expect configs in requested state.
+	for _, xdsC := range []*xdsclient.XDSClient{client1, client2} {
+		for _, target := range ldsTargets {
+			xdsC.WatchResource(xdsresource.V3ListenerURL, target, noopListenerWatcher{})
+		}
+	}
+	wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the resources on the management server.
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Dump resources and expect ACK configs.
+	wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the first resource of each type in the management server to a
+	// value which is expected to be NACK'ed by the xDS client.
+	listeners[0] = func() *v3listenerpb.Listener {
+		hcm := testutils.MarshalAny(t, &v3httppb.HttpConnectionManager{})
+		return &v3listenerpb.Listener{
+			Name:        ldsTargets[0],
+			ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm},
+		}
+	}()
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners,
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_NACKED, listenerAnys[0], &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests the scenario where there are multiple xDS client talking to different
+// management server, and requesting different set of resources.
+func (s) TestDumpResources_ManyToMany(t *testing.T) {
+	// Initialize the xDS resources to be used in this test:
+	// - The first xDS client watches old style resource names, and thereby
+	//   requests these resources from the top-level xDS server.
+	// - The second xDS client watches new style resource names with a non-empty
+	//   authority, and thereby requests these resources from the server
+	//   configuration for that authority.
+	authority := strings.Join(strings.Split(t.Name(), "/"), "")
+	ldsTargets := []string{
+		"lds.target.good:0000",
+		fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/lds.targer.good:1111", authority),
+	}
+	rdsTargets := []string{
+		"route-config-0",
+		fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/route-config-1", authority),
+	}
+	listeners := make([]*v3listenerpb.Listener, len(ldsTargets))
+	listenerAnys := make([]*anypb.Any, len(ldsTargets))
+	for i := range ldsTargets {
+		listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i])
+		listenerAnys[i] = testutils.MarshalAny(t, listeners[i])
+	}
+
+	// Start two management servers.
+	mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// The first of the above management servers will be the top-level xDS
+	// server in the bootstrap configuration, and the second will be the xDS
+	// server corresponding to the test authority.
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{}
+	listenerType := listenerType
+	resourceTypes[xdsresource.V3ListenerURL] = listenerType
+	si1 := clients.ServerIdentifier{
+		ServerURI:  mgmtServer1.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	si2 := clients.ServerIdentifier{
+		ServerURI:  mgmtServer2.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si1}},
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp style resource names used in this test use a slash removed
+		// version of t.Name as their authority, and the empty config
+		// results in the top-level xds server configuration being used for
+		// this authority.
+		Authorities: map[string]xdsclient.Authority{
+			authority: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: si2}}},
+		},
+	}
+
+	// Create two xDS clients with the above config.
+	client1, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client1.Close()
+	client2, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client2.Close()
+
+	// Check the resource dump before configuring resources on the management server.
+	// Dump resources and expect empty configs.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	wantNode := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+	}
+	wantResp := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node: wantNode,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register watches, the first xDS client watches old style resource names,
+	// while the second xDS client watches new style resource names.
+	client1.WatchResource(xdsresource.V3ListenerURL, ldsTargets[0], noopListenerWatcher{})
+	client2.WatchResource(xdsresource.V3ListenerURL, ldsTargets[1], noopListenerWatcher{})
+
+	// Check the resource dump. Both clients should have all resources in
+	// REQUESTED state.
+	wantConfigs1 := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantConfigs2 := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "", v3adminpb.ClientResourceStatus_REQUESTED, nil, nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure resources on the first management server.
+	if err := mgmtServer1.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners[:1],
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the resource dump. One client should have resources in ACKED state,
+	// while the other should still have resources in REQUESTED state.
+	wantConfigs1 = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[0], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[0], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure resources on the second management server.
+	if err := mgmtServer2.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      listeners[1:],
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Check the resource dump. Both clients should have appropriate resources
+	// in REQUESTED state.
+	wantConfigs2 = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[1], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[1], nil),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs1,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client1); err != nil {
+		t.Fatal(err)
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs2,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client2); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests that when a resource update is NACKed by the xDS client, the resource
+// dump mentions that the xDS client is using the old good version of the
+// resource, but contains the error state for the most recent bad update.
+func (s) TestDumpResources_ACK_NACK(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Spin up an xDS management server on a local port.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	nodeID := uuid.New().String()
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	client := createXDSClient(t, mgmtServer.Address, nodeID, grpctransport.NewBuilder(configs))
+
+	// Start watching a listener resource.
+	client.WatchResource(xdsresource.V3ListenerURL, ldsName, noopListenerWatcher{})
+
+	// Configure the management server with a good version of the listener resource.
+	goodListener := e2e.DefaultClientListener(ldsName, rdsName)
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{goodListener},
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Dump resources and expect ACK config with version "1".
+	wantNode := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw"},
+	}
+	goodListenerAny := testutils.MarshalAny(t, goodListener)
+	wantConfigs := []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_ACKED, goodListenerAny, nil),
+	}
+	wantResp := &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update the management server with a bad version of the listener resource
+	// which is expected to be NACKed by the xDS client.
+	badListener := badListenerResource(t, ldsName)
+	if err := mgmtServer.Update(ctx, e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{badListener},
+		SkipValidation: true,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Dump resources and verify that the xDS client is using the old good
+	// version of the resource ("1"), but contains the error state for the most
+	// recent bad update ("2").
+	wantConfigs = []*v3statuspb.ClientConfig_GenericXdsConfig{
+		makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsName, "1", v3adminpb.ClientResourceStatus_NACKED, goodListenerAny, &v3adminpb.UpdateFailureState{VersionInfo: "2"}),
+	}
+	wantResp = &v3statuspb.ClientStatusResponse{
+		Config: []*v3statuspb.ClientConfig{
+			{
+				Node:              wantNode,
+				GenericXdsConfigs: wantConfigs,
+			},
+		},
+	}
+	if err := checkResourceDump(ctx, wantResp, client); err != nil {
+		t.Fatal(err)
+	}
+}

internal/xds/clients/xdsclient/test/helpers_test.go

@@ -0,0 +1,390 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/internal/buffer"
+	"google.golang.org/grpc/internal/xds/clients/internal/pretty"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
+	"github.com/google/go-cmp/cmp"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	defaultTestWatchExpiryTimeout = 500 * time.Millisecond
+	defaultTestTimeout            = 10 * time.Second
+	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
+
+	// ListenerResourceTypeName represents the transport agnostic name for the
+	// listener resource.
+	listenerResourceTypeName = "ListenerResource"
+
+	ldsName         = "xdsclient-test-lds-resource"
+	rdsName         = "xdsclient-test-rds-resource"
+	ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource"
+	rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource"
+)
+
+var (
+	// Singleton instantiation of the resource type implementation.
+	listenerType = xdsclient.ResourceType{
+		TypeURL:                    xdsresource.V3ListenerURL,
+		TypeName:                   listenerResourceTypeName,
+		AllResourcesRequiredInSotW: true,
+		Decoder:                    listenerDecoder{},
+	}
+)
+
+func unmarshalListenerResource(rProto *anypb.Any) (string, listenerUpdate, error) {
+	rProto, err := xdsresource.UnwrapResource(rProto)
+	if err != nil {
+		return "", listenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err)
+	}
+	if !xdsresource.IsListenerResource(rProto.GetTypeUrl()) {
+		return "", listenerUpdate{}, fmt.Errorf("unexpected listener resource type: %q", rProto.GetTypeUrl())
+	}
+	lis := &v3listenerpb.Listener{}
+	if err := proto.Unmarshal(rProto.GetValue(), lis); err != nil {
+		return "", listenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err)
+	}
+
+	lu, err := processListener(lis)
+	if err != nil {
+		return lis.GetName(), listenerUpdate{}, err
+	}
+	lu.Raw = rProto.GetValue()
+	return lis.GetName(), *lu, nil
+}
+
+func processListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
+	if lis.GetApiListener() != nil {
+		return processClientSideListener(lis)
+	}
+	return processServerSideListener(lis)
+}
+
+// processClientSideListener checks if the provided Listener proto meets
+// the expected criteria. If so, it returns a non-empty routeConfigName.
+func processClientSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
+	update := &listenerUpdate{}
+
+	apiLisAny := lis.GetApiListener().GetApiListener()
+	if !xdsresource.IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) {
+		return nil, fmt.Errorf("unexpected http connection manager resource type: %q", apiLisAny.GetTypeUrl())
+	}
+	apiLis := &v3httppb.HttpConnectionManager{}
+	if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal api_listener: %v", err)
+	}
+
+	switch apiLis.RouteSpecifier.(type) {
+	case *v3httppb.HttpConnectionManager_Rds:
+		if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil {
+			return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis)
+		}
+		name := apiLis.GetRds().GetRouteConfigName()
+		if name == "" {
+			return nil, fmt.Errorf("empty route_config_name: %+v", lis)
+		}
+		update.RouteConfigName = name
+	case *v3httppb.HttpConnectionManager_RouteConfig:
+		routeU := apiLis.GetRouteConfig()
+		if routeU == nil {
+			return nil, fmt.Errorf("empty inline RDS resp:: %+v", lis)
+		}
+		if routeU.Name == "" {
+			return nil, fmt.Errorf("empty route_config_name in inline RDS resp: %+v", lis)
+		}
+		update.RouteConfigName = routeU.Name
+	case nil:
+		return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis)
+	default:
+		return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier)
+	}
+
+	return update, nil
+}
+
+func processServerSideListener(lis *v3listenerpb.Listener) (*listenerUpdate, error) {
+	if n := len(lis.ListenerFilters); n != 0 {
+		return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n)
+	}
+	if lis.GetUseOriginalDst().GetValue() {
+		return nil, errors.New("unsupported field 'use_original_dst' is present and set to true")
+	}
+	addr := lis.GetAddress()
+	if addr == nil {
+		return nil, fmt.Errorf("no address field in LDS response: %+v", lis)
+	}
+	sockAddr := addr.GetSocketAddress()
+	if sockAddr == nil {
+		return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis)
+	}
+	lu := &listenerUpdate{
+		InboundListenerCfg: &inboundListenerConfig{
+			Address: sockAddr.GetAddress(),
+			Port:    strconv.Itoa(int(sockAddr.GetPortValue())),
+		},
+	}
+
+	return lu, nil
+}
+
+type listenerDecoder struct{}
+
+// Decode deserializes and validates an xDS resource serialized inside the
+// provided `Any` proto, as received from the xDS management server.
+func (listenerDecoder) Decode(resource *xdsclient.AnyProto, _ xdsclient.DecodeOptions) (*xdsclient.DecodeResult, error) {
+	name, listener, err := unmarshalListenerResource(resource.ToAny())
+	switch {
+	case name == "":
+		// Name is unset only when protobuf deserialization fails.
+		return nil, err
+	case err != nil:
+		// Protobuf deserialization succeeded, but resource validation failed.
+		return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listenerUpdate{}}}, err
+	}
+
+	return &xdsclient.DecodeResult{Name: name, Resource: &listenerResourceData{Resource: listener}}, nil
+
+}
+
+// listenerResourceData wraps the configuration of a Listener resource as
+// received from the management server.
+//
+// Implements the ResourceData interface.
+type listenerResourceData struct {
+	xdsclient.ResourceData
+
+	Resource listenerUpdate
+}
+
+// Equal returns true if other is equal to l.
+func (l *listenerResourceData) Equal(other xdsclient.ResourceData) bool {
+	if l == nil && other == nil {
+		return true
+	}
+	if (l == nil) != (other == nil) {
+		return false
+	}
+	return bytes.Equal(l.Resource.Raw, other.Bytes())
+}
+
+// ToJSON returns a JSON string representation of the resource data.
+func (l *listenerResourceData) ToJSON() string {
+	return pretty.ToJSON(l.Resource)
+}
+
+// Bytes returns the underlying raw protobuf form of the listener resource.
+func (l *listenerResourceData) Bytes() []byte {
+	return l.Resource.Raw
+}
+
+// ListenerUpdate contains information received in an LDS response, which is of
+// interest to the registered LDS watcher.
+type listenerUpdate struct {
+	// RouteConfigName is the route configuration name corresponding to the
+	// target which is being watched through LDS.
+	RouteConfigName string
+
+	// InboundListenerCfg contains inbound listener configuration.
+	InboundListenerCfg *inboundListenerConfig
+
+	// Raw is the resource from the xds response.
+	Raw []byte
+}
+
+// InboundListenerConfig contains information about the inbound listener, i.e
+// the server-side listener.
+type inboundListenerConfig struct {
+	// Address is the local address on which the inbound listener is expected to
+	// accept incoming connections.
+	Address string
+	// Port is the local port on which the inbound listener is expected to
+	// accept incoming connections.
+	Port string
+}
+
+func makeAuthorityName(name string) string {
+	segs := strings.Split(name, "/")
+	return strings.Join(segs, "")
+}
+
+func makeNewStyleLDSName(authority string) string {
+	return fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource", authority)
+}
+
+// buildResourceName returns the resource name in the format of an xdstp://
+// resource.
+func buildResourceName(typeName, auth, id string, ctxParams map[string]string) string {
+	var typS string
+	switch typeName {
+	case listenerResourceTypeName:
+		typS = "envoy.config.listener.v3.Listener"
+	default:
+		// If the name doesn't match any of the standard resources fallback
+		// to the type name.
+		typS = typeName
+	}
+	return (&xdsresource.Name{
+		Scheme:        "xdstp",
+		Authority:     auth,
+		Type:          typS,
+		ID:            id,
+		ContextParams: ctxParams,
+	}).String()
+}
+
+// testMetricsReporter is a MetricsReporter to be used in tests. It sends
+// recording events on channels and provides helpers to check if certain events
+// have taken place.
+type testMetricsReporter struct {
+	metricsCh *buffer.Unbounded[any]
+
+	mu             sync.Mutex
+	asyncReporters map[clients.AsyncReporter]struct{}
+}
+
+// newTestMetricsReporter returns a new testMetricsReporter.
+func newTestMetricsReporter() *testMetricsReporter {
+	return &testMetricsReporter{
+		metricsCh:      buffer.NewUnbounded[any](),
+		asyncReporters: make(map[clients.AsyncReporter]struct{}),
+	}
+}
+
+// waitForMetric waits for a metric to be recorded and verifies that the
+// recorded metrics data matches the expected metricsDataWant. Returns
+// an error if failed to wait or received wrong data.
+func (r *testMetricsReporter) waitForMetric(ctx context.Context, metricsDataWant any) error {
+	select {
+	case got := <-r.metricsCh.Get():
+		r.metricsCh.Load()
+		if diff := cmp.Diff(got, metricsDataWant); diff != "" {
+			return fmt.Errorf("received unexpected metrics value (-got, +want): %v", diff)
+		}
+		return nil
+	case <-ctx.Done():
+		return fmt.Errorf("timeout waiting for metric check: %v", ctx.Err())
+	}
+}
+
+// waitForSpecificMetric waits for a specific metric to be recorded, ignoring
+// and discarding any other metrics received in the meantime. Returns an error
+// if the context expires before the requested metric is received.
+//
+// This is necessary when multiple distinct metrics (e.g., ServerFailure,
+// ResourceUpdateValid, and XDSClientConnected) might be emitted asynchronously
+// and concurrently. Using waitForMetric would fail the test if an unrelated
+// metric happens to be at the front of the channel.
+func (r *testMetricsReporter) waitForSpecificMetric(ctx context.Context, metricsDataWant any) error {
+	for {
+		select {
+		case got := <-r.metricsCh.Get():
+			r.metricsCh.Load()
+			if diff := cmp.Diff(got, metricsDataWant); diff == "" {
+				return nil
+			}
+		case <-ctx.Done():
+			return fmt.Errorf("timeout waiting for specific metric: %v", ctx.Err())
+		}
+	}
+}
+
+// Receive waits for a metric to be recorded and returns it. Returns an error
+// if the context expires before a metric is received.
+func (r *testMetricsReporter) Receive(ctx context.Context) (any, error) {
+	select {
+	case got := <-r.metricsCh.Get():
+		r.metricsCh.Load()
+		return got, nil
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// Drain clears all accumulated metrics from the channel.
+func (r *testMetricsReporter) Drain() {
+	for {
+		select {
+		case <-r.metricsCh.Get():
+			r.metricsCh.Load()
+		default:
+			return
+		}
+	}
+}
+
+// ReportMetric sends the metrics data to the metricsCh channel.
+func (r *testMetricsReporter) ReportMetric(m any) {
+	r.metricsCh.Put(m)
+}
+
+func (r *testMetricsReporter) RegisterAsyncReporter(reporter clients.AsyncReporter) func() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.asyncReporters[reporter] = struct{}{}
+	return sync.OnceFunc(func() {
+		r.mu.Lock()
+		defer r.mu.Unlock()
+		delete(r.asyncReporters, reporter)
+	})
+}
+
+// triggerAsyncMetrics invokes the Report method on all registered asynchronous
+// metric reporters. This allows tests to explicitly compel the emission of
+// async gauge metrics at specific precise points in time for verification.
+func (r *testMetricsReporter) triggerAsyncMetrics() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for reporter := range r.asyncReporters {
+		reporter.Report(r)
+	}
+}
+
+func (r *testMetricsReporter) numAsyncReporters() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return len(r.asyncReporters)
+}

internal/xds/clients/xdsclient/test/metrics_test.go

@@ -0,0 +1,911 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/metrics"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// TestResourceUpdateMetrics configures an xDS client, and a management server
+// to send valid and invalid LDS updates, and verifies that the expected metrics
+// for both good and bad updates are emitted.
+func (s) TestResourceUpdateMetrics(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
+	const listenerResourceName = "test-listener-resource"
+	const routeConfigurationName = "test-route-configuration-resource"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	// Watch the valid listener configured on the management server. This should
+	// cause a resource update valid metric to emit eventually.
+	client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Update management server with a bad update. This should cause a resource
+	// update invalid metric to emit eventually.
+	resources = e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	resources.Listeners[0].ApiListener = nil
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Resource update valid metric should have not emitted.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if err := tmr.waitForMetric(sCtx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err == nil {
+		t.Fatal("tmr.WaitForInt64Count(ctx, mdWant) succeeded when expected to timeout.")
+	}
+}
+
+// TestServerFailureMetrics_BeforeResponseRecv configures an xDS client, and a
+// management server. It then register a watcher and stops the management
+// server before sending a resource update, and verifies that the expected
+// metric for server failure is emitted.
+func (s) TestServerFailureMetrics_BeforeResponseRecv(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+
+	lis := testutils.NewRestartableListener(l)
+	streamOpened := make(chan struct{}, 1)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamOpen: func(context.Context, int64, string) error {
+			select {
+			case streamOpened <- struct{}{}:
+			default:
+			}
+			return nil
+		},
+	})
+
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	const listenerResourceName = "test-listener-resource"
+
+	// Watch for the listener on the above management server.
+	client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
+	// Verify that an ADS stream is opened and an LDS request with the above
+	// resource name is sent.
+	select {
+	case <-streamOpened:
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for ADS stream to open")
+	}
+
+	// Close the listener and ensure that the ADS stream breaks. This should
+	// cause a server failure metric to emit eventually.
+	lis.Stop()
+
+	// Restart to prevent the attempt to create a new ADS stream after back off.
+	lis.Restart()
+
+	if err := tmr.waitForMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestServerFailureMetrics_AfterResponseRecv configures an xDS client and a
+// management server to send a valid LDS update, and verifies that the
+// successful update metric is emitted. When the client ACKs the update, the
+// server returns an error, breaking the stream. The test then verifies that the
+// server failure metric is not emitted, because the ADS stream was closed after
+// a response was received on the stream. Finally, the test waits for the client
+// to establish a new stream and verifies that the client emits a metric after
+// receiving a successful update.
+func (s) TestServerFailureMetrics_AfterResponseRecv(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+	streamCreationQuota := make(chan struct{}, 1)
+	streamCreationQuota <- struct{}{}
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamOpen: func(context.Context, int64, string) error {
+			// The following select block is used to block stream creation after
+			// the first stream has failed, but while we are waiting to verify
+			// that the failure metric is not reported.
+			select {
+			case <-streamCreationQuota:
+			case <-ctx.Done():
+			}
+			return nil
+		},
+		OnStreamRequest: func(streamID int64, req *v3discoverypb.DiscoveryRequest) error {
+			// We only want the ACK on the first stream to return an error
+			// (leading to stream closure), without effecting subsequent stream
+			// attempts.
+			if streamID == 1 && req.GetVersionInfo() != "" {
+				return errors.New("test configured error")
+			}
+			return nil
+		}},
+	)
+	const listenerResourceName = "test-listener-resource"
+	const routeConfigurationName = "test-route-configuration-resource"
+	nodeID := uuid.New().String()
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerResourceName, routeConfigurationName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	// Watch the valid listener configured on the management server. This should
+	// cause a resource update valid metric to emit eventually.
+	client.WatchResource(listenerType.TypeURL, listenerResourceName, noopListenerWatcher{})
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// When the client sends an ACK, the management server would reply with an
+	// error, breaking the stream.
+	// Server failure should still have no recording point.
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	failureMetric := &metrics.ServerFailure{ServerURI: mgmtServer.Address}
+	if err := tmr.waitForMetric(sCtx, failureMetric); err == nil {
+		t.Fatalf("tmr.waitForMetric(%v) succeeded when expected to timeout.", failureMetric)
+	} else if sCtx.Err() == nil {
+		t.Fatalf("tmr.WaitForInt64Count(%v) = %v, want context deadline exceeded", failureMetric, err)
+	}
+	// Unblock stream creation and verify that an update is received
+	// successfully.
+	close(streamCreationQuota)
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestConnectedMetric verifies the "grpc.xds_client.connected" metric state
+// transitions. It begins by ensuring no metrics are reported before connection
+// is attempted. Then it establishes a connection by watching a valid resource
+// and verifies the connected state pulses to 1. Finally, it stops the
+// management server and verifies the state drops back to 0.
+func (s) TestConnectedMetric(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
+	nodeID := uuid.New().String()
+
+	xdsClientConfig := xdsclient.Config{
+		Servers: []xdsclient.ServerConfig{{
+			ServerIdentifier: clients.ServerIdentifier{
+				ServerURI:  mgmtServer.Address,
+				Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+			},
+		}},
+		Node: clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
+			"insecure": {Credentials: insecure.NewBundle()},
+		}),
+		ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	tmr.triggerAsyncMetrics()
+	sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if err := tmr.waitForSpecificMetric(sCtx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address}); err == nil {
+		t.Fatal("XDSClientConnected metric reported before any watch was started")
+	}
+
+	const listenerName = "test-listener-resource"
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+	client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
+
+	// Wait for the update to ensure we are connected.
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Now trigger async metrics.
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
+		t.Fatal(err)
+	}
+
+	mgmtServer.Stop()
+
+	// Wait for the synchronous server failure metric to confirm disconnect.
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
+		t.Fatal(err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify async metric reporters are unregistered when client closes.
+	client.Close()
+
+	if count := tmr.numAsyncReporters(); count != 0 {
+		t.Fatalf("Async reporter not unregistered after client close, count: %d", count)
+	}
+
+	// Drain the channel of any leftover metrics from previous pulses.
+	tmr.Drain()
+
+	tmr.triggerAsyncMetrics()
+	// No metrics should be reported now because there are no reporters.
+	sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := tmr.Receive(sCtx); err == nil {
+		t.Fatal("Metrics reported after all reporters were unregistered")
+	}
+}
+
+// TestResourceMetrics verifies that the xDS client correctly tracks resource
+// states (acked, nacked_but_cached) in "grpc.xds_client.resources" metric. It
+// watches a resource, pushes a valid update from the management server, and
+// asserts that the resource transitions to 'acked' state. Then it pushes an
+// invalid update and asserts that the resource transitions to
+// 'nacked_but_cached' state.
+func (s) TestResourceMetrics(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
+	nodeID := uuid.New().String()
+
+	xdsClientConfig := xdsclient.Config{
+		Servers: []xdsclient.ServerConfig{{
+			ServerIdentifier: clients.ServerIdentifier{
+				ServerURI:  mgmtServer.Address,
+				Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+			},
+		}},
+		Node: clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
+			"insecure": {Credentials: insecure.NewBundle()},
+		}),
+		ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	const listenerName = "test-listener-resource"
+	const routeConfigName = "test-route-configuration-resource"
+
+	client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
+
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, routeConfigName)},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Trigger async metrics.
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old", // Default value for old-style non-xdstp as per gRFC A78
+		ResourceType: "ListenerResource",
+		CacheState:   "acked",
+		Count:        1,
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	resources.Listeners[0].ApiListener = nil
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	// Wait for Invalid update.
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateInvalid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old",
+		ResourceType: "ListenerResource",
+		CacheState:   "nacked_but_cached",
+		Count:        1,
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestResourceMetrics_Extended verifies complex resource state transitions
+// (requested, does_not_exist, nacked) in "grpc.xds_client.resources" metric
+// across multiple resources. It watches several resources, pushes a partial
+// update (some valid, some invalid) to transition them to 'acked' and 'nacked'
+// states, then management server removes a resource and asserts that active
+// watchers transition back to 'requested' while omitted resources transition
+// to 'does_not_exist'.
+func (s) TestResourceMetrics_Extended(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	tmr := newTestMetricsReporter()
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: l})
+	nodeID := uuid.New().String()
+
+	xdsClientConfig := xdsclient.Config{
+		Servers: []xdsclient.ServerConfig{{
+			ServerIdentifier: clients.ServerIdentifier{
+				ServerURI:  mgmtServer.Address,
+				Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+			},
+		}},
+		Node: clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
+			"insecure": {Credentials: insecure.NewBundle()},
+		}),
+		ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	resRequested1 := "res-requested-1"
+	resRequested2 := "res-requested-2"
+	resNacked1 := "res-nacked-1"
+	resNacked2 := "res-nacked-2"
+	resRemoved := "res-not-exist"
+
+	resList := []string{resRequested1, resRequested2, resNacked1, resNacked2, resRemoved}
+	for _, res := range resList {
+		client.WatchResource(listenerType.TypeURL, res, noopListenerWatcher{})
+	}
+
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			e2e.DefaultClientListener(resNacked1, "route-config"),
+			e2e.DefaultClientListener(resNacked2, "route-config"),
+			e2e.DefaultClientListener(resRemoved, "route-config"),
+		},
+		SkipValidation: true,
+	}
+	resources.Listeners[0].ApiListener = nil
+	resources.Listeners[1].ApiListener = nil
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	// Wait for the resource update to be accepted.
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+	// Push an empty update to remove all resources from the management server.
+	resourcesEmpty := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resourcesEmpty); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old",
+		ResourceType: "ListenerResource",
+		CacheState:   "requested",
+		Count:        2,
+	}); err != nil {
+		t.Fatalf("Failed to verify requested count: %v", err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old",
+		ResourceType: "ListenerResource",
+		CacheState:   "nacked",
+		Count:        2,
+	}); err != nil {
+		t.Fatalf("Failed to verify nacked count: %v", err)
+	}
+
+	// Wait for the xDS client to process the empty update and trigger
+	// ResourceNotFound for omitted resources immediately.
+	lw := newListenerWatcher()
+	client.WatchResource(listenerType.TypeURL, resRemoved, lw)
+
+	// Verify that resources missing from the authoritative update transition to
+	// the does_not_exist state.
+	if err := verifyResourceErrorType(ctx, lw.resourceErrCh, xdsresource.ErrorTypeResourceNotFound, ""); err != nil {
+		t.Fatal(err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old",
+		ResourceType: "ListenerResource",
+		CacheState:   "does_not_exist",
+		Count:        1,
+	}); err != nil {
+		t.Fatalf("Failed to verify does_not_exist count: %v", err)
+	}
+}
+
+// TestConnectedMetric_Reconnection verifies "grpc.xds_client.connected" metric
+// accuracy during flaky network conditions. It uses a restartable listener and
+// a dial interceptor to pause connection attempts. It asserts that the
+// connected state pulses to 1 when a stream succeeds, remains 1 immediately
+// after failures before retries, transitions to 0 after subsequent failures,
+// and returns to 1 only after successful stream recreation.
+func (s) TestConnectedMetric_Reconnection(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	l, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	sendResponse := make(chan struct{})
+	streamOpened := make(chan struct{}, 10)
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener: lis,
+		OnStreamOpen: func(ctx context.Context, _ int64, _ string) error {
+			select {
+			case streamOpened <- struct{}{}:
+			case <-ctx.Done():
+			}
+			return nil
+		},
+		OnStreamResponse: func(ctx context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, _ *v3discoverypb.DiscoveryResponse) {
+			select {
+			case <-sendResponse:
+			case <-ctx.Done():
+			}
+		},
+	})
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+	// streamAttemptCh is used by the interceptor to notify the test that a
+	// stream is attempting to open.
+	streamAttemptCh := testutils.NewChannel()
+	// commandCh is used by the test to tell the interceptor whether to block or pass.
+	commandCh := testutils.NewChannel()
+	// blocked is used by the interceptor to signal the test that it has
+	// successfully blocked the stream creation attempt.
+	blocked := make(chan struct{})
+	// unblock is used by the test to release the interceptor and allow the
+	// stream creation to proceed.
+	unblock := make(chan struct{})
+
+	// customGRPCNewClient overrides the gRPC client creation to inject a stream
+	// interceptor that can block ADS stream creation attempts for testing
+	// reconnection behavior.
+	customGRPCNewClient := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
+		interceptor := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, cops ...grpc.CallOption) (grpc.ClientStream, error) {
+			streamAttemptCh.Send(struct{}{}) // Notify test
+
+			cmd, err := commandCh.Receive(ctx) // Wait for command
+			if err != nil {
+				return nil, err
+			}
+
+			if cmd.(string) == "block" {
+				select {
+				case blocked <- struct{}{}:
+				case <-ctx.Done():
+				}
+				// Pause stream creation until released by the test.
+				<-unblock
+			}
+			return streamer(ctx, desc, cc, method, cops...)
+		}
+		opts = append(opts, grpc.WithStreamInterceptor(interceptor))
+		return grpc.NewClient(target, opts...)
+	}
+
+	tmr := newTestMetricsReporter()
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle(), GRPCNewClient: customGRPCNewClient}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+		MetricsReporter: tmr,
+	}
+
+	waitForStreamSuccess := func() {
+		for {
+			select {
+			case <-streamOpened:
+				return
+			case <-streamAttemptCh.C:
+				commandCh.Send("pass")
+			case <-ctx.Done():
+				t.Fatalf("Timeout waiting for stream success")
+			}
+		}
+	}
+
+	// Stop listener to force initial connection failure.
+	lis.Stop()
+
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	const listenerName = "test-listener-resource"
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
+		SkipValidation: true,
+	}
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	// Initiate watch to trigger connection attempt.
+	client.WatchResource(listenerType.TypeURL, listenerName, noopListenerWatcher{})
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
+		t.Fatalf("XDSClientConnected check failed at start - got: %v, want 0", err)
+	}
+
+	// Verify state transitions to connected (Value: 1) when stream succeeds.
+	lis.Restart()
+
+	waitForStreamSuccess()
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
+		t.Fatalf("XDSClientConnected check failed after 1st NewStream - got: %v, want 1", err)
+	}
+
+	// Push resource update to confirm the connection is functional.
+	close(sendResponse)
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify state remains connected (Value: 1) immediately after failure before
+	// NewStream retry.
+	lis.Stop()
+
+	if _, err := streamAttemptCh.Receive(ctx); err != nil {
+		t.Fatalf("Timeout waiting for 2nd stream attempt (block): %v", err)
+	}
+	commandCh.Send("block")
+
+	<-blocked
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
+		t.Fatalf("XDSClientConnected check failed while NewStream is blocked - got: %v, want 1", err)
+	}
+
+	// Verify state transitions to disconnected (Value: 0) after NewStream attempt
+	// fails.
+	close(unblock)
+
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.ServerFailure{ServerURI: mgmtServer.Address}); err != nil {
+		t.Fatal(err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
+		t.Fatalf("XDSClientConnected check failed after NewStream failure - got: %v, want 0", err)
+	}
+
+	sendResponse = make(chan struct{})
+
+	// Verify state remains disconnected (Value: 0) while waiting for the first
+	// response on the new stream.
+	lis.Restart()
+
+	// Clear channels to ensure fresh readings.
+	for len(streamOpened) > 0 {
+		<-streamOpened
+	}
+
+	waitForStreamSuccess()
+
+	tmr.Drain()
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 0}); err != nil {
+		t.Fatalf("XDSClientConnected check failed after successful NewStream but before response - got: %v, want 0", err)
+	}
+
+	// Verify state returns to connected (Value: 1) after receiving the first
+	// response on the new stream.
+	close(sendResponse)
+
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientConnected{ServerURI: mgmtServer.Address, Value: 1}); err != nil {
+		t.Fatalf("XDSClientConnected check failed after response- got: %v, want 1", err)
+	}
+}
+
+// TestResourceMetrics_AuthorityOldStyle verifies that the xDS client correctly
+// falls back to '#old' in "grpc.xds_client.resources" metrics for legacy
+// authorities. It watches a resource using default authority and asserts that
+// the metric is reported under '#old' instead of an empty string.
+func (s) TestResourceMetrics_AuthorityOldStyle(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+	nodeID := uuid.New().String()
+
+	serverCfg := xdsclient.ServerConfig{
+		ServerIdentifier: clients.ServerIdentifier{
+			ServerURI:  mgmtServer.Address,
+			Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+		},
+	}
+
+	tmr := newTestMetricsReporter()
+	xdsClientConfig := xdsclient.Config{
+		Servers: []xdsclient.ServerConfig{serverCfg},
+		Node:    clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(map[string]grpctransport.Config{
+			"insecure": {Credentials: insecure.NewBundle()},
+		}),
+		ResourceTypes: map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType},
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{serverCfg}},
+		},
+		MetricsReporter: tmr,
+	}
+
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	const listenerName = "test-listener"
+
+	client.WatchResource(listenerType.TypeURL, listenerName, newListenerWatcher())
+
+	resources := e2e.UpdateOptions{
+		NodeID:         nodeID,
+		Listeners:      []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, "route-config")},
+		SkipValidation: true,
+	}
+
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server: %v", err)
+	}
+
+	if err := tmr.waitForMetric(ctx, &metrics.ResourceUpdateValid{ServerURI: mgmtServer.Address, ResourceType: "ListenerResource"}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify that the client substitutes empty authority with '#old' in metrics.
+	tmr.triggerAsyncMetrics()
+	if err := tmr.waitForSpecificMetric(ctx, &metrics.XDSClientResourceStats{
+		Authority:    "#old",
+		ResourceType: "ListenerResource",
+		CacheState:   "acked",
+		Count:        1,
+	}); err != nil {
+		t.Fatalf("Failed to observe grpc.xds.authority '#old' metric substitution: %v", err)
+	}
+}

internal/xds/clients/xdsclient/test/misc_watchers_test.go

@@ -0,0 +1,529 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package xdsclient_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient"
+	"google.golang.org/grpc/internal/xds/clients/xdsclient/internal/xdsresource"
+	"google.golang.org/protobuf/types/known/anypb"
+
+	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
+	v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+)
+
+// testLDSWatcher is a test watcher that registers two watches corresponding to
+// the names passed in at creation time on a valid update.
+type testLDSWatcher struct {
+	client           *xdsclient.XDSClient
+	name1, name2     string
+	lw1, lw2         *listenerWatcher
+	cancel1, cancel2 func()
+	updateCh         *testutils.Channel
+}
+
+func newTestLDSWatcher(client *xdsclient.XDSClient, name1, name2 string) *testLDSWatcher {
+	return &testLDSWatcher{
+		client:   client,
+		name1:    name1,
+		name2:    name2,
+		lw1:      newListenerWatcher(),
+		lw2:      newListenerWatcher(),
+		updateCh: testutils.NewChannelWithSize(1),
+	}
+}
+
+func (lw *testLDSWatcher) ResourceChanged(update xdsclient.ResourceData, onDone func()) {
+	lisData, ok := update.(*listenerResourceData)
+	if !ok {
+		lw.updateCh.Send(listenerUpdateErrTuple{resourceErr: fmt.Errorf("unexpected resource type: %T", update)})
+		onDone()
+		return
+	}
+	lw.updateCh.Send(listenerUpdateErrTuple{update: lisData.Resource})
+
+	lw.cancel1 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name1, lw.lw1)
+	lw.cancel2 = lw.client.WatchResource(xdsresource.V3ListenerURL, lw.name2, lw.lw2)
+	onDone()
+}
+
+func (lw *testLDSWatcher) AmbientError(err error, onDone func()) {
+	// When used with a go-control-plane management server that continuously
+	// resends resources which are NACKed by the xDS client, using a `Replace()`
+	// here and in OnResourceDoesNotExist() simplifies tests which will have
+	// access to the most recently received error.
+	lw.updateCh.Replace(listenerUpdateErrTuple{ambientErr: err})
+	onDone()
+}
+
+func (lw *testLDSWatcher) ResourceError(_ error, onDone func()) {
+	lw.updateCh.Replace(listenerUpdateErrTuple{resourceErr: xdsresource.NewError(xdsresource.ErrorTypeResourceNotFound, "Listener not found in received response")})
+	onDone()
+}
+
+func (lw *testLDSWatcher) cancel() {
+	lw.cancel1()
+	lw.cancel2()
+}
+
+// TestWatchCallAnotherWatch tests the scenario where a watch is registered for
+// a resource, and more watches are registered from the first watch's callback.
+// The test verifies that this scenario does not lead to a deadlock.
+func (s) TestWatchCallAnotherWatch(t *testing.T) {
+	// Start an xDS management server and set the option to allow it to respond
+	// to requests which only specify a subset of the configured resources.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
+
+	nodeID := uuid.New().String()
+	authority := makeAuthorityName(t.Name())
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp style resource names used in this test use a slash removed
+		// version of t.Name as their authority, and the empty config
+		// results in the top-level xds server configuration being used for
+		// this authority.
+		Authorities: map[string]xdsclient.Authority{
+			authority: {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	// Configure the management server to return two listener resources,
+	// corresponding to the registered watches.
+	ldsNameNewStyle := makeNewStyleLDSName(authority)
+	resources := e2e.UpdateOptions{
+		NodeID: nodeID,
+		Listeners: []*v3listenerpb.Listener{
+			e2e.DefaultClientListener(ldsName, rdsName),
+			e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle),
+		},
+		SkipValidation: true,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	if err := mgmtServer.Update(ctx, resources); err != nil {
+		t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
+	}
+
+	// Create a listener watcher that registers two more watches from
+	// the OnUpdate callback:
+	// - one for the same resource name as this watch, which would be
+	//   satisfied from xdsClient cache
+	// - the other for a different resource name, which would be
+	//   satisfied from the server
+	lw := newTestLDSWatcher(client, ldsName, ldsNameNewStyle)
+	defer lw.cancel()
+	ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, ldsName, lw)
+	defer ldsCancel()
+
+	// Verify the contents of the received update for the all watchers.
+	// Verify the contents of the received update for the all watchers. The two
+	// resources returned differ only in the resource name. Therefore the
+	// expected update is the same for all the watchers.
+	wantUpdate12 := listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: rdsName,
+		},
+	}
+	// Verify the contents of the received update for the all watchers. The two
+	// resources returned differ only in the resource name. Therefore the
+	// expected update is the same for all the watchers.
+	wantUpdate3 := listenerUpdateErrTuple{
+		update: listenerUpdate{
+			RouteConfigName: rdsNameNewStyle,
+		},
+	}
+	if err := verifyListenerUpdate(ctx, lw.updateCh, wantUpdate12); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyListenerUpdate(ctx, lw.lw1.updateCh, wantUpdate12); err != nil {
+		t.Fatal(err)
+	}
+	if err := verifyListenerUpdate(ctx, lw.lw2.updateCh, wantUpdate3); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestNodeProtoSentOnlyInFirstRequest verifies that a non-empty node proto gets
+// sent only on the first discovery request message on the ADS stream.
+//
+// It also verifies the same behavior holds after a stream restart.
+func (s) TestNodeProtoSentOnlyInFirstRequest(t *testing.T) {
+	// Create a restartable listener which can close existing connections.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("Error while listening. Err: %v", err)
+	}
+	lis := testutils.NewRestartableListener(l)
+
+	// Start a fake xDS management server with the above restartable listener.
+	//
+	// We are unable to use the go-control-plane server here, because it caches
+	// the node proto received in the first request message and adds it to
+	// subsequent requests before invoking the OnStreamRequest() callback.
+	// Therefore we cannot verify what is sent by the xDS client.
+	mgmtServer, cleanup, err := fakeserver.StartServer(lis)
+	if err != nil {
+		t.Fatalf("Failed to start fake xDS server: %v", err)
+	}
+	defer cleanup()
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	const (
+		serviceName     = "my-service-client-side-xds"
+		routeConfigName = "route-" + serviceName
+		clusterName     = "cluster-" + serviceName
+		serviceName2    = "my-service-client-side-xds-2"
+	)
+
+	// Register a watch for the Listener resource.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	watcher := newListenerWatcher()
+	ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, serviceName, watcher)
+	defer ldsCancel1()
+
+	// Ensure the watch results in a discovery request with an empty node proto.
+	if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure a listener resource on the fake xDS server.
+	lisAny, err := anypb.New(e2e.DefaultClientListener(serviceName, routeConfigName))
+	if err != nil {
+		t.Fatalf("Failed to marshal listener resource into an Any proto: %v", err)
+	}
+	mgmtServer.XDSResponseChan <- &fakeserver.Response{
+		Resp: &v3discoverypb.DiscoveryResponse{
+			TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+			VersionInfo: "1",
+			Resources:   []*anypb.Any{lisAny},
+		},
+	}
+
+	// The xDS client is expected to ACK the Listener resource. The discovery
+	// request corresponding to the ACK must contain a nil node proto.
+	if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
+		t.Fatal(err)
+	}
+
+	// Register a watch for another Listener resource.
+	ldscancel2 := client.WatchResource(xdsresource.V3ListenerURL, serviceName2, watcher)
+	defer ldscancel2()
+
+	// Ensure the watch results in a discovery request with an empty node proto.
+	if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
+		t.Fatal(err)
+	}
+
+	// Configure the other listener resource on the fake xDS server.
+	lisAny2, err := anypb.New(e2e.DefaultClientListener(serviceName2, routeConfigName))
+
+	if err != nil {
+		t.Fatalf("Failed to marshal route configuration resource into an Any proto: %v", err)
+	}
+
+	mgmtServer.XDSResponseChan <- &fakeserver.Response{
+		Resp: &v3discoverypb.DiscoveryResponse{
+			TypeUrl:     "type.googleapis.com/envoy.config.listener.v3.Listener",
+			VersionInfo: "1",
+			Resources:   []*anypb.Any{lisAny2},
+		},
+	}
+
+	// Ensure the discovery request for the ACK contains an empty node proto.
+	if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
+		t.Fatal(err)
+	}
+
+	// Stop the management server and expect the error callback to be invoked.
+	lis.Stop()
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for the connection error to be propagated to the watcher")
+	case <-watcher.ambientErrCh.C:
+	}
+	// Restart the management server.
+	lis.Restart()
+
+	// The xDS client is expected to re-request previously requested resources.
+	// Here, we expect 1 DiscoveryRequest messages with both the listener resources.
+	// The message should contain a non-nil node proto (since its the first
+	// request after restart).
+	//
+	// And since we don't push any responses on the response channel of the fake
+	// server, we do not expect any ACKs here.
+	if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// readDiscoveryResponseAndCheckForEmptyNodeProto reads a discovery request
+// message out of the provided reqCh. It returns an error if it fails to read a
+// message before the context deadline expires, or if the read message contains
+// a non-empty node proto.
+func readDiscoveryResponseAndCheckForEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error {
+	v, err := reqCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message")
+	}
+	req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
+	if node := req.GetNode(); node != nil {
+		return fmt.Errorf("Node proto received in DiscoveryRequest message is %v, want empty node proto", node)
+	}
+	return nil
+}
+
+// readDiscoveryResponseAndCheckForNonEmptyNodeProto reads a discovery request
+// message out of the provided reqCh. It returns an error if it fails to read a
+// message before the context deadline expires, or if the read message contains
+// an empty node proto.
+func readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error {
+	v, err := reqCh.Receive(ctx)
+	if err != nil {
+		return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message")
+	}
+	req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest)
+	if node := req.GetNode(); node == nil {
+		return fmt.Errorf("Empty node proto received in DiscoveryRequest message, want non-empty node proto")
+	}
+	return nil
+}
+
+// Tests that the errors returned by the xDS client when watching a resource
+// contain the node ID that was used to create the client. This test covers two
+// scenarios:
+//
+//  1. When a watch is registered for an already registered resource type, but
+//     a new watch is registered with a type url which is not present in
+//     provided resource types.
+//  2. When a watch is registered for a resource name whose authority is not
+//     found in the xDS client config.
+func (s) TestWatchErrorsContainNodeID(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	t.Run("Right_Wrong_ResourceType_Implementations", func(t *testing.T) {
+		const listenerName = "listener-name"
+		watcher := newListenerWatcher()
+		client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
+
+		sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+		defer sCancel()
+		select {
+		case <-sCtx.Done():
+		case <-watcher.updateCh.C:
+			t.Fatal("Unexpected resource update")
+		case <-watcher.resourceErrCh.C:
+			t.Fatal("Unexpected resource error")
+		}
+
+		client.WatchResource("non-existent-type-url", listenerName, watcher)
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for error callback to be invoked")
+		case u, ok := <-watcher.resourceErrCh.C:
+			if !ok {
+				t.Fatalf("got no update, wanted listener resource error from the management server")
+			}
+			gotErr := u.(listenerUpdateErrTuple).resourceErr
+			if !strings.Contains(gotErr.Error(), nodeID) {
+				t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+			}
+		}
+	})
+
+	t.Run("Missing_Authority", func(t *testing.T) {
+		const listenerName = "xdstp://nonexistant-authority/envoy.config.listener.v3.Listener/listener-name"
+		watcher := newListenerWatcher()
+		client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
+
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for error callback to be invoked")
+		case u, ok := <-watcher.resourceErrCh.C:
+			if !ok {
+				t.Fatalf("got no update, wanted listener resource error from the management server")
+			}
+			gotErr := u.(listenerUpdateErrTuple).resourceErr
+			if !strings.Contains(gotErr.Error(), nodeID) {
+				t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+			}
+		}
+	})
+}
+
+// erroringTransportBuilder is a transport builder which always returns a nil
+// transport along with an error.
+type erroringTransportBuilder struct{}
+
+func (*erroringTransportBuilder) Build(_ clients.ServerIdentifier) (clients.Transport, error) {
+	return nil, fmt.Errorf("failed to create transport")
+}
+
+// Tests that the errors returned by the xDS client when watching a resource
+// contain the node ID when channel creation to the management server fails.
+func (s) TestWatchErrorsContainNodeID_ChannelCreationFailure(t *testing.T) {
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{})
+
+	// Create bootstrap configuration pointing to the above management server.
+	nodeID := uuid.New().String()
+
+	resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
+	si := clients.ServerIdentifier{
+		ServerURI:  mgmtServer.Address,
+		Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
+	}
+
+	xdsClientConfig := xdsclient.Config{
+		Servers:          []xdsclient.ServerConfig{{ServerIdentifier: si}},
+		Node:             clients.Node{ID: nodeID},
+		TransportBuilder: &erroringTransportBuilder{},
+		ResourceTypes:    resourceTypes,
+		// Xdstp resource names used in this test do not specify an
+		// authority. These will end up looking up an entry with the
+		// empty key in the authorities map. Having an entry with an
+		// empty key and empty configuration, results in these
+		// resources also using the top-level configuration.
+		Authorities: map[string]xdsclient.Authority{
+			"": {XDSServers: []xdsclient.ServerConfig{}},
+		},
+	}
+
+	// Create an xDS client with the above config.
+	client, err := xdsclient.New(xdsClientConfig)
+	if err != nil {
+		t.Fatalf("Failed to create xDS client: %v", err)
+	}
+	defer client.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	const listenerName = "listener-name"
+	watcher := newListenerWatcher()
+	client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
+
+	select {
+	case <-ctx.Done():
+		t.Fatal("Timeout when waiting for error callback to be invoked")
+	case u, ok := <-watcher.resourceErrCh.C:
+		if !ok {
+			t.Fatalf("got no update, wanted listener resource error from the management server")
+		}
+		gotErr := u.(listenerUpdateErrTuple).resourceErr
+		if !strings.Contains(gotErr.Error(), nodeID) {
+			t.Fatalf("Unexpected error: %v, want error with node ID: %q", err, nodeID)
+		}
+	}
+}