snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/xds/clients/lrsclient/internal/internal.go

@@ -0,0 +1,26 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package internal contains functionality internal to the lrsclient package.
+package internal
+
+import "time"
+
+var (
+	// TimeNow is used to get the current time. It can be overridden in tests.
+	TimeNow func() time.Time
+)

internal/xds/clients/lrsclient/load_store.go

@@ -0,0 +1,438 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"google.golang.org/grpc/internal/xds/clients"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+)
+
+// A LoadStore aggregates loads for multiple clusters and services that are
+// intended to be reported via LRS.
+//
+// LoadStore stores loads reported to a single LRS server. Use multiple stores
+// for multiple servers.
+//
+// It is safe for concurrent use.
+type LoadStore struct {
+	// stop is the function to call to Stop the LoadStore reporting.
+	stop func(ctx context.Context)
+
+	// mu only protects the map (2 layers). The read/write to
+	// *PerClusterReporter doesn't need to hold the mu.
+	mu sync.Mutex
+	// clusters is a map with cluster name as the key. The second layer is a
+	// map with service name as the key. Each value (PerClusterReporter)
+	// contains data for a (cluster, service) pair.
+	//
+	// Note that new entries are added to this map, but never removed. This is
+	// potentially a memory leak. But the memory is allocated for each new
+	// (cluster,service) pair, and the memory allocated is just pointers and
+	// maps. So this shouldn't get too bad.
+	clusters map[string]map[string]*PerClusterReporter
+}
+
+func init() {
+	lrsclientinternal.TimeNow = time.Now
+}
+
+// newLoadStore creates a LoadStore.
+func newLoadStore() *LoadStore {
+	return &LoadStore{
+		clusters: make(map[string]map[string]*PerClusterReporter),
+	}
+}
+
+// Stop signals the LoadStore to stop reporting.
+//
+// Before closing the underlying LRS stream, this method may block until a
+// final load report send attempt completes or the provided context `ctx`
+// expires.
+//
+// The provided context must have a deadline or timeout set to prevent Stop
+// from blocking indefinitely if the final send attempt fails to complete.
+//
+// Calling Stop on an already stopped LoadStore is a no-op.
+func (ls *LoadStore) Stop(ctx context.Context) {
+	ls.stop(ctx)
+}
+
+// ReporterForCluster returns the PerClusterReporter for the given cluster and
+// service.
+func (ls *LoadStore) ReporterForCluster(clusterName, serviceName string) *PerClusterReporter {
+	ls.mu.Lock()
+	defer ls.mu.Unlock()
+	c, ok := ls.clusters[clusterName]
+	if !ok {
+		c = make(map[string]*PerClusterReporter)
+		ls.clusters[clusterName] = c
+	}
+
+	if p, ok := c[serviceName]; ok {
+		return p
+	}
+	p := &PerClusterReporter{
+		cluster:          clusterName,
+		service:          serviceName,
+		lastLoadReportAt: lrsclientinternal.TimeNow(),
+	}
+	c[serviceName] = p
+	return p
+}
+
+// stats returns the load data for the given cluster names. Data is returned in
+// a slice with no specific order.
+//
+// If no clusterName is given (an empty slice), all data for all known clusters
+// is returned.
+//
+// If a cluster's loadData is empty (no load to report), it's not appended to
+// the returned slice.
+func (ls *LoadStore) stats(clusterNames []string) []*loadData {
+	ls.mu.Lock()
+	defer ls.mu.Unlock()
+
+	var ret []*loadData
+	if len(clusterNames) == 0 {
+		for _, c := range ls.clusters {
+			ret = appendClusterStats(ret, c)
+		}
+		return ret
+	}
+	for _, n := range clusterNames {
+		if c, ok := ls.clusters[n]; ok {
+			ret = appendClusterStats(ret, c)
+		}
+	}
+
+	return ret
+}
+
+// PerClusterReporter records load data pertaining to a single cluster. It
+// provides methods to record call starts, finishes, server-reported loads,
+// and dropped calls.
+//
+// It is safe for concurrent use.
+//
+// TODO(purnesh42h): Use regular maps with mutexes instead of sync.Map here.
+// The latter is optimized for two common use cases: (1) when the entry for a
+// given key is only ever written once but read many times, as in caches that
+// only grow, or (2) when multiple goroutines read, write, and overwrite
+// entries for disjoint sets of keys. In these two cases, use of a Map may
+// significantly reduce lock contention compared to a Go map paired with a
+// separate Mutex or RWMutex.
+// Neither of these conditions are met here, and we should transition to a
+// regular map with a mutex for better type safety.
+type PerClusterReporter struct {
+	cluster, service string
+	drops            sync.Map // map[string]*uint64
+	localityRPCCount sync.Map // map[clients.Locality]*rpcCountData
+
+	mu               sync.Mutex
+	lastLoadReportAt time.Time
+}
+
+// CallStarted records a call started in the LoadStore.
+func (p *PerClusterReporter) CallStarted(locality clients.Locality) {
+	s, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		tp := newRPCCountData()
+		s, _ = p.localityRPCCount.LoadOrStore(locality, tp)
+	}
+	s.(*rpcCountData).incrInProgress()
+	s.(*rpcCountData).incrIssued()
+}
+
+// CallFinished records a call finished in the LoadStore.
+func (p *PerClusterReporter) CallFinished(locality clients.Locality, err error) {
+	f, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for call-finish is not found should never happen.
+		return
+	}
+	f.(*rpcCountData).decrInProgress()
+	if err == nil {
+		f.(*rpcCountData).incrSucceeded()
+	} else {
+		f.(*rpcCountData).incrErrored()
+	}
+}
+
+// CallServerLoad records the server load in the LoadStore.
+func (p *PerClusterReporter) CallServerLoad(locality clients.Locality, name string, val float64) {
+	s, ok := p.localityRPCCount.Load(locality)
+	if !ok {
+		// The map is never cleared, only values in the map are reset. So the
+		// case where entry for callServerLoad is not found should never happen.
+		return
+	}
+	s.(*rpcCountData).addServerLoad(name, val)
+}
+
+// CallDropped records a call dropped in the LoadStore.
+func (p *PerClusterReporter) CallDropped(category string) {
+	d, ok := p.drops.Load(category)
+	if !ok {
+		tp := new(uint64)
+		d, _ = p.drops.LoadOrStore(category, tp)
+	}
+	atomic.AddUint64(d.(*uint64), 1)
+}
+
+// stats returns and resets all loads reported to the store, except inProgress
+// rpc counts.
+//
+// It returns nil if the store doesn't contain any (new) data.
+func (p *PerClusterReporter) stats() *loadData {
+	sd := newLoadData(p.cluster, p.service)
+	p.drops.Range(func(key, val any) bool {
+		d := atomic.SwapUint64(val.(*uint64), 0)
+		if d == 0 {
+			return true
+		}
+		sd.totalDrops += d
+		keyStr := key.(string)
+		if keyStr != "" {
+			// Skip drops without category. They are counted in total_drops, but
+			// not in per category. One example is drops by circuit breaking.
+			sd.drops[keyStr] = d
+		}
+		return true
+	})
+	p.localityRPCCount.Range(func(key, val any) bool {
+		countData := val.(*rpcCountData)
+		succeeded := countData.loadAndClearSucceeded()
+		inProgress := countData.loadInProgress()
+		errored := countData.loadAndClearErrored()
+		issued := countData.loadAndClearIssued()
+		if succeeded == 0 && inProgress == 0 && errored == 0 && issued == 0 {
+			return true
+		}
+
+		ld := localityData{
+			requestStats: requestData{
+				succeeded:  succeeded,
+				errored:    errored,
+				inProgress: inProgress,
+				issued:     issued,
+			},
+			loadStats: make(map[string]serverLoadData),
+		}
+		countData.serverLoads.Range(func(key, val any) bool {
+			sum, count := val.(*rpcLoadData).loadAndClear()
+			if count == 0 {
+				return true
+			}
+			ld.loadStats[key.(string)] = serverLoadData{
+				count: count,
+				sum:   sum,
+			}
+			return true
+		})
+		sd.localityStats[key.(clients.Locality)] = ld
+		return true
+	})
+
+	p.mu.Lock()
+	sd.reportInterval = lrsclientinternal.TimeNow().Sub(p.lastLoadReportAt)
+	p.lastLoadReportAt = lrsclientinternal.TimeNow()
+	p.mu.Unlock()
+
+	if sd.totalDrops == 0 && len(sd.drops) == 0 && len(sd.localityStats) == 0 {
+		return nil
+	}
+	return sd
+}
+
+// loadData contains all load data reported to the LoadStore since the most recent
+// call to stats().
+type loadData struct {
+	// cluster is the name of the cluster this data is for.
+	cluster string
+	// service is the name of the EDS service this data is for.
+	service string
+	// totalDrops is the total number of dropped requests.
+	totalDrops uint64
+	// drops is the number of dropped requests per category.
+	drops map[string]uint64
+	// localityStats contains load reports per locality.
+	localityStats map[clients.Locality]localityData
+	// reportInternal is the duration since last time load was reported (stats()
+	// was called).
+	reportInterval time.Duration
+}
+
+// localityData contains load data for a single locality.
+type localityData struct {
+	// requestStats contains counts of requests made to the locality.
+	requestStats requestData
+	// loadStats contains server load data for requests made to the locality,
+	// indexed by the load type.
+	loadStats map[string]serverLoadData
+}
+
+// requestData contains request counts.
+type requestData struct {
+	// succeeded is the number of succeeded requests.
+	succeeded uint64
+	// errored is the number of requests which ran into errors.
+	errored uint64
+	// inProgress is the number of requests in flight.
+	inProgress uint64
+	// issued is the total number requests that were sent.
+	issued uint64
+}
+
+// serverLoadData contains server load data.
+type serverLoadData struct {
+	// count is the number of load reports.
+	count uint64
+	// sum is the total value of all load reports.
+	sum float64
+}
+
+// appendClusterStats gets the Data for all the given clusters, append to ret,
+// and return the new slice.
+//
+// Data is only appended to ret if it's not empty.
+func appendClusterStats(ret []*loadData, clusters map[string]*PerClusterReporter) []*loadData {
+	for _, d := range clusters {
+		data := d.stats()
+		if data == nil {
+			// Skip this data if it doesn't contain any information.
+			continue
+		}
+		ret = append(ret, data)
+	}
+	return ret
+}
+
+func newLoadData(cluster, service string) *loadData {
+	return &loadData{
+		cluster:       cluster,
+		service:       service,
+		drops:         make(map[string]uint64),
+		localityStats: make(map[clients.Locality]localityData),
+	}
+}
+
+type rpcCountData struct {
+	// Only atomic accesses are allowed for the fields.
+	succeeded  *uint64
+	errored    *uint64
+	inProgress *uint64
+	issued     *uint64
+
+	// Map from load desc to load data (sum+count). Loading data from map is
+	// atomic, but updating data takes a lock, which could cause contention when
+	// multiple RPCs try to report loads for the same desc.
+	//
+	// To fix the contention, shard this map.
+	serverLoads sync.Map // map[string]*rpcLoadData
+}
+
+func newRPCCountData() *rpcCountData {
+	return &rpcCountData{
+		succeeded:  new(uint64),
+		errored:    new(uint64),
+		inProgress: new(uint64),
+		issued:     new(uint64),
+	}
+}
+
+func (rcd *rpcCountData) incrSucceeded() {
+	atomic.AddUint64(rcd.succeeded, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearSucceeded() uint64 {
+	return atomic.SwapUint64(rcd.succeeded, 0)
+}
+
+func (rcd *rpcCountData) incrErrored() {
+	atomic.AddUint64(rcd.errored, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearErrored() uint64 {
+	return atomic.SwapUint64(rcd.errored, 0)
+}
+
+func (rcd *rpcCountData) incrInProgress() {
+	atomic.AddUint64(rcd.inProgress, 1)
+}
+
+func (rcd *rpcCountData) decrInProgress() {
+	atomic.AddUint64(rcd.inProgress, ^uint64(0)) // atomic.Add(x, -1)
+}
+
+func (rcd *rpcCountData) loadInProgress() uint64 {
+	return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading.
+}
+
+func (rcd *rpcCountData) incrIssued() {
+	atomic.AddUint64(rcd.issued, 1)
+}
+
+func (rcd *rpcCountData) loadAndClearIssued() uint64 {
+	return atomic.SwapUint64(rcd.issued, 0)
+}
+
+func (rcd *rpcCountData) addServerLoad(name string, d float64) {
+	loads, ok := rcd.serverLoads.Load(name)
+	if !ok {
+		tl := newRPCLoadData()
+		loads, _ = rcd.serverLoads.LoadOrStore(name, tl)
+	}
+	loads.(*rpcLoadData).add(d)
+}
+
+// rpcLoadData is data for server loads (from trailers or oob). Fields in this
+// struct must be updated consistently.
+//
+// The current solution is to hold a lock, which could cause contention. To fix,
+// shard serverLoads map in rpcCountData.
+type rpcLoadData struct {
+	mu    sync.Mutex
+	sum   float64
+	count uint64
+}
+
+func newRPCLoadData() *rpcLoadData {
+	return &rpcLoadData{}
+}
+
+func (rld *rpcLoadData) add(v float64) {
+	rld.mu.Lock()
+	rld.sum += v
+	rld.count++
+	rld.mu.Unlock()
+}
+
+func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) {
+	rld.mu.Lock()
+	s, rld.sum = rld.sum, 0
+	c, rld.count = rld.count, 0
+	rld.mu.Unlock()
+	return s, c
+}

internal/xds/clients/lrsclient/load_store_test.go

@@ -0,0 +1,525 @@
+/*
+ *
+ * Copyright 2020 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lrsclient
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/xds/clients"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+)
+
+var (
+	dropCategories = []string{"drop_for_real", "drop_for_fun"}
+	localities     = []clients.Locality{{Region: "locality-A"}, {Region: "locality-B"}}
+	errTest        = fmt.Errorf("test error")
+)
+
+// rpcData wraps the rpc counts and load data to be pushed to the store.
+type rpcData struct {
+	start, success, failure int
+	serverData              map[string]float64 // Will be reported with successful RPCs.
+}
+
+func verifyLoadStoreData(wantStoreData, gotStoreData []*loadData) error {
+	if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmp.AllowUnexported(loadData{}, localityData{}, requestData{}, serverLoadData{}), cmpopts.IgnoreFields(loadData{}, "reportInterval"), sortDataSlice); diff != "" {
+		return fmt.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff)
+	}
+	return nil
+}
+
+// TestDrops spawns a bunch of goroutines which report drop data. After the
+// goroutines have exited, the test dumps the stats from the Store and makes
+// sure they are as expected.
+func TestDrops(t *testing.T) {
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+			"":                10,
+		}
+		wantStoreData = &loadData{
+			totalDrops: 80,
+			drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+		}
+	)
+
+	ls := PerClusterReporter{}
+	var wg sync.WaitGroup
+	for category, count := range drops {
+		for i := 0; i < count; i++ {
+			wg.Add(1)
+			go func(c string) {
+				ls.CallDropped(c)
+				wg.Done()
+			}(category)
+		}
+	}
+	wg.Wait()
+
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+// TestLocalityStats spawns a bunch of goroutines which report rpc and load
+// data. After the goroutines have exited, the test dumps the stats from the
+// Store and makes sure they are as expected.
+func TestLocalityStats(t *testing.T) {
+	var (
+		ld = map[clients.Locality]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &loadData{
+			localityStats: map[clients.Locality]localityData{
+				localities[0]: {
+					requestStats: requestData{
+						succeeded:  20,
+						errored:    10,
+						inProgress: 10,
+						issued:     40,
+					},
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 20, sum: 20},
+						"disk": {count: 20, sum: 40},
+						"cpu":  {count: 20, sum: 60},
+						"mem":  {count: 20, sum: 80},
+					},
+				},
+				localities[1]: {
+					requestStats: requestData{
+						succeeded:  40,
+						errored:    20,
+						inProgress: 20,
+						issued:     80,
+					},
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 40, sum: 40},
+						"disk": {count: 40, sum: 80},
+						"cpu":  {count: 40, sum: 120},
+						"mem":  {count: 40, sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	ls := PerClusterReporter{}
+	var wg sync.WaitGroup
+	for locality, data := range ld {
+		wg.Add(data.start)
+		for i := 0; i < data.start; i++ {
+			go func(l clients.Locality) {
+				ls.CallStarted(l)
+				wg.Done()
+			}(locality)
+		}
+		// The calls to callStarted() need to happen before the other calls are
+		// made. Hence the wait here.
+		wg.Wait()
+
+		wg.Add(data.success)
+		for i := 0; i < data.success; i++ {
+			go func(l clients.Locality, serverData map[string]float64) {
+				ls.CallFinished(l, nil)
+				for n, d := range serverData {
+					ls.CallServerLoad(l, n, d)
+				}
+				wg.Done()
+			}(locality, data.serverData)
+		}
+		wg.Add(data.failure)
+		for i := 0; i < data.failure; i++ {
+			go func(l clients.Locality) {
+				ls.CallFinished(l, errTest)
+				wg.Done()
+			}(locality)
+		}
+		wg.Wait()
+	}
+
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestResetAfterStats(t *testing.T) {
+	// Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero.
+	// Dump the stats. Verify expected
+	// Push the same set of loads as before
+	// Now dump and verify the newly expected ones.
+	var (
+		drops = map[string]int{
+			dropCategories[0]: 30,
+			dropCategories[1]: 40,
+		}
+		ld = map[clients.Locality]rpcData{
+			localities[0]: {
+				start:      40,
+				success:    20,
+				failure:    10,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+			localities[1]: {
+				start:      80,
+				success:    40,
+				failure:    20,
+				serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4},
+			},
+		}
+		wantStoreData = &loadData{
+			totalDrops: 70,
+			drops: map[string]uint64{
+				dropCategories[0]: 30,
+				dropCategories[1]: 40,
+			},
+			localityStats: map[clients.Locality]localityData{
+				localities[0]: {
+					requestStats: requestData{
+						succeeded:  20,
+						errored:    10,
+						inProgress: 10,
+						issued:     40,
+					},
+
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 20, sum: 20},
+						"disk": {count: 20, sum: 40},
+						"cpu":  {count: 20, sum: 60},
+						"mem":  {count: 20, sum: 80},
+					},
+				},
+				localities[1]: {
+					requestStats: requestData{
+						succeeded:  40,
+						errored:    20,
+						inProgress: 20,
+						issued:     80,
+					},
+
+					loadStats: map[string]serverLoadData{
+						"net":  {count: 40, sum: 40},
+						"disk": {count: 40, sum: 80},
+						"cpu":  {count: 40, sum: 120},
+						"mem":  {count: 40, sum: 160},
+					},
+				},
+			},
+		}
+	)
+
+	reportLoad := func(ls *PerClusterReporter) {
+		for category, count := range drops {
+			for i := 0; i < count; i++ {
+				ls.CallDropped(category)
+			}
+		}
+		for locality, data := range ld {
+			for i := 0; i < data.start; i++ {
+				ls.CallStarted(locality)
+			}
+			for i := 0; i < data.success; i++ {
+				ls.CallFinished(locality, nil)
+				for n, d := range data.serverData {
+					ls.CallServerLoad(locality, n, d)
+				}
+			}
+			for i := 0; i < data.failure; i++ {
+				ls.CallFinished(locality, errTest)
+			}
+		}
+	}
+
+	ls := PerClusterReporter{}
+	reportLoad(&ls)
+	gotStoreData := ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+
+	// The above call to stats() should have reset all load reports except the
+	// inProgress rpc count. We are now going to push the same load data into
+	// the store. So, we should expect to see twice the count for inProgress.
+	for _, l := range localities {
+		ls := wantStoreData.localityStats[l]
+		ls.requestStats.inProgress *= 2
+		wantStoreData.localityStats[l] = ls
+	}
+	reportLoad(&ls)
+	gotStoreData = ls.stats()
+	if err := verifyLoadStoreData([]*loadData{wantStoreData}, []*loadData{gotStoreData}); err != nil {
+		t.Error(err)
+	}
+}
+
+var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*loadData) []*loadData {
+	out := append([]*loadData(nil), in...) // Copy input to avoid mutating it
+	sort.Slice(out,
+		func(i, j int) bool {
+			if out[i].cluster < out[j].cluster {
+				return true
+			}
+			if out[i].cluster == out[j].cluster {
+				return out[i].service < out[j].service
+			}
+			return false
+		},
+	)
+	return out
+})
+
+// Test all load are returned for the given clusters, and all clusters are
+// reported if no cluster is specified.
+func TestStoreStats(t *testing.T) {
+	var (
+		testClusters = []string{"c0", "c1", "c2"}
+		testServices = []string{"s0", "s1"}
+		testLocality = clients.Locality{Region: "test-locality"}
+	)
+
+	store := newLoadStore()
+	for _, c := range testClusters {
+		for _, s := range testServices {
+			store.ReporterForCluster(c, s).CallStarted(testLocality)
+			store.ReporterForCluster(c, s).CallServerLoad(testLocality, "abc", 123)
+			store.ReporterForCluster(c, s).CallDropped("dropped")
+			store.ReporterForCluster(c, s).CallFinished(testLocality, nil)
+		}
+	}
+
+	wantC0 := []*loadData{
+		{
+			cluster: "c0", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c0", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with just "c0", this should return data for "c0", and not
+	// touch data for other clusters.
+	gotC0 := store.stats([]string{"c0"})
+	verifyLoadStoreData(wantC0, gotC0)
+
+	wantOther := []*loadData{
+		{
+			cluster: "c1", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c2", service: "s0",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+		{
+			cluster: "c2", service: "s1",
+			totalDrops: 1, drops: map[string]uint64{"dropped": 1},
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {
+					requestStats: requestData{succeeded: 1, issued: 1},
+					loadStats:    map[string]serverLoadData{"abc": {count: 1, sum: 123}},
+				},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// remaining clusters, and not include c0 (because c0 data was cleared).
+	gotOther := store.stats(nil)
+	if err := verifyLoadStoreData(wantOther, gotOther); err != nil {
+		t.Error(err)
+	}
+}
+
+// Test the cases that if a cluster doesn't have load to report, its data is not
+// appended to the slice returned by Stats().
+func TestStoreStatsEmptyDataNotReported(t *testing.T) {
+	var (
+		testServices = []string{"s0", "s1"}
+		testLocality = clients.Locality{Region: "test-locality"}
+	)
+
+	store := newLoadStore()
+	// "c0"'s RPCs all finish with success.
+	for _, s := range testServices {
+		store.ReporterForCluster("c0", s).CallStarted(testLocality)
+		store.ReporterForCluster("c0", s).CallFinished(testLocality, nil)
+	}
+	// "c1"'s RPCs never finish (always inprocess).
+	for _, s := range testServices {
+		store.ReporterForCluster("c1", s).CallStarted(testLocality)
+	}
+
+	want0 := []*loadData{
+		{
+			cluster: "c0", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{succeeded: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c0", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{succeeded: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1, issued: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1, issued: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice, this should return data for all the
+	// clusters.
+	got0 := store.stats(nil)
+	if err := verifyLoadStoreData(want0, got0); err != nil {
+		t.Error(err)
+	}
+
+	want1 := []*loadData{
+		{
+			cluster: "c1", service: "s0",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1}},
+			},
+		},
+		{
+			cluster: "c1", service: "s1",
+			localityStats: map[clients.Locality]localityData{
+				testLocality: {requestStats: requestData{inProgress: 1}},
+			},
+		},
+	}
+	// Call Stats with empty slice again, this should return data only for "c1",
+	// because "c0" data was cleared, but "c1" has in-progress RPCs.
+	got1 := store.stats(nil)
+	if err := verifyLoadStoreData(want1, got1); err != nil {
+		t.Error(err)
+	}
+}
+
+// TestStoreReportInterval verify that the load report interval gets
+// calculated at every stats() call and is the duration between start of last
+// load reporting to next stats() call.
+func TestStoreReportInterval(t *testing.T) {
+	originalTimeNow := lrsclientinternal.TimeNow
+	t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow })
+
+	// Initial time for reporter creation
+	currentTime := time.Now()
+	lrsclientinternal.TimeNow = func() time.Time {
+		return currentTime
+	}
+
+	store := newLoadStore()
+	reporter := store.ReporterForCluster("test-cluster", "test-service")
+	// Report dummy drop to ensure stats1 is not nil.
+	reporter.CallDropped("dummy-category")
+
+	// Update currentTime to simulate the passage of time between the reporter
+	// creation and first stats() call.
+	currentTime = currentTime.Add(5 * time.Second)
+	stats1 := reporter.stats()
+
+	if stats1 == nil {
+		t.Fatalf("stats1 is nil after reporting a drop, want non-nil")
+	}
+	// Verify stats() call calculate the report interval from the time of
+	// reporter creation.
+	if got, want := stats1.reportInterval, 5*time.Second; got != want {
+		t.Errorf("stats1.reportInterval = %v, want %v", stats1.reportInterval, want)
+	}
+
+	// Update currentTime to simulate the passage of time between the first
+	// and second stats() call.
+	currentTime = currentTime.Add(10 * time.Second)
+	// Report another dummy drop to ensure stats2 is not nil.
+	reporter.CallDropped("dummy-category-2")
+	stats2 := reporter.stats()
+
+	if stats2 == nil {
+		t.Fatalf("stats2 is nil after reporting a drop, want non-nil")
+	}
+	// Verify stats() call calculate the report interval from the time of first
+	// stats() call.
+	if got, want := stats2.reportInterval, 10*time.Second; got != want {
+		t.Errorf("stats2.reportInterval = %v, want %v", stats2.reportInterval, want)
+	}
+}

internal/xds/clients/lrsclient/loadreport_test.go

@@ -0,0 +1,730 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient_test
+
+import (
+	"context"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpctest"
+	grpctestutils "google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/grpc/internal/xds/clients/grpctransport"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/e2e"
+	"google.golang.org/grpc/internal/xds/clients/internal/testutils/fakeserver"
+	"google.golang.org/grpc/internal/xds/clients/lrsclient"
+	lrsclientinternal "google.golang.org/grpc/internal/xds/clients/lrsclient/internal"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/testing/protocmp"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const (
+	testKey1                      = "test-key1"
+	testKey2                      = "test-key2"
+	defaultTestWatchExpiryTimeout = 100 * time.Millisecond
+	defaultTestTimeout            = 5 * time.Second
+	defaultTestShortTimeout       = 10 * time.Millisecond // For events expected to *not* happen.
+)
+
+var (
+	testLocality1     = clients.Locality{Region: "test-region1"}
+	testLocality2     = clients.Locality{Region: "test-region2"}
+	toleranceCmpOpt   = cmpopts.EquateApprox(0, 1e-5)
+	ignoreOrderCmpOpt = protocmp.FilterField(&v3endpointpb.ClusterStats{}, "upstream_locality_stats",
+		cmpopts.SortSlices(func(a, b protocmp.Message) bool {
+			return a.String() < b.String()
+		}),
+	)
+)
+
+type wrappedListener struct {
+	net.Listener
+	newConnChan *testutils.Channel // Connection attempts are pushed here.
+}
+
+func (wl *wrappedListener) Accept() (net.Conn, error) {
+	c, err := wl.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	wl.newConnChan.Send(struct{}{})
+	return c, err
+}
+
+// Tests a load reporting scenario where the LRS client is reporting loads to
+// multiple servers. Verifies the following:
+//   - calling the load reporting API with different server configuration
+//     results in connections being created to those corresponding servers
+//   - the same load.Store is not returned when the load reporting API called
+//     with different server configurations
+//   - canceling the load reporting from the client results in the LRS stream
+//     being canceled on the server
+func (s) TestReportLoad_ConnectionCreation(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create two management servers that also serve LRS.
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	newConnChan1 := testutils.NewChannelWithSize(1)
+	lis1 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan1,
+	}
+	mgmtServer1 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis1,
+		SupportLoadReportingService: true,
+	})
+	l, err = net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen() failed: %v", err)
+	}
+	newConnChan2 := testutils.NewChannelWithSize(1)
+	lis2 := &wrappedListener{
+		Listener:    l,
+		newConnChan: newConnChan2,
+	}
+	mgmtServer2 := e2e.StartManagementServer(t, e2e.ManagementServerOptions{
+		Listener:                    lis2,
+		SupportLoadReportingService: true,
+	})
+
+	// Create an LRS client with a configuration that contains both of
+	// the above two servers. The authority name is immaterial here since load
+	// reporting is per-server and not per-authority.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	serverIdentifier1 := clients.ServerIdentifier{ServerURI: mgmtServer1.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier1)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	defer loadStore1.Stop(ssCtx)
+
+	// Call the load reporting API to report load to the first management
+	// server, and ensure that a connection to the server is created.
+	if _, err := newConnChan1.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the first management server, after starting load reporting")
+	}
+	if _, err := mgmtServer1.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	// Call the load reporting API to report load to the first management
+	// server, and ensure that a connection to the server is created.
+	serverIdentifier2 := clients.ServerIdentifier{ServerURI: mgmtServer2.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore2, err := client.ReportLoad(serverIdentifier2)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	if _, err := newConnChan2.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for a connection to the second management server, after starting load reporting")
+	}
+	if _, err := mgmtServer2.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout when waiting for LRS stream to be created")
+	}
+
+	if loadStore1 == loadStore2 {
+		t.Fatalf("Got same store for different servers, want different")
+	}
+
+	// Push some loads on the received store.
+	loadStore2.ReporterForCluster("cluster", "eds").CallDropped("test")
+
+	// Ensure the initial load reporting request is received at the server.
+	lrsServer := mgmtServer2.LRSServer
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for LRS request with loads: %v", err)
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster",
+		ClusterServiceName:   "eds",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Stop this load reporting stream, server should see error canceled.
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore2.Stop(ssCtx)
+
+	// Server should receive a stream canceled error. There may be additional
+	// load reports from the client in the channel.
+	for {
+		if ctx.Err() != nil {
+			t.Fatal("Timeout when waiting for the LRS stream to be canceled on the server")
+		}
+		u, err := lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		// Ignore load reports sent before the stream was cancelled.
+		if u.(*fakeserver.Request).Err == nil {
+			continue
+		}
+		if status.Code(u.(*fakeserver.Request).Err) != codes.Canceled {
+			t.Fatalf("Unexpected LRS request: %v, want error canceled", u)
+		}
+		break
+	}
+}
+
+// Tests a load reporting scenario where the load reporting API is called
+// multiple times for the same server. The test verifies the following:
+//   - calling the load reporting API the second time for the same server
+//     configuration does not create a new LRS stream
+//   - the LRS stream is closed *only* after all the API calls invoke their
+//     cancel functions
+//   - creating new streams after the previous one was closed works
+func (s) TestReportLoad_StreamCreation(t *testing.T) {
+	grpctestutils.SetEnvConfig(t, &envconfig.XDSORCAToLRSPropEnabled, true)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	// Push some loads on the received store.
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality1)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 3.14)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, testKey1, 2.718)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "cpu_utilization", 1.5)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "mem_utilization", 2.0)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality1, "application_utilization", 3.0)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality1, nil)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallStarted(testLocality2)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallServerLoad(testLocality2, testKey2, 1.618)
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallFinished(testLocality2, nil)
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster1",
+		ClusterServiceName:   "eds1",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		UpstreamLocalityStats: []*v3endpointpb.UpstreamLocalityStats{
+			{
+				Locality: &v3corepb.Locality{Region: "test-region1"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					// TotalMetricValue is the aggregation of 3.14 + 2.718 = 5.858
+					{MetricName: testKey1, NumRequestsFinishedWithMetric: 2, TotalMetricValue: 5.858}},
+				CpuUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.5,
+				},
+				MemUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 2.0,
+				},
+				ApplicationUtilization: &v3endpointpb.UnnamedEndpointLoadMetricStats{
+					NumRequestsFinishedWithMetric: 1, TotalMetricValue: 3.0,
+				},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+			{
+				Locality: &v3corepb.Locality{Region: "test-region2"},
+				LoadMetricStats: []*v3endpointpb.EndpointLoadMetricStats{
+					{MetricName: testKey2, NumRequestsFinishedWithMetric: 1, TotalMetricValue: 1.618}},
+				TotalSuccessfulRequests: 1,
+				TotalIssuedRequests:     1,
+			},
+		},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Make another call to the load reporting API, and ensure that a new LRS
+	// stream is not created.
+	loadStore2, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("New LRS stream created when expected to use an existing one")
+	}
+
+	// Push more loads.
+	loadStore2.ReporterForCluster("cluster2", "eds2").CallDropped("test")
+
+	// Ensure that loads are seen on the server. We need a loop here because
+	// there could have been some requests from the client in the time between
+	// us reading the first request and now. Those would have been queued in the
+	// request channel that we read out of.
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for new loads to be seen on the server")
+		}
+
+		req, err = lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil {
+			continue
+		}
+		gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+		if l := len(gotLoad); l != 1 {
+			continue
+		}
+		gotLoad[0].LoadReportInterval = nil
+		wantLoad := &v3endpointpb.ClusterStats{
+			ClusterName:          "cluster2",
+			ClusterServiceName:   "eds2",
+			TotalDroppedRequests: 1,
+			DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		}
+		if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
+			t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+			continue
+		}
+		break
+	}
+
+	// Cancel the first load reporting call, and ensure that the stream does not
+	// close (because we have another call open).
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore1.Stop(ssCtx)
+	sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer sCancel()
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded {
+		t.Fatal("LRS stream closed when expected to stay open")
+	}
+
+	// Stop the second load reporting call, and ensure the stream is closed.
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore2.Stop(ssCtx)
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for LRS stream to close")
+	}
+
+	// Calling the load reporting API again should result in the creation of a
+	// new LRS stream. This ensures that creating and closing multiple streams
+	// works smoothly.
+	loadStore3, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+	ssCtx, ssCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout)
+	defer ssCancel()
+	loadStore3.Stop(ssCtx)
+}
+
+// TestReportLoad_StopWithContext tests the behavior of LoadStore.Stop() when
+// called with a context. It verifies that:
+//   - Stop() blocks until the context expires or final load send attempt is
+//     made.
+//   - Final load report is seen on the server after stop is called.
+//   - The stream is closed after Stop() returns.
+func (s) TestReportLoad_StopWithContext(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	// Push some loads on the received store.
+	loadStore.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+
+	// This field is set by the client to indicate the actual time elapsed since
+	// the last report was sent. We cannot deterministically compare this, and
+	// we cannot use the cmpopts.IgnoreFields() option on proto structs, since
+	// we already use the protocmp.Transform() which marshals the struct into
+	// another message. Hence setting this field to nil is the best option here.
+	gotLoad[0].LoadReportInterval = nil
+	wantLoad := &v3endpointpb.ClusterStats{
+		ClusterName:          "cluster1",
+		ClusterServiceName:   "eds1",
+		TotalDroppedRequests: 1,
+		DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+	}
+	if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform(), toleranceCmpOpt, ignoreOrderCmpOpt); diff != "" {
+		t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Create a context for Stop() that remains until the end of test to ensure
+	// that only possibility of Stop()s to finish is if final load send attempt
+	// is made. If final load attempt is not made, test will timeout.
+	stopCtx, stopCancel := context.WithCancel(ctx)
+	defer stopCancel()
+
+	// Push more loads.
+	loadStore.ReporterForCluster("cluster2", "eds2").CallDropped("test")
+
+	stopCalled := make(chan struct{})
+	// Call Stop in a separate goroutine. It will block until
+	// final load send attempt is made.
+	go func() {
+		loadStore.Stop(stopCtx)
+		close(stopCalled)
+	}()
+	<-stopCalled
+
+	// Ensure that loads are seen on the server. We need a loop here because
+	// there could have been some requests from the client in the time between
+	// us reading the first request and now. Those would have been queued in the
+	// request channel that we read out of.
+	for {
+		if ctx.Err() != nil {
+			t.Fatalf("Timeout when waiting for new loads to be seen on the server")
+		}
+
+		req, err = lrsServer.LRSRequestChan.Receive(ctx)
+		if err != nil || req.(*fakeserver.Request).Err != nil {
+			continue
+		}
+		if req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) == nil {
+			// This can happen due to a race:
+			// 1. Load for "cluster2" is reported just before Stop().
+			// 2. The periodic ticker might send this load before Stop()'s
+			//    final send mechanism processes it, clearing the data.
+			// 3. Stop()'s final send might then send an empty report.
+			//    This is acceptable for this test because we only need to verify
+			//    if the final load report send attempt was made.
+			t.Logf("Empty final load report sent on server")
+			break
+		}
+		gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+		if l := len(gotLoad); l != 1 {
+			continue
+		}
+		gotLoad[0].LoadReportInterval = nil
+		wantLoad := &v3endpointpb.ClusterStats{
+			ClusterName:          "cluster2",
+			ClusterServiceName:   "eds2",
+			TotalDroppedRequests: 1,
+			DroppedRequests:      []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}},
+		}
+		if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" {
+			t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff)
+			continue
+		}
+		break
+	}
+
+	// Verify the stream is eventually closed on the server side.
+	if _, err := lrsServer.LRSStreamCloseChan.Receive(ctx); err != nil {
+		t.Fatal("Timeout waiting for LRS stream to close")
+	}
+}
+
+// TestReportLoad_LoadReportInterval tests verify that the load report interval
+// received by the LRS server is the duration between start of last load
+// reporting by the client and the time when the load is reported to the LRS
+// server.
+func (s) TestReportLoad_LoadReportInterval(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	originalTimeNow := lrsclientinternal.TimeNow
+	t.Cleanup(func() { lrsclientinternal.TimeNow = originalTimeNow })
+
+	// Create a management server that serves LRS.
+	mgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{SupportLoadReportingService: true})
+
+	// Create an LRS client with configuration pointing to the above server.
+	nodeID := uuid.New().String()
+
+	configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
+	config := lrsclient.Config{
+		Node:             clients.Node{ID: nodeID, UserAgentName: "user-agent", UserAgentVersion: "0.0.0.0"},
+		TransportBuilder: grpctransport.NewBuilder(configs),
+	}
+	client, err := lrsclient.New(config)
+	if err != nil {
+		t.Fatalf("lrsclient.New() failed: %v", err)
+	}
+
+	// Call the load reporting API, and ensure that an LRS stream is created.
+	serverIdentifier := clients.ServerIdentifier{ServerURI: mgmtServer.Address, Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}}
+	loadStore1, err := client.ReportLoad(serverIdentifier)
+	if err != nil {
+		t.Fatalf("client.ReportLoad() failed: %v", err)
+	}
+	lrsServer := mgmtServer.LRSServer
+	if _, err := lrsServer.LRSStreamOpenChan.Receive(ctx); err != nil {
+		t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err)
+	}
+
+	stateMu := sync.Mutex{}
+	// Initial time for reporter creation
+	currentTime := time.Now()
+	lrsclientinternal.TimeNow = func() time.Time {
+		stateMu.Lock()
+		defer stateMu.Unlock()
+		return currentTime
+	}
+
+	// Report dummy drop to ensure stats is not nil.
+	loadStore1.ReporterForCluster("cluster1", "eds1").CallDropped("test")
+
+	// Update currentTime to simulate the passage of time between the reporter
+	// creation and first stats() call.
+	stateMu.Lock()
+	currentTime = currentTime.Add(5 * time.Second)
+	stateMu.Unlock()
+
+	// Ensure the initial load reporting request is received at the server.
+	req, err := lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatalf("Timeout when waiting for initial LRS request: %v", err)
+	}
+	gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest)
+	nodeProto := &v3corepb.Node{
+		Id:                   nodeID,
+		UserAgentName:        "user-agent",
+		UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "0.0.0.0"},
+		ClientFeatures:       []string{"envoy.lb.does_not_support_overprovisioning", "xds.config.resource-in-sotw", "envoy.lrs.supports_send_all_clusters"},
+	}
+	wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto}
+	if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" {
+		t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff)
+	}
+
+	// Send a response from the server with a small deadline.
+	lrsServer.LRSResponseChan <- &fakeserver.Response{
+		Resp: &v3lrspb.LoadStatsResponse{
+			SendAllClusters:       true,
+			LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms
+		},
+	}
+
+	// Ensure that loads are seen on the server.
+	req, err = lrsServer.LRSRequestChan.Receive(ctx)
+	if err != nil {
+		t.Fatal("Timeout when waiting for LRS request with loads")
+	}
+	gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats
+	if l := len(gotLoad); l != 1 {
+		t.Fatalf("Received load for %d clusters, want 1", l)
+	}
+	// Verify load received at LRS server has load report interval calculated
+	// from the time of reporter creation.
+	if got, want := gotLoad[0].GetLoadReportInterval().AsDuration(), 5*time.Second; got != want {
+		t.Errorf("Got load report interval %v, want %v", got, want)
+	}
+
+	ssCtx, ssCancel := context.WithTimeout(context.Background(), time.Millisecond)
+	defer ssCancel()
+	loadStore1.Stop(ssCtx)
+}

internal/xds/clients/lrsclient/logging.go

@@ -0,0 +1,36 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/grpclog"
+	internalgrpclog "google.golang.org/grpc/internal/grpclog"
+)
+
+var logger = grpclog.Component("xds")
+
+func prefixLogger(c *LRSClient) *internalgrpclog.PrefixLogger {
+	return internalgrpclog.NewPrefixLogger(logger, clientPrefix(c))
+}
+
+func clientPrefix(c *LRSClient) string {
+	return fmt.Sprintf("[lrs-client %p] ", c)
+}

internal/xds/clients/lrsclient/lrs_stream.go

@@ -0,0 +1,334 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package lrsclient
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/internal/envconfig"
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/xds/clients"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
+	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3"
+)
+
+// Any per-RPC level logs which print complete request or response messages
+// should be gated at this verbosity level. Other per-RPC level logs which print
+// terse output should be at `INFO` and verbosity 2.
+const perRPCVerbosityLevel = 9
+
+// streamImpl provides all the functionality associated with an LRS (Load
+// Reporting Service) stream on the client-side. It manages the lifecycle of
+// the LRS stream, including starting, stopping, and retrying the stream. It
+// also provides a LoadStore that can be used to report load, with a Stop
+// function that should be called when the load reporting is no longer
+// needed.
+type streamImpl struct {
+	// The following fields are initialized when a stream instance is created
+	// and are read-only afterwards, and hence can be accessed without a mutex.
+	transport clients.Transport       // Transport to use for LRS stream.
+	backoff   func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto *v3corepb.Node          // Identifies the gRPC application.
+	doneCh    chan struct{}           // To notify exit of LRS goroutine.
+	logger    *igrpclog.PrefixLogger
+
+	cancelStream context.CancelFunc // Cancel the stream. If nil, the stream is not active.
+	loadStore    *LoadStore         // LoadStore returned to user for pushing loads.
+
+	finalSendRequest chan struct{} // To request for the final attempt to send loads.
+	finalSendDone    chan error    // To signal completion of the final attempt of sending loads.
+}
+
+// streamOpts holds the options for creating an lrsStream.
+type streamOpts struct {
+	transport clients.Transport       // xDS transport to create the stream on.
+	backoff   func(int) time.Duration // Backoff for retries, after stream failures.
+	nodeProto *v3corepb.Node          // Node proto to identify the gRPC application.
+	logPrefix string                  // Prefix to be used for log messages.
+}
+
+// newStreamImpl creates a new StreamImpl with the provided options.
+//
+// The actual streaming RPC call is initiated when the first call to ReportLoad
+// is made, and is terminated when the last call to ReportLoad is canceled.
+func newStreamImpl(opts streamOpts) *streamImpl {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	lrs := &streamImpl{
+		transport:        opts.transport,
+		backoff:          opts.backoff,
+		nodeProto:        opts.nodeProto,
+		cancelStream:     cancel,
+		doneCh:           make(chan struct{}),
+		finalSendRequest: make(chan struct{}, 1),
+		finalSendDone:    make(chan error, 1),
+	}
+
+	l := grpclog.Component("xds")
+	lrs.logger = igrpclog.NewPrefixLogger(l, opts.logPrefix+fmt.Sprintf("[lrs-stream %p] ", lrs))
+	lrs.loadStore = newLoadStore()
+	go lrs.runner(ctx)
+	return lrs
+}
+
+// runner is responsible for managing the lifetime of an LRS streaming call. It
+// creates the stream, sends the initial LoadStatsRequest, receives the first
+// LoadStatsResponse, and then starts a goroutine to periodically send
+// LoadStatsRequests. The runner will restart the stream if it encounters any
+// errors.
+func (lrs *streamImpl) runner(ctx context.Context) {
+	defer close(lrs.doneCh)
+
+	// This feature indicates that the client supports the
+	// LoadStatsResponse.send_all_clusters field in the LRS response.
+	node := proto.Clone(lrs.nodeProto).(*v3corepb.Node)
+	node.ClientFeatures = append(node.ClientFeatures, "envoy.lrs.supports_send_all_clusters")
+
+	runLoadReportStream := func() error {
+		// streamCtx is created and canceled in case we terminate the stream
+		// early for any reason, to avoid gRPC-Go leaking the RPC's monitoring
+		// goroutine.
+		streamCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		stream, err := lrs.transport.NewStream(streamCtx, "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats")
+		if err != nil {
+			lrs.logger.Warningf("Failed to create new LRS streaming RPC: %v", err)
+			return nil
+		}
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("LRS stream created")
+		}
+
+		if err := lrs.sendFirstLoadStatsRequest(stream, node); err != nil {
+			lrs.logger.Warningf("Sending first LRS request failed: %v", err)
+			return nil
+		}
+
+		clusters, interval, err := lrs.recvFirstLoadStatsResponse(stream)
+		if err != nil {
+			lrs.logger.Warningf("Reading from LRS streaming RPC failed: %v", err)
+			return nil
+		}
+
+		// We reset backoff state when we successfully receive at least one
+		// message from the server.
+		lrs.sendLoads(streamCtx, stream, clusters, interval)
+		return backoff.ErrResetBackoff
+	}
+	backoff.RunF(ctx, runLoadReportStream, lrs.backoff)
+}
+
+// sendLoads is responsible for periodically sending load reports to the LRS
+// server at the specified interval for the specified clusters, until the passed
+// in context is canceled.
+func (lrs *streamImpl) sendLoads(ctx context.Context, stream clients.Stream, clusterNames []string, interval time.Duration) {
+	tick := time.NewTicker(interval)
+	defer tick.Stop()
+	for {
+		select {
+		case <-tick.C:
+		case <-ctx.Done():
+			return
+		case <-lrs.finalSendRequest:
+			var finalSendErr error
+			if lrs.logger.V(2) {
+				lrs.logger.Infof("Final send request received. Attempting final LRS report.")
+			}
+			if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil {
+				lrs.logger.Warningf("Failed to send final load report. Writing to LRS stream failed: %v", err)
+				finalSendErr = err
+			}
+			if lrs.logger.V(2) {
+				lrs.logger.Infof("Successfully sent final load report.")
+			}
+			lrs.finalSendDone <- finalSendErr
+			return
+		}
+
+		if err := lrs.sendLoadStatsRequest(stream, lrs.loadStore.stats(clusterNames)); err != nil {
+			lrs.logger.Warningf("Failed to send periodic load report. Writing to LRS stream failed: %v", err)
+			return
+		}
+	}
+}
+
+func (lrs *streamImpl) sendFirstLoadStatsRequest(stream clients.Stream, node *v3corepb.Node) error {
+	req := &v3lrspb.LoadStatsRequest{Node: node}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Sending initial LoadStatsRequest: %s", pretty.ToJSON(req))
+	}
+	msg, err := proto.Marshal(req)
+	if err != nil {
+		lrs.logger.Warningf("Failed to marshal LoadStatsRequest: %v", err)
+		return err
+	}
+	err = stream.Send(msg)
+	if err == io.EOF {
+		return getStreamError(stream)
+	}
+	return err
+}
+
+// recvFirstLoadStatsResponse receives the first LoadStatsResponse from the LRS
+// server.  Returns the following:
+//   - a list of cluster names requested by the server or an empty slice if the
+//     server requested for load from all clusters
+//   - the load reporting interval, and
+//   - any error encountered
+func (lrs *streamImpl) recvFirstLoadStatsResponse(stream clients.Stream) ([]string, time.Duration, error) {
+	r, err := stream.Recv()
+	if err != nil {
+		return nil, 0, fmt.Errorf("lrs: failed to receive first LoadStatsResponse: %v", err)
+	}
+	var resp v3lrspb.LoadStatsResponse
+	if err := proto.Unmarshal(r, &resp); err != nil {
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("Failed to unmarshal response to LoadStatsResponse: %v", err)
+		}
+		return nil, time.Duration(0), fmt.Errorf("lrs: unexpected message type %T", r)
+	}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Received first LoadStatsResponse: %s", pretty.ToJSON(&resp))
+	}
+
+	internal := resp.GetLoadReportingInterval()
+	if internal.CheckValid() != nil {
+		return nil, 0, fmt.Errorf("lrs: invalid load_reporting_interval: %v", err)
+	}
+	loadReportingInterval := internal.AsDuration()
+
+	clusters := resp.Clusters
+	if resp.SendAllClusters {
+		// Return an empty slice to send stats for all clusters.
+		clusters = []string{}
+	}
+
+	return clusters, loadReportingInterval, nil
+}
+
+func (lrs *streamImpl) sendLoadStatsRequest(stream clients.Stream, loads []*loadData) error {
+	clusterStats := make([]*v3endpointpb.ClusterStats, 0, len(loads))
+	for _, sd := range loads {
+		droppedReqs := make([]*v3endpointpb.ClusterStats_DroppedRequests, 0, len(sd.drops))
+		for category, count := range sd.drops {
+			droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{
+				Category:     category,
+				DroppedCount: count,
+			})
+		}
+		localityStats := make([]*v3endpointpb.UpstreamLocalityStats, 0, len(sd.localityStats))
+		for lid, localityData := range sd.localityStats {
+			loadMetricStats := make([]*v3endpointpb.EndpointLoadMetricStats, 0, len(localityData.loadStats))
+			var cpuUtilization, memUtilization, appUtilization *v3endpointpb.UnnamedEndpointLoadMetricStats
+			for name, loadData := range localityData.loadStats {
+				if envconfig.XDSORCAToLRSPropEnabled {
+					switch name {
+					case "cpu_utilization":
+						cpuUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					case "mem_utilization":
+						memUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					case "application_utilization":
+						appUtilization = &v3endpointpb.UnnamedEndpointLoadMetricStats{
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						}
+					default:
+						loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+							MetricName:                    name,
+							NumRequestsFinishedWithMetric: loadData.count,
+							TotalMetricValue:              loadData.sum,
+						})
+					}
+				} else {
+					loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{
+						MetricName:                    name,
+						NumRequestsFinishedWithMetric: loadData.count,
+						TotalMetricValue:              loadData.sum,
+					})
+				}
+			}
+			localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{
+				Locality: &v3corepb.Locality{
+					Region:  lid.Region,
+					Zone:    lid.Zone,
+					SubZone: lid.SubZone,
+				},
+				TotalSuccessfulRequests: localityData.requestStats.succeeded,
+				TotalRequestsInProgress: localityData.requestStats.inProgress,
+				TotalErrorRequests:      localityData.requestStats.errored,
+				TotalIssuedRequests:     localityData.requestStats.issued,
+				CpuUtilization:          cpuUtilization,
+				MemUtilization:          memUtilization,
+				ApplicationUtilization:  appUtilization,
+				LoadMetricStats:         loadMetricStats,
+				UpstreamEndpointStats:   nil, // TODO: populate for per endpoint loads.
+			})
+		}
+
+		clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{
+			ClusterName:           sd.cluster,
+			ClusterServiceName:    sd.service,
+			UpstreamLocalityStats: localityStats,
+			TotalDroppedRequests:  sd.totalDrops,
+			DroppedRequests:       droppedReqs,
+			LoadReportInterval:    durationpb.New(sd.reportInterval),
+		})
+	}
+
+	req := &v3lrspb.LoadStatsRequest{ClusterStats: clusterStats}
+	if lrs.logger.V(perRPCVerbosityLevel) {
+		lrs.logger.Infof("Sending LRS loads: %s", pretty.ToJSON(req))
+	}
+	msg, err := proto.Marshal(req)
+	if err != nil {
+		if lrs.logger.V(2) {
+			lrs.logger.Infof("Failed to marshal LoadStatsRequest: %v", err)
+		}
+		return err
+	}
+	err = stream.Send(msg)
+	if err == io.EOF {
+		return getStreamError(stream)
+	}
+	return err
+}
+
+func getStreamError(stream clients.Stream) error {
+	for {
+		if _, err := stream.Recv(); err != nil {
+			return err
+		}
+	}
+}

internal/xds/clients/lrsclient/lrsclient.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package lrsclient provides an LRS (Load Reporting Service) client.
+//
+// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/service/load_stats/v3/lrs.proto
+package lrsclient
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	igrpclog "google.golang.org/grpc/internal/grpclog"
+	"google.golang.org/grpc/internal/xds/clients"
+	clientsinternal "google.golang.org/grpc/internal/xds/clients/internal"
+	"google.golang.org/grpc/internal/xds/clients/internal/backoff"
+)
+
+const (
+	clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning"
+	clientFeatureResourceWrapper    = "xds.config.resource-in-sotw"
+)
+
+var (
+	defaultExponentialBackoff = backoff.DefaultExponential.Backoff
+)
+
+// LRSClient is an LRS (Load Reporting Service) client.
+type LRSClient struct {
+	transportBuilder clients.TransportBuilder
+	node             clients.Node
+	backoff          func(int) time.Duration // Backoff for LRS stream failures.
+	logger           *igrpclog.PrefixLogger
+
+	// The LRSClient owns a bunch of streams to individual LRS servers.
+	//
+	// Once all references to a stream are dropped, the stream is closed.
+	mu         sync.Mutex
+	lrsStreams map[clients.ServerIdentifier]*streamImpl // Map from server config to in-use streamImpls.
+	lrsRefs    map[clients.ServerIdentifier]int         // Map from server config to number of references.
+}
+
+// New returns a new LRS Client configured with the provided config.
+func New(config Config) (*LRSClient, error) {
+	if config.TransportBuilder == nil {
+		return nil, errors.New("lrsclient: transport builder is nil")
+	}
+
+	c := &LRSClient{
+		transportBuilder: config.TransportBuilder,
+		node:             config.Node,
+		backoff:          defaultExponentialBackoff,
+		lrsStreams:       make(map[clients.ServerIdentifier]*streamImpl),
+		lrsRefs:          make(map[clients.ServerIdentifier]int),
+	}
+	c.logger = prefixLogger(c)
+	return c, nil
+}
+
+// ReportLoad creates and returns a LoadStore for the caller to report loads
+// using a LoadReportingStream.
+//
+// Caller must call Stop on the returned LoadStore when they are done reporting
+// load to this server.
+func (c *LRSClient) ReportLoad(si clients.ServerIdentifier) (*LoadStore, error) {
+	lrs, err := c.getOrCreateLRSStream(si)
+	if err != nil {
+		return nil, err
+	}
+	return lrs.loadStore, nil
+}
+
+// getOrCreateLRSStream returns an lrs stream for the given server identifier.
+//
+// If an active lrs stream exists for the given server identifier, it is
+// returned. Otherwise, a new lrs stream is created and returned.
+func (c *LRSClient) getOrCreateLRSStream(serverIdentifier clients.ServerIdentifier) (*streamImpl, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if c.logger.V(2) {
+		c.logger.Infof("Received request for a reference to an lrs stream for server identifier %q", serverIdentifier)
+	}
+
+	// Use an existing stream, if one exists for this server identifier.
+	if s, ok := c.lrsStreams[serverIdentifier]; ok {
+		if c.logger.V(2) {
+			c.logger.Infof("Reusing an existing lrs stream for server identifier %q", serverIdentifier)
+		}
+		c.lrsRefs[serverIdentifier]++
+		return s, nil
+	}
+
+	if c.logger.V(2) {
+		c.logger.Infof("Creating a new lrs stream for server identifier %q", serverIdentifier)
+	}
+
+	// Create a new transport and create a new lrs stream, and add it to the
+	// map of lrs streams.
+	tr, err := c.transportBuilder.Build(serverIdentifier)
+	if err != nil {
+		return nil, fmt.Errorf("lrsclient: failed to create transport for server identifier %s: %v", serverIdentifier, err)
+	}
+
+	nodeProto := clientsinternal.NodeProto(c.node)
+	nodeProto.ClientFeatures = []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}
+	lrs := newStreamImpl(streamOpts{
+		transport: tr,
+		backoff:   c.backoff,
+		nodeProto: nodeProto,
+		logPrefix: clientPrefix(c),
+	})
+
+	// Register a stop function that decrements the reference count, stops
+	// the LRS stream when the last reference is removed and closes the
+	// transport and removes the lrs stream and its references from the
+	// respective maps. Before closing the stream, it waits for the provided
+	// context to be done (timeout or cancellation).
+	stop := func(ctx context.Context) {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+
+		if r, ok := c.lrsRefs[serverIdentifier]; !ok || r == 0 {
+			c.logger.Errorf("Attempting to stop already stopped StreamImpl")
+			return
+		}
+		c.lrsRefs[serverIdentifier]--
+		if c.lrsRefs[serverIdentifier] != 0 {
+			return
+		}
+
+		lrs.finalSendRequest <- struct{}{}
+
+		select {
+		case err := <-lrs.finalSendDone:
+			if err != nil {
+				c.logger.Warningf("Final send attempt failed: %v", err)
+			}
+		case <-ctx.Done():
+			c.logger.Warningf("Context canceled before finishing the final send attempt: %v", err)
+		}
+
+		lrs.cancelStream()
+		lrs.cancelStream = nil
+		lrs.logger.Infof("Stopping LRS stream")
+		<-lrs.doneCh
+
+		delete(c.lrsStreams, serverIdentifier)
+		tr.Close()
+	}
+	lrs.loadStore.stop = stop
+
+	c.lrsStreams[serverIdentifier] = lrs
+	c.lrsRefs[serverIdentifier] = 1
+
+	return lrs, nil
+}

internal/xds/clients/lrsclient/lrsconfig.go

@@ -0,0 +1,35 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package lrsclient
+
+import (
+	"google.golang.org/grpc/internal/xds/clients"
+)
+
+// Config is used to configure an LRS client. After one has been passed to the
+// LRS client's New function, no part of it may modified. A Config may be
+// reused; the lrsclient package will also not modify it.
+type Config struct {
+	// Node is the identity of the client application reporting load to the
+	// LRS server.
+	Node clients.Node
+
+	// TransportBuilder is used to connect to the LRS server.
+	TransportBuilder clients.TransportBuilder
+}