snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/stats/labels.go

@@ -0,0 +1,74 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package stats provides internal stats related functionality.
+package stats
+
+import (
+	"context"
+	"maps"
+)
+
+// LabelCallback is a function that is executed when telemetry
+// label keys are updated.
+type LabelCallback func(map[string]string)
+type telemetryLabelCallbackKey struct{}
+
+// UpdateLabels executes registered telemetry callbacks with the update labels. Labels
+// are copied before being processed by any callbacks to ensure mutations are not
+// shared among derived contexts.
+//
+// It is the responsibility of the registrant to handle conflicts or label resets.
+func UpdateLabels(ctx context.Context, update map[string]string) {
+	executeTelemetryLabelCallbacks(ctx, update)
+}
+
+// RegisterTelemetryLabelCallback registers a callback function that is executed whenever
+// telemetry labels are updated.
+func RegisterTelemetryLabelCallback(ctx context.Context, callback LabelCallback) context.Context {
+	if callback == nil {
+		return ctx
+	}
+
+	callbacks, ok := ctx.Value(telemetryLabelCallbackKey{}).([]LabelCallback)
+	if !ok {
+		return context.WithValue(ctx, telemetryLabelCallbackKey{}, []LabelCallback{callback})
+	}
+	return context.WithValue(ctx, telemetryLabelCallbackKey{}, append(append([]LabelCallback(nil), callbacks...), callback))
+
+}
+
+// executeTelemetryLabelCallback runs the registered callbacks in the order they were
+// registered on the context with the provided labels. If no callbacks are registered
+// it does nothing.
+//
+// To ensure callbacks do not mutate the state of the provided label map it is copied
+// before execution.
+func executeTelemetryLabelCallbacks(ctx context.Context, labels map[string]string) {
+	callbacks, ok := ctx.Value(telemetryLabelCallbackKey{}).([]LabelCallback)
+	if !ok {
+		return
+	}
+
+	labelsCopy := map[string]string{}
+	maps.Copy(labelsCopy, labels)
+	for _, callback := range callbacks {
+		callback(labelsCopy)
+	}
+
+}

internal/stats/labels_test.go

@@ -0,0 +1,216 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package stats_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+
+	"google.golang.org/grpc/internal/stats"
+)
+
+// TestTelemetryLabelCallbacks tests registering a callback function with the context and
+// the effects of executing the callback on a local label state tracker. Each test
+// case constructs a new context with the provided callback registered.
+func (s) TestTelemetryLabels(t *testing.T) {
+	commonLabelValues := map[string]string{"grpc.lb.backend_service": "grpc.lb.backend_service_val", "grpc.lb.locality": "grpc.lb.locality_val"}
+
+	tests := map[string]struct {
+		callbacks        func(tracker map[string]string) []func(map[string]string)
+		additionalLabels map[string]string
+		wantLabels       map[string]string
+	}{
+		"NilCallback": {
+			callbacks: func(map[string]string) []func(map[string]string) {
+				return []func(map[string]string){nil}
+			},
+			wantLabels: map[string]string{},
+		},
+		"NoOPCallback": {
+			callbacks: func(map[string]string) []func(map[string]string) {
+				return []func(map[string]string){
+					func(map[string]string) {},
+				}
+			},
+			wantLabels: map[string]string{},
+		},
+		"TrackingCallback": {
+			callbacks: func(tracker map[string]string) []func(map[string]string) {
+				return []func(map[string]string){
+					func(u map[string]string) {
+						for key, value := range u {
+							tracker[key] = value
+						}
+					},
+				}
+			},
+			wantLabels: map[string]string{
+				"grpc.lb.backend_service": "grpc.lb.backend_service_val",
+				"grpc.lb.locality":        "grpc.lb.locality_val",
+			},
+		},
+		"MultipleInvocations": {
+			callbacks: func(tracker map[string]string) []func(map[string]string) {
+				return []func(map[string]string){
+					func(u map[string]string) {
+						for key, value := range u {
+							tracker[key] = value
+						}
+					},
+				}
+			},
+			additionalLabels: map[string]string{"grpc.lb.other_label": "grpc.lb.other_val"},
+			wantLabels: map[string]string{
+				"grpc.lb.backend_service": "grpc.lb.backend_service_val",
+				"grpc.lb.locality":        "grpc.lb.locality_val",
+				"grpc.lb.other_label":     "grpc.lb.other_val",
+			},
+		},
+		"MultipleInvocationsWithSameLabel": {
+			callbacks: func(tracker map[string]string) []func(map[string]string) {
+				return []func(map[string]string){
+					func(u map[string]string) {
+						for key, value := range u {
+							tracker[key] = value
+						}
+					},
+				}
+			},
+			additionalLabels: map[string]string{"grpc.lb.backend_service": "grpc.lb.backend_service_other_val"},
+			wantLabels: map[string]string{
+				"grpc.lb.backend_service": "grpc.lb.backend_service_other_val",
+				"grpc.lb.locality":        "grpc.lb.locality_val",
+			},
+		},
+		"MultipleCallbacks": {
+			callbacks: func(tracker map[string]string) []func(map[string]string) {
+				return []func(map[string]string){
+					func(u map[string]string) {
+						for key, value := range u {
+							k, v := key+"_first", value+"_first"
+							tracker[k] = v
+						}
+					},
+					func(u map[string]string) {
+						for key, value := range u {
+							k, v := key+"_second", value+"_second"
+							tracker[k] = v
+						}
+					},
+				}
+			},
+			wantLabels: map[string]string{
+				"grpc.lb.backend_service_first":  "grpc.lb.backend_service_val_first",
+				"grpc.lb.locality_first":         "grpc.lb.locality_val_first",
+				"grpc.lb.backend_service_second": "grpc.lb.backend_service_val_second",
+				"grpc.lb.locality_second":        "grpc.lb.locality_val_second",
+			},
+		},
+	}
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			tracker := map[string]string{}
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+
+			for _, c := range test.callbacks(tracker) {
+				ctx = stats.RegisterTelemetryLabelCallback(ctx, c)
+			}
+
+			stats.UpdateLabels(ctx, commonLabelValues)
+			stats.UpdateLabels(ctx, test.additionalLabels)
+
+			if diff := cmp.Diff(tracker, test.wantLabels); diff != "" {
+				t.Fatalf("tracked labels did not match expected values (-got, +want): %v", diff)
+			}
+		})
+	}
+}
+
+// TestTelemetryLabelsMutation verifies that callbacks mutating labels do not
+// affect the original map passed to UpdateLabels.
+func (s) TestTelemetryLabelsMutation(t *testing.T) {
+	originalLabels := map[string]string{"key1": "val1", "key2": "val2"}
+	// Create a copy to compare against after calling UpdateLabels.
+	wantLabels := map[string]string{"key1": "val1", "key2": "val2"}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+
+	ctx = stats.RegisterTelemetryLabelCallback(ctx, func(u map[string]string) {
+		u["key1"] = "mutated1"
+		u["newkey"] = "newval"
+	})
+
+	stats.UpdateLabels(ctx, originalLabels)
+
+	if diff := cmp.Diff(originalLabels, wantLabels); diff != "" {
+		t.Fatalf("original labels were mutated by callback (-got, +want): %v", diff)
+	}
+}
+
+// TestTelemetryLabelsDerivedContexts verifies that creating derived contexts
+// with additional callbacks does not override or affect sibling contexts.
+func (s) TestTelemetryLabelsDerivedContexts(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+	defer cancel()
+
+	var baseExecuted, derived1Executed, derived2Executed uint
+
+	ctxBase := stats.RegisterTelemetryLabelCallback(ctx, func(map[string]string) {
+		baseExecuted++
+	})
+
+	ctxDerived1 := stats.RegisterTelemetryLabelCallback(ctxBase, func(map[string]string) {
+		derived1Executed++
+	})
+
+	ctxDerived2 := stats.RegisterTelemetryLabelCallback(ctxBase, func(map[string]string) {
+		derived2Executed++
+	})
+
+	labels := map[string]string{"k": "v"}
+
+	// Run UpdateLabels to execute the telemetry callbacks on ctxDerived1 which should only increment
+	// the base and first derived context counters
+	stats.UpdateLabels(ctxDerived1, labels)
+
+	if baseExecuted != 1 || derived1Executed != 1 || derived2Executed != 0 {
+		t.Errorf("UpdateLabels(ctxDerived1) executed: base=%v, derived1=%v, derived2=%v; want 1, 1, 0", baseExecuted, derived1Executed, derived2Executed)
+	}
+
+	// Run UpdateLabels to execute the telemetry callbacks on ctxDerived2 which should only increment
+	// the base and second derived context counters
+	stats.UpdateLabels(ctxDerived2, labels)
+
+	if baseExecuted != 2 || derived1Executed != 1 || derived2Executed != 1 {
+		t.Errorf("UpdateLabels(ctxDerived2) executed: base=%v, derived1=%v, derived2=%v; want 2, 1, 1", baseExecuted, derived1Executed, derived2Executed)
+	}
+
+	// Run UpdateLabels to execute the telemetry callbacks on ctxBase which should only increment
+	// the base context counters
+	stats.UpdateLabels(ctxBase, labels)
+
+	if baseExecuted != 3 || derived1Executed != 1 || derived2Executed != 1 {
+		t.Errorf("UpdateLabels(ctxDerived2) executed: base=%v, derived1=%v, derived2=%v; want 3, 1, 1", baseExecuted, derived1Executed, derived2Executed)
+	}
+}

internal/stats/metrics_recorder_list.go

@@ -0,0 +1,175 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package stats
+
+import (
+	"fmt"
+
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats"
+)
+
+// MetricsRecorderList forwards Record calls to all of its metricsRecorders.
+//
+// It eats any record calls where the label values provided do not match the
+// number of label keys.
+type MetricsRecorderList struct {
+	internal.EnforceMetricsRecorderEmbedding
+	// metricsRecorders are the metrics recorders this list will forward to.
+	metricsRecorders []estats.MetricsRecorder
+}
+
+// NewMetricsRecorderList creates a new metric recorder list with all the stats
+// handlers provided which implement the MetricsRecorder interface.
+// If no stats handlers provided implement the MetricsRecorder interface,
+// the MetricsRecorder list returned is a no-op.
+func NewMetricsRecorderList(shs []stats.Handler) *MetricsRecorderList {
+	var mrs []estats.MetricsRecorder
+	for _, sh := range shs {
+		if mr, ok := sh.(estats.MetricsRecorder); ok {
+			mrs = append(mrs, mr)
+		}
+	}
+	return &MetricsRecorderList{
+		metricsRecorders: mrs,
+	}
+}
+
+func verifyLabels(desc *estats.MetricDescriptor, labelsRecv ...string) {
+	if got, want := len(labelsRecv), len(desc.Labels)+len(desc.OptionalLabels); got != want {
+		panic(fmt.Sprintf("Received %d labels in call to record metric %q, but expected %d.", got, desc.Name, want))
+	}
+}
+
+// RecordInt64Count records the measurement alongside labels on the int
+// count associated with the provided handle.
+func (l *MetricsRecorderList) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordInt64Count(handle, incr, labels...)
+	}
+}
+
+// RecordInt64UpDownCount records the measurement alongside labels on the int
+// count associated with the provided handle.
+func (l *MetricsRecorderList) RecordInt64UpDownCount(handle *estats.Int64UpDownCountHandle, incr int64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordInt64UpDownCount(handle, incr, labels...)
+	}
+}
+
+// RecordFloat64Count records the measurement alongside labels on the float
+// count associated with the provided handle.
+func (l *MetricsRecorderList) RecordFloat64Count(handle *estats.Float64CountHandle, incr float64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordFloat64Count(handle, incr, labels...)
+	}
+}
+
+// RecordInt64Histo records the measurement alongside labels on the int
+// histo associated with the provided handle.
+func (l *MetricsRecorderList) RecordInt64Histo(handle *estats.Int64HistoHandle, incr int64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordInt64Histo(handle, incr, labels...)
+	}
+}
+
+// RecordFloat64Histo records the measurement alongside labels on the float
+// histo associated with the provided handle.
+func (l *MetricsRecorderList) RecordFloat64Histo(handle *estats.Float64HistoHandle, incr float64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordFloat64Histo(handle, incr, labels...)
+	}
+}
+
+// RecordInt64Gauge records the measurement alongside labels on the int
+// gauge associated with the provided handle.
+func (l *MetricsRecorderList) RecordInt64Gauge(handle *estats.Int64GaugeHandle, incr int64, labels ...string) {
+	verifyLabels(handle.Descriptor(), labels...)
+
+	for _, metricRecorder := range l.metricsRecorders {
+		metricRecorder.RecordInt64Gauge(handle, incr, labels...)
+	}
+}
+
+// RegisterAsyncReporter forwards the registration to all underlying metrics
+// recorders.
+//
+// It returns a cleanup function that, when called, invokes the cleanup function
+// returned by each underlying recorder, ensuring the reporter is unregistered
+// from all of them.
+func (l *MetricsRecorderList) RegisterAsyncReporter(reporter estats.AsyncMetricReporter, metrics ...estats.AsyncMetric) func() {
+	descriptorsMap := make(map[*estats.MetricDescriptor]bool, len(metrics))
+	for _, m := range metrics {
+		descriptorsMap[m.Descriptor()] = true
+	}
+	unregisterFns := make([]func(), 0, len(l.metricsRecorders))
+	for _, mr := range l.metricsRecorders {
+		// Wrap the AsyncMetricsRecorder to intercept calls to RecordInt64Gauge
+		// and validate the labels.
+		wrappedCallback := func(recorder estats.AsyncMetricsRecorder) error {
+			wrappedRecorder := &asyncRecorderWrapper{
+				delegate:    recorder,
+				descriptors: descriptorsMap,
+			}
+			return reporter.Report(wrappedRecorder)
+		}
+		unregisterFns = append(unregisterFns, mr.RegisterAsyncReporter(estats.AsyncMetricReporterFunc(wrappedCallback), metrics...))
+	}
+
+	// Wrap the cleanup function using the internal delegate.
+	// In production, this returns realCleanup as-is.
+	// In tests, the leak checker can swap this to track the registration lifetime.
+	return internal.AsyncReporterCleanupDelegate(defaultCleanUp(unregisterFns))
+}
+
+func defaultCleanUp(unregisterFns []func()) func() {
+	return func() {
+		for _, unregister := range unregisterFns {
+			unregister()
+		}
+	}
+}
+
+type asyncRecorderWrapper struct {
+	delegate    estats.AsyncMetricsRecorder
+	descriptors map[*estats.MetricDescriptor]bool
+}
+
+// RecordIntAsync64Gauge records the measurement alongside labels on the int
+// gauge associated with the provided handle.
+func (w *asyncRecorderWrapper) RecordInt64AsyncGauge(handle *estats.Int64AsyncGaugeHandle, value int64, labels ...string) {
+	// Ensure only metrics for descriptors passed during callback registration
+	// are emitted.
+	d := handle.Descriptor()
+	if _, ok := w.descriptors[d]; !ok {
+		return
+	}
+	// Validate labels and delegate.
+	verifyLabels(d, labels...)
+	w.delegate.RecordInt64AsyncGauge(handle, value, labels...)
+}

internal/stats/metrics_recorder_list_test.go

@@ -0,0 +1,336 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package stats_test implements an e2e test for the Metrics Recorder List
+// component of the Client Conn.
+package stats_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/pickfirst"
+	"google.golang.org/grpc/credentials/insecure"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+	istats "google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/testutils/stats"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/serviceconfig"
+	gstats "google.golang.org/grpc/stats"
+)
+
+var defaultTestTimeout = 5 * time.Second
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var (
+	intCountHandle = estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "simple counter",
+		Description:    "sum of all emissions from tests",
+		Unit:           "int",
+		Labels:         []string{"int counter label"},
+		OptionalLabels: []string{"int counter optional label"},
+		Default:        false,
+	})
+	floatCountHandle = estats.RegisterFloat64Count(estats.MetricDescriptor{
+		Name:           "float counter",
+		Description:    "sum of all emissions from tests",
+		Unit:           "float",
+		Labels:         []string{"float counter label"},
+		OptionalLabels: []string{"float counter optional label"},
+		Default:        false,
+	})
+	intHistoHandle = estats.RegisterInt64Histo(estats.MetricDescriptor{
+		Name:           "int histo",
+		Description:    "sum of all emissions from tests",
+		Unit:           "int",
+		Labels:         []string{"int histo label"},
+		OptionalLabels: []string{"int histo optional label"},
+		Default:        false,
+	})
+	floatHistoHandle = estats.RegisterFloat64Histo(estats.MetricDescriptor{
+		Name:           "float histo",
+		Description:    "sum of all emissions from tests",
+		Unit:           "float",
+		Labels:         []string{"float histo label"},
+		OptionalLabels: []string{"float histo optional label"},
+		Default:        false,
+	})
+	intGaugeHandle = estats.RegisterInt64Gauge(estats.MetricDescriptor{
+		Name:           "simple gauge",
+		Description:    "the most recent int emitted by test",
+		Unit:           "int",
+		Labels:         []string{"int gauge label"},
+		OptionalLabels: []string{"int gauge optional label"},
+		Default:        false,
+	})
+)
+
+func init() {
+	balancer.Register(recordingLoadBalancerBuilder{})
+}
+
+const recordingLoadBalancerName = "recording_load_balancer"
+
+type recordingLoadBalancerBuilder struct{}
+
+func (recordingLoadBalancerBuilder) Name() string {
+	return recordingLoadBalancerName
+}
+
+func (recordingLoadBalancerBuilder) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer {
+	intCountHandle.Record(cc.MetricsRecorder(), 1, "int counter label val", "int counter optional label val")
+	floatCountHandle.Record(cc.MetricsRecorder(), 2, "float counter label val", "float counter optional label val")
+	intHistoHandle.Record(cc.MetricsRecorder(), 3, "int histo label val", "int histo optional label val")
+	floatHistoHandle.Record(cc.MetricsRecorder(), 4, "float histo label val", "float histo optional label val")
+	intGaugeHandle.Record(cc.MetricsRecorder(), 5, "int gauge label val", "int gauge optional label val")
+	return &recordingLoadBalancer{
+		Balancer: balancer.Get(pickfirst.Name).Build(cc, bOpts),
+	}
+}
+
+type recordingLoadBalancer struct {
+	balancer.Balancer
+}
+
+// TestMetricsRecorderList tests the metrics recorder list functionality of the
+// ClientConn. It configures a global and local stats handler Dial Option. These
+// stats handlers implement the MetricsRecorder interface. It also configures a
+// balancer which registers metrics and records on metrics at build time. This
+// test then asserts that the recorded metrics show up on both configured stats
+// handlers.
+func (s) TestMetricsRecorderList(t *testing.T) {
+	cleanup := internal.SnapshotMetricRegistryForTesting()
+	defer cleanup()
+
+	mr := manual.NewBuilderWithScheme("test-metrics-recorder-list")
+	defer mr.Close()
+
+	json := `{"loadBalancingConfig": [{"recording_load_balancer":{}}]}`
+	sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(json)
+	mr.InitialState(resolver.State{
+		ServiceConfig: sc,
+	})
+
+	// Create two stats.Handlers which also implement MetricsRecorder, configure
+	// one as a global dial option and one as a local dial option.
+	mr1 := stats.NewTestMetricsRecorder()
+	mr2 := stats.NewTestMetricsRecorder()
+
+	defer internal.ClearGlobalDialOptions()
+	internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(grpc.WithStatsHandler(mr1))
+
+	cc, err := grpc.NewClient(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(mr2))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+
+	tsc := testgrpc.NewTestServiceClient(cc)
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Trigger the recording_load_balancer to build, which will trigger metrics
+	// to record.
+	tsc.UnaryCall(ctx, &testpb.SimpleRequest{})
+
+	mdWant := stats.MetricsData{
+		Handle:    intCountHandle.Descriptor(),
+		IntIncr:   1,
+		LabelKeys: []string{"int counter label", "int counter optional label"},
+		LabelVals: []string{"int counter label val", "int counter optional label val"},
+	}
+	if err := mr1.WaitForInt64Count(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := mr2.WaitForInt64Count(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	mdWant = stats.MetricsData{
+		Handle:    floatCountHandle.Descriptor(),
+		FloatIncr: 2,
+		LabelKeys: []string{"float counter label", "float counter optional label"},
+		LabelVals: []string{"float counter label val", "float counter optional label val"},
+	}
+	if err := mr1.WaitForFloat64Count(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := mr2.WaitForFloat64Count(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	mdWant = stats.MetricsData{
+		Handle:    intHistoHandle.Descriptor(),
+		IntIncr:   3,
+		LabelKeys: []string{"int histo label", "int histo optional label"},
+		LabelVals: []string{"int histo label val", "int histo optional label val"},
+	}
+	if err := mr1.WaitForInt64Histo(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := mr2.WaitForInt64Histo(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	mdWant = stats.MetricsData{
+		Handle:    floatHistoHandle.Descriptor(),
+		FloatIncr: 4,
+		LabelKeys: []string{"float histo label", "float histo optional label"},
+		LabelVals: []string{"float histo label val", "float histo optional label val"},
+	}
+	if err := mr1.WaitForFloat64Histo(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := mr2.WaitForFloat64Histo(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	mdWant = stats.MetricsData{
+		Handle:    intGaugeHandle.Descriptor(),
+		IntIncr:   5,
+		LabelKeys: []string{"int gauge label", "int gauge optional label"},
+		LabelVals: []string{"int gauge label val", "int gauge optional label val"},
+	}
+	if err := mr1.WaitForInt64Gauge(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+	if err := mr2.WaitForInt64Gauge(ctx, mdWant); err != nil {
+		t.Fatal(err.Error())
+	}
+}
+
+// TestMetricRecorderListPanic tests that the metrics recorder list panics if
+// received the wrong number of labels for a particular metric.
+func (s) TestMetricRecorderListPanic(t *testing.T) {
+	cleanup := internal.SnapshotMetricRegistryForTesting()
+	defer cleanup()
+
+	intCountHandle := estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "simple counter",
+		Description:    "sum of all emissions from tests",
+		Unit:           "int",
+		Labels:         []string{"int counter label"},
+		OptionalLabels: []string{"int counter optional label"},
+		Default:        false,
+	})
+	mrl := istats.NewMetricsRecorderList(nil)
+
+	want := `Received 1 labels in call to record metric "simple counter", but expected 2.`
+	defer func() {
+		if r := recover(); !strings.Contains(fmt.Sprint(r), want) {
+			t.Errorf("expected panic contains %q, got %q", want, r)
+		}
+	}()
+
+	intCountHandle.Record(mrl, 1, "only one label")
+}
+
+// TestMetricsRecorderList_RegisterAsyncReporter verifies that the list implementation
+// correctly fans out registration calls to all underlying recorders and
+// aggregates the cleanup calls.
+func (s) TestMetricsRecorderList_RegisterAsyncReporter(t *testing.T) {
+	spy1 := &spyMetricsRecorder{name: "spy1"}
+	spy2 := &spyMetricsRecorder{name: "spy2"}
+	spy3 := &spyMetricsRecorder{name: "spy3"}
+
+	list := istats.NewMetricsRecorderList([]gstats.Handler{spy1, spy2, spy3})
+
+	desc := &estats.MetricDescriptor{Name: "test_metric", Description: "test"}
+	mockMetric := &mockAsyncMetric{d: desc}
+
+	dummyReporter := estats.AsyncMetricReporterFunc(func(estats.AsyncMetricsRecorder) error {
+		return nil
+	})
+	cleanup := list.RegisterAsyncReporter(dummyReporter, mockMetric)
+
+	// Check that RegisterAsyncReporter was called exactly once on ALL spies
+	if spy1.registerCalledCount != 1 {
+		t.Errorf("spy1 register called %d times, want 1", spy1.registerCalledCount)
+	}
+	if spy2.registerCalledCount != 1 {
+		t.Errorf("spy2 register called %d times, want 1", spy2.registerCalledCount)
+	}
+	if spy3.registerCalledCount != 1 {
+		t.Errorf("spy3 register called %d times, want 1", spy3.registerCalledCount)
+	}
+
+	// Verify that cleanup has NOT been called yet
+	if spy1.cleanupCalledCount != 0 {
+		t.Error("spy1 cleanup called prematurely")
+	}
+
+	cleanup()
+
+	// Check that the cleanup function returned by the list actually triggers
+	// the cleanup logic on ALL underlying spies.
+	if spy1.cleanupCalledCount != 1 {
+		t.Errorf("spy1 cleanup called %d times, want 1", spy1.cleanupCalledCount)
+	}
+	if spy2.cleanupCalledCount != 1 {
+		t.Errorf("spy2 cleanup called %d times, want 1", spy2.cleanupCalledCount)
+	}
+	if spy3.cleanupCalledCount != 1 {
+		t.Errorf("spy3 cleanup called %d times, want 1", spy3.cleanupCalledCount)
+	}
+}
+
+// --- Helpers & Spies ---
+
+// mockAsyncMetric implements estats.AsyncMetric
+type mockAsyncMetric struct {
+	estats.AsyncMetric
+	d *estats.MetricDescriptor
+}
+
+func (m *mockAsyncMetric) Descriptor() *estats.MetricDescriptor {
+	return m.d
+}
+
+// spyMetricsRecorder implements estats.MetricsRecorder
+type spyMetricsRecorder struct {
+	stats.TestMetricsRecorder
+	name                string
+	registerCalledCount int
+	cleanupCalledCount  int
+}
+
+// RegisterAsyncReporter implements the interface and tracks calls.
+func (s *spyMetricsRecorder) RegisterAsyncReporter(estats.AsyncMetricReporter, ...estats.AsyncMetric) func() {
+	s.registerCalledCount++
+
+	// Return a cleanup function that tracks if it was called
+	return func() {
+		s.cleanupCalledCount++
+	}
+}

internal/stats/stats.go

@@ -0,0 +1,70 @@
+/*
+ *
+ * Copyright 2025 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package stats
+
+import (
+	"context"
+
+	"google.golang.org/grpc/stats"
+)
+
+type combinedHandler struct {
+	handlers []stats.Handler
+}
+
+// NewCombinedHandler combines multiple stats.Handlers into a single handler.
+//
+// It returns nil if no handlers are provided. If only one handler is
+// provided, it is returned directly without wrapping.
+func NewCombinedHandler(handlers ...stats.Handler) stats.Handler {
+	switch len(handlers) {
+	case 0:
+		return nil
+	case 1:
+		return handlers[0]
+	default:
+		return &combinedHandler{handlers: handlers}
+	}
+}
+
+func (ch *combinedHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	for _, h := range ch.handlers {
+		ctx = h.TagRPC(ctx, info)
+	}
+	return ctx
+}
+
+func (ch *combinedHandler) HandleRPC(ctx context.Context, stats stats.RPCStats) {
+	for _, h := range ch.handlers {
+		h.HandleRPC(ctx, stats)
+	}
+}
+
+func (ch *combinedHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context {
+	for _, h := range ch.handlers {
+		ctx = h.TagConn(ctx, info)
+	}
+	return ctx
+}
+
+func (ch *combinedHandler) HandleConn(ctx context.Context, stats stats.ConnStats) {
+	for _, h := range ch.handlers {
+		h.HandleConn(ctx, stats)
+	}
+}