snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

stats/opentelemetry/client_metrics.go

@@ -0,0 +1,287 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"maps"
+	"sync/atomic"
+	"time"
+
+	otelattribute "go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
+	"google.golang.org/grpc"
+	estats "google.golang.org/grpc/experimental/stats"
+	istats "google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+type clientMetricsHandler struct {
+	estats.MetricsRecorder
+	options       Options
+	clientMetrics clientMetrics
+}
+
+func (h *clientMetricsHandler) initializeMetrics() {
+	// Will set no metrics to record, logically making this stats handler a
+	// no-op.
+	if h.options.MetricsOptions.MeterProvider == nil {
+		return
+	}
+
+	meter := h.options.MetricsOptions.MeterProvider.Meter("grpc-go", otelmetric.WithInstrumentationVersion(grpc.Version))
+	if meter == nil {
+		return
+	}
+
+	metrics := h.options.MetricsOptions.Metrics
+	if metrics == nil {
+		metrics = DefaultMetrics()
+	}
+
+	h.clientMetrics.attemptStarted = createInt64Counter(metrics.Metrics(), "grpc.client.attempt.started", meter, otelmetric.WithUnit("{attempt}"), otelmetric.WithDescription("Number of client call attempts started."))
+	h.clientMetrics.attemptDuration = createFloat64Histogram(metrics.Metrics(), "grpc.client.attempt.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("End-to-end time taken to complete a client call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...))
+	h.clientMetrics.attemptSentTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.client.attempt.sent_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes sent per client call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...))
+	h.clientMetrics.attemptRcvdTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.client.attempt.rcvd_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes received per call attempt."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...))
+	h.clientMetrics.callDuration = createFloat64Histogram(metrics.Metrics(), "grpc.client.call.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("Time taken by gRPC to complete an RPC from application's perspective."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...))
+
+	rm := &registryMetrics{
+		optionalLabels: h.options.MetricsOptions.OptionalLabels,
+	}
+	h.MetricsRecorder = rm
+	rm.registerMetrics(metrics, meter)
+}
+
+// getOrCreateCallInfo returns the existing callInfo from context if present,
+// or creates and attaches a new one.
+func getOrCreateCallInfo(ctx context.Context, cc *grpc.ClientConn, method string, opts ...grpc.CallOption) (context.Context, *callInfo) {
+	ci := getCallInfo(ctx)
+	if ci == nil {
+		ci = &callInfo{
+			target: cc.CanonicalTarget(),
+			method: determineMethod(method, opts...),
+		}
+		ctx = context.WithValue(ctx, callInfoKey{}, ci)
+	}
+	return ctx, ci
+}
+
+func (h *clientMetricsHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+	ctx, ci := getOrCreateCallInfo(ctx, cc, method, opts...)
+
+	if h.options.MetricsOptions.pluginOption != nil {
+		md := h.options.MetricsOptions.pluginOption.GetMetadata()
+		for k, vs := range md {
+			for _, v := range vs {
+				ctx = metadata.AppendToOutgoingContext(ctx, k, v)
+			}
+		}
+	}
+
+	startTime := time.Now()
+	err := invoker(ctx, method, req, reply, cc, opts...)
+	h.perCallMetrics(ctx, err, startTime, ci)
+	return err
+}
+
+// determineMethod determines the method to record attributes with. This will be
+// "other" if StaticMethod isn't specified or if method filter is set and
+// specifies, the method name as is otherwise.
+func determineMethod(method string, opts ...grpc.CallOption) string {
+	for _, opt := range opts {
+		if _, ok := opt.(grpc.StaticMethodCallOption); ok {
+			return removeLeadingSlash(method)
+		}
+	}
+	return "other"
+}
+
+func (h *clientMetricsHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	ctx, ci := getOrCreateCallInfo(ctx, cc, method, opts...)
+
+	if h.options.MetricsOptions.pluginOption != nil {
+		md := h.options.MetricsOptions.pluginOption.GetMetadata()
+		for k, vs := range md {
+			for _, v := range vs {
+				ctx = metadata.AppendToOutgoingContext(ctx, k, v)
+			}
+		}
+	}
+
+	startTime := time.Now()
+	callback := func(err error) {
+		h.perCallMetrics(ctx, err, startTime, ci)
+	}
+	opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...)
+	return streamer(ctx, desc, cc, method, opts...)
+}
+
+// perCallMetrics records per call metrics for both unary and stream calls.
+func (h *clientMetricsHandler) perCallMetrics(ctx context.Context, err error, startTime time.Time, ci *callInfo) {
+	callLatency := float64(time.Since(startTime)) / float64(time.Second)
+	attributes := []otelattribute.KeyValue{
+		otelattribute.String("grpc.method", ci.method),
+		otelattribute.String("grpc.target", ci.target),
+		otelattribute.String("grpc.status", canonicalString(status.Code(err))),
+	}
+	for _, o := range h.options.MetricsOptions.OptionalLabels {
+		if o == "grpc.client.call.custom" {
+			label := estats.CustomLabelFromContext(ctx)
+			attributes = append(attributes, otelattribute.String(o, label))
+		}
+	}
+	attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))
+	h.clientMetrics.callDuration.Record(ctx, callLatency, attrs)
+}
+
+// TagConn exists to satisfy stats.Handler.
+func (h *clientMetricsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+// HandleConn exists to satisfy stats.Handler.
+func (h *clientMetricsHandler) HandleConn(context.Context, stats.ConnStats) {}
+
+// TagRPC implements per RPC attempt context management for metrics.
+func (h *clientMetricsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	ctx, ri := getOrCreateClientRPCInfo(ctx, info)
+	// Numerous stats handlers can be used for the same channel. This callback
+	// ensures that all label updates are propagated to the rpc attempt info across
+	// derived contexts.
+	ctx = istats.RegisterTelemetryLabelCallback(ctx, func(labels map[string]string) {
+		maps.Copy(ri.ai.xdsLabels, labels)
+	})
+
+	return ctx
+}
+
+// HandleRPC handles per RPC stats implementation.
+func (h *clientMetricsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	ri := clientRPCInfo(ctx)
+	if ri == nil {
+		logger.Error("ctx passed into client side stats handler metrics event handling has no client attempt data present")
+		return
+	}
+	h.processRPCEvent(ctx, rs, ri.ai)
+}
+
+func (h *clientMetricsHandler) processRPCEvent(ctx context.Context, s stats.RPCStats, ai *attemptInfo) {
+	switch st := s.(type) {
+	case *stats.Begin:
+		ci := getCallInfo(ctx)
+		if ci == nil {
+			logger.Error("ctx passed into client side stats handler metrics event handling has no metrics data present")
+			return
+		}
+
+		attributes := []otelattribute.KeyValue{
+			otelattribute.String("grpc.method", ci.method),
+			otelattribute.String("grpc.target", ci.target),
+		}
+		for _, o := range h.options.MetricsOptions.OptionalLabels {
+			if o == "grpc.client.call.custom" {
+				label := estats.CustomLabelFromContext(ctx)
+				attributes = append(attributes, otelattribute.String(o, label))
+			}
+		}
+
+		attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))
+		h.clientMetrics.attemptStarted.Add(ctx, 1, attrs)
+	case *stats.OutPayload:
+		atomic.AddInt64(&ai.sentCompressedBytes, int64(st.CompressedLength))
+	case *stats.InPayload:
+		atomic.AddInt64(&ai.recvCompressedBytes, int64(st.CompressedLength))
+	case *stats.InHeader:
+		h.setLabelsFromPluginOption(ai, st.Header)
+	case *stats.InTrailer:
+		h.setLabelsFromPluginOption(ai, st.Trailer)
+	case *stats.End:
+		h.processRPCEnd(ctx, ai, st)
+	default:
+	}
+}
+
+func (h *clientMetricsHandler) setLabelsFromPluginOption(ai *attemptInfo, incomingMetadata metadata.MD) {
+	if ai.pluginOptionLabels == nil && h.options.MetricsOptions.pluginOption != nil {
+		labels := h.options.MetricsOptions.pluginOption.GetLabels(incomingMetadata)
+		if labels == nil {
+			labels = map[string]string{} // Shouldn't return a nil map. Make it empty if so to ignore future Get Calls for this Attempt.
+		}
+		ai.pluginOptionLabels = labels
+	}
+}
+
+func (h *clientMetricsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo, e *stats.End) {
+	ci := getCallInfo(ctx)
+	if ci == nil {
+		logger.Error("ctx passed into client side stats handler metrics event handling has no metrics data present")
+		return
+	}
+	latency := float64(time.Since(ai.startTime)) / float64(time.Second)
+	st := "OK"
+	if e.Error != nil {
+		s, _ := status.FromError(e.Error)
+		st = canonicalString(s.Code())
+	}
+
+	attributes := []otelattribute.KeyValue{
+		otelattribute.String("grpc.method", ci.method),
+		otelattribute.String("grpc.target", ci.target),
+		otelattribute.String("grpc.status", st),
+	}
+
+	for k, v := range ai.pluginOptionLabels {
+		attributes = append(attributes, otelattribute.String(k, v))
+	}
+
+	for _, o := range h.options.MetricsOptions.OptionalLabels {
+		// TODO: Add a filter for converting to unknown if not present in the
+		// CSM Plugin Option layer by adding an optional labels API.
+		if val, ok := ai.xdsLabels[o]; ok {
+			attributes = append(attributes, otelattribute.String(o, val))
+		} else if o == "grpc.client.call.custom" {
+			label := estats.CustomLabelFromContext(ctx)
+			attributes = append(attributes, otelattribute.String(o, label))
+		}
+	}
+
+	// Allocate vararg slice once.
+	opts := []otelmetric.RecordOption{otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))}
+	h.clientMetrics.attemptDuration.Record(ctx, latency, opts...)
+	h.clientMetrics.attemptSentTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.sentCompressedBytes), opts...)
+	h.clientMetrics.attemptRcvdTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.recvCompressedBytes), opts...)
+}
+
+const (
+	// ClientAttemptStartedMetricName is the number of client call attempts
+	// started.
+	ClientAttemptStartedMetricName string = "grpc.client.attempt.started"
+	// ClientAttemptDurationMetricName is the end-to-end time taken to complete
+	// a client call attempt.
+	ClientAttemptDurationMetricName string = "grpc.client.attempt.duration"
+	// ClientAttemptSentCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes sent per client call attempt.
+	ClientAttemptSentCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.sent_total_compressed_message_size"
+	// ClientAttemptRcvdCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes received per call attempt.
+	ClientAttemptRcvdCompressedTotalMessageSizeMetricName string = "grpc.client.attempt.rcvd_total_compressed_message_size"
+	// ClientCallDurationMetricName is the time taken by gRPC to complete an RPC
+	// from application's perspective.
+	ClientCallDurationMetricName string = "grpc.client.call.duration"
+)

stats/opentelemetry/client_tracing.go

@@ -0,0 +1,155 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"log"
+	"strings"
+
+	"go.opentelemetry.io/otel/attribute"
+	otelcodes "go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	grpccodes "google.golang.org/grpc/codes"
+	"google.golang.org/grpc/stats"
+	otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	delayedResolutionEventName = "Delayed name resolution complete"
+	tracerName                 = "grpc-go"
+)
+
+type clientTracingHandler struct {
+	options Options
+}
+
+func (h *clientTracingHandler) initializeTraces() {
+	if h.options.TraceOptions.TracerProvider == nil {
+		log.Printf("TracerProvider is not provided in client TraceOptions")
+		return
+	}
+}
+
+func (h *clientTracingHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+	ctx, _ = getOrCreateCallInfo(ctx, cc, method, opts...)
+
+	var span trace.Span
+	ctx, span = h.createCallTraceSpan(ctx, method)
+	err := invoker(ctx, method, req, reply, cc, opts...)
+	h.finishTrace(err, span)
+	return err
+}
+
+func (h *clientTracingHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) {
+	ctx, _ = getOrCreateCallInfo(ctx, cc, method, opts...)
+
+	var span trace.Span
+	ctx, span = h.createCallTraceSpan(ctx, method)
+	callback := func(err error) { h.finishTrace(err, span) }
+	opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...)
+	return streamer(ctx, desc, cc, method, opts...)
+}
+
+// finishTrace sets the span status based on the RPC result and ends the span.
+// It is used to finalize tracing for both unary and streaming calls.
+func (h *clientTracingHandler) finishTrace(err error, ts trace.Span) {
+	s := status.Convert(err)
+	if s.Code() == grpccodes.OK {
+		ts.SetStatus(otelcodes.Ok, s.Message())
+	} else {
+		ts.SetStatus(otelcodes.Error, s.Message())
+	}
+	ts.End()
+}
+
+// traceTagRPC populates provided context with a new span using the
+// TextMapPropagator supplied in trace options and internal itracing.carrier.
+// It creates a new outgoing carrier which serializes information about this
+// span into gRPC Metadata, if TextMapPropagator is provided in the trace
+// options. if TextMapPropagator is not provided, it returns the context as is.
+//
+// Note: The passed attemptInfo pointer (ai) is mutated in-place. Fields such as
+// ai.traceSpan are updated directly. No new attemptInfo is returned.
+func (h *clientTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo, nameResolutionDelayed bool) context.Context {
+	// Add a "Delayed name resolution complete" event to the call span
+	// if there was name resolution delay. In case of multiple retry attempts,
+	// ensure that event is added only once.
+	callSpan := trace.SpanFromContext(ctx)
+	ci := getCallInfo(ctx)
+	if nameResolutionDelayed && !ci.nameResolutionEventAdded.Swap(true) && callSpan.SpanContext().IsValid() {
+		callSpan.AddEvent(delayedResolutionEventName)
+	}
+	mn := "Attempt." + strings.Replace(ai.method, "/", ".", -1)
+	tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version))
+	ctx, span := tracer.Start(ctx, mn)
+	carrier := otelinternaltracing.NewOutgoingCarrier(ctx)
+	h.options.TraceOptions.TextMapPropagator.Inject(ctx, carrier)
+	ai.traceSpan = span
+	return carrier.Context()
+}
+
+// createCallTraceSpan creates a call span to put in the provided context using
+// provided TraceProvider. If TraceProvider is nil, it returns context as is.
+func (h *clientTracingHandler) createCallTraceSpan(ctx context.Context, method string) (context.Context, trace.Span) {
+	mn := "Sent." + strings.Replace(removeLeadingSlash(method), "/", ".", -1)
+	tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version))
+	ctx, span := tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindClient))
+	return ctx, span
+}
+
+// TagConn exists to satisfy stats.Handler for tracing.
+func (h *clientTracingHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+// HandleConn exists to satisfy stats.Handler for tracing.
+func (h *clientTracingHandler) HandleConn(context.Context, stats.ConnStats) {}
+
+// TagRPC implements per RPC attempt context management for traces.
+func (h *clientTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	ctx, ri := getOrCreateClientRPCInfo(ctx, info)
+	ci := getCallInfo(ctx)
+	if ci == nil {
+		logger.Error("context passed into client side stats handler (TagRPC) has no call info")
+		return ctx
+	}
+	ctx = h.traceTagRPC(ctx, ri.ai, info.NameResolutionDelay)
+	return ctx
+}
+
+// HandleRPC handles per RPC tracing implementation.
+func (h *clientTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	ri := clientRPCInfo(ctx)
+	if ri == nil {
+		logger.Error("ctx passed into client side tracing handler trace event handling has no client attempt data present")
+		return
+	}
+
+	// Client-specific Begin attributes.
+	if begin, ok := rs.(*stats.Begin); ok {
+		ci := getCallInfo(ctx)
+		previousRPCAttempts := ci.previousRPCAttempts.Add(1) - 1
+		ri.ai.traceSpan.SetAttributes(
+			attribute.Int64("previous-rpc-attempts", int64(previousRPCAttempts)),
+			attribute.Bool("transparent-retry", begin.IsTransparentRetryAttempt),
+		)
+	}
+	populateSpan(rs, ri.ai)
+}

stats/opentelemetry/csm/observability.go

@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"net/url"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats/opentelemetry"
+	otelinternal "google.golang.org/grpc/stats/opentelemetry/internal"
+)
+
+// EnableObservability sets up CSM Observability for the binary globally.
+//
+// The CSM Stats Plugin is instantiated with local labels and metadata exchange
+// labels pulled from the environment, and emits metadata exchange labels from
+// the peer and local labels. Context timeouts do not trigger an error, but set
+// certain labels to "unknown".
+//
+// This function is not thread safe, and should only be invoked once in main
+// before any channels or servers are created. Returns a cleanup function to be
+// deferred in main.
+func EnableObservability(ctx context.Context, options opentelemetry.Options) func() {
+	csmPluginOption := newPluginOption(ctx)
+	clientSideOTelWithCSM := dialOptionWithCSMPluginOption(options, csmPluginOption)
+	clientSideOTel := opentelemetry.DialOption(options)
+	internal.AddGlobalPerTargetDialOptions.(func(opt any))(&perTargetDialOption{
+		clientSideOTelWithCSM: clientSideOTelWithCSM,
+		clientSideOTel:        clientSideOTel,
+	})
+
+	serverSideOTelWithCSM := serverOptionWithCSMPluginOption(options, csmPluginOption)
+	internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(serverSideOTelWithCSM)
+
+	return func() {
+		internal.ClearGlobalServerOptions()
+		internal.ClearGlobalPerTargetDialOptions()
+	}
+}
+
+type perTargetDialOption struct {
+	clientSideOTelWithCSM grpc.DialOption
+	clientSideOTel        grpc.DialOption
+}
+
+func (o *perTargetDialOption) DialOptionForTarget(parsedTarget url.URL) grpc.DialOption {
+	if determineTargetCSM(&parsedTarget) {
+		return o.clientSideOTelWithCSM
+	}
+	return o.clientSideOTel
+}
+
+func dialOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption {
+	options.MetricsOptions.OptionalLabels = []string{"csm.service_name", "csm.service_namespace_name"} // Attach the two xDS Optional Labels for this component to not filter out.
+	return dialOptionSetCSM(options, po)
+}
+
+func dialOptionSetCSM(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption {
+	otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po)
+	return opentelemetry.DialOption(options)
+}
+
+func serverOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.ServerOption {
+	otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po)
+	return opentelemetry.ServerOption(options)
+}

stats/opentelemetry/csm/observability_test.go

@@ -0,0 +1,649 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/encoding/gzip"
+	estats "google.golang.org/grpc/experimental/stats"
+	istats "google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/stubserver"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats/opentelemetry"
+	itestutils "google.golang.org/grpc/stats/opentelemetry/internal/testutils"
+)
+
+// setupEnv configures the environment for CSM Observability Testing. It sets
+// the bootstrap env var to a bootstrap file with a nodeID provided. It sets CSM
+// Env Vars as well, and mocks the resource detector's returned attribute set to
+// simulate the environment. It registers a cleanup function on the provided t
+// to restore the environment to its original state.
+func setupEnv(t *testing.T, resourceDetectorEmissions map[string]string, meshID, csmCanonicalServiceName, csmWorkloadName string) {
+	oldCSMMeshID, csmMeshIDPresent := os.LookupEnv("CSM_MESH_ID")
+	oldCSMCanonicalServiceName, csmCanonicalServiceNamePresent := os.LookupEnv("CSM_CANONICAL_SERVICE_NAME")
+	oldCSMWorkloadName, csmWorkloadNamePresent := os.LookupEnv("CSM_WORKLOAD_NAME")
+	os.Setenv("CSM_MESH_ID", meshID)
+	os.Setenv("CSM_CANONICAL_SERVICE_NAME", csmCanonicalServiceName)
+	os.Setenv("CSM_WORKLOAD_NAME", csmWorkloadName)
+
+	var attributes []attribute.KeyValue
+	for k, v := range resourceDetectorEmissions {
+		attributes = append(attributes, attribute.String(k, v))
+	}
+	// Return the attributes configured as part of the test in place
+	// of reading from resource.
+	attrSet := attribute.NewSet(attributes...)
+	origGetAttrSet := getAttrSetFromResourceDetector
+	getAttrSetFromResourceDetector = func(context.Context) *attribute.Set {
+		return &attrSet
+	}
+	t.Cleanup(func() {
+		if csmMeshIDPresent {
+			os.Setenv("CSM_MESH_ID", oldCSMMeshID)
+		} else {
+			os.Unsetenv("CSM_MESH_ID")
+		}
+		if csmCanonicalServiceNamePresent {
+			os.Setenv("CSM_CANONICAL_SERVICE_NAME", oldCSMCanonicalServiceName)
+		} else {
+			os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+		}
+		if csmWorkloadNamePresent {
+			os.Setenv("CSM_WORKLOAD_NAME", oldCSMWorkloadName)
+		} else {
+			os.Unsetenv("CSM_WORKLOAD_NAME")
+		}
+
+		getAttrSetFromResourceDetector = origGetAttrSet
+	})
+}
+
+// TestCSMPluginOptionUnary tests the CSM Plugin Option and labels. It
+// configures the environment for the CSM Plugin Option to read from. It then
+// configures a system with a gRPC Client and gRPC server with the OpenTelemetry
+// Dial and Server Option configured with a CSM Plugin Option with a certain
+// unary handler set to induce different ways of setting metadata exchange
+// labels, and makes a Unary RPC. This RPC should cause certain recording for
+// each registered metric observed through a Manual Metrics Reader on the
+// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the
+// plugin option should be attached to the relevant metrics.
+func (s) TestCSMPluginOptionUnary(t *testing.T) {
+	resourceDetectorEmissions := map[string]string{
+		"cloud.platform":     "gcp_kubernetes_engine",
+		"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+		"cloud.account.id":   "cloud_account_id_val",
+		"k8s.namespace.name": "k8s_namespace_name_val",
+		"k8s.cluster.name":   "k8s_cluster_name_val",
+	}
+	const meshID = "mesh_id"
+	const csmCanonicalServiceName = "csm_canonical_service_name"
+	const csmWorkloadName = "csm_workload_name"
+	setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName)
+
+	attributesWant := map[string]string{
+		"csm.workload_canonical_service": csmCanonicalServiceName, // from env
+		"csm.mesh_id":                    "mesh_id",               // from bootstrap env var
+
+		// No xDS Labels - this happens in a test below.
+
+		"csm.remote_workload_type":              "gcp_kubernetes_engine",
+		"csm.remote_workload_canonical_service": csmCanonicalServiceName,
+		"csm.remote_workload_project_id":        "cloud_account_id_val",
+		"csm.remote_workload_cluster_name":      "k8s_cluster_name_val",
+		"csm.remote_workload_namespace_name":    "k8s_namespace_name_val",
+		"csm.remote_workload_location":          "cloud_region_val",
+		"csm.remote_workload_name":              csmWorkloadName,
+	}
+
+	var csmLabels []attribute.KeyValue
+	for k, v := range attributesWant {
+		csmLabels = append(csmLabels, attribute.String(k, v))
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	unaryCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}))
+	tests := []struct {
+		name string
+		// To test the different operations for Unary RPC's from the interceptor
+		// level that can plumb metadata exchange header in.
+		unaryCallFunc func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
+		opts          itestutils.MetricDataOptions
+	}{
+		{
+			name: "normal-flow",
+			unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "trailers-only",
+			unaryCallFunc: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return nil, errors.New("some error") // return an error and no message - this triggers trailers only - no messages or headers sent
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:       csmLabels,
+				UnaryCallFailed: true,
+			},
+		},
+		{
+			name: "set-header",
+			unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				grpc.SetHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "send-header",
+			unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				grpc.SendHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "send-msg",
+			unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			reader := metric.NewManualReader()
+			provider := metric.NewMeterProvider(metric.WithReader(reader))
+			ss := &stubserver.StubServer{UnaryCallF: test.unaryCallFunc}
+			po := newPluginOption(ctx)
+			sopts := []grpc.ServerOption{
+				serverOptionWithCSMPluginOption(opentelemetry.Options{
+					MetricsOptions: opentelemetry.MetricsOptions{
+						MeterProvider: provider,
+						Metrics:       opentelemetry.DefaultMetrics(),
+					}}, po),
+			}
+			dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{
+				MetricsOptions: opentelemetry.MetricsOptions{
+					MeterProvider:  provider,
+					Metrics:        opentelemetry.DefaultMetrics(),
+					OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"},
+				},
+			}, po)}
+			if err := ss.Start(sopts, dopts...); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			var request *testpb.SimpleRequest
+			if test.opts.UnaryCompressedMessageSize != 0 {
+				request = &testpb.SimpleRequest{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}}
+			}
+			// Make a Unary RPC. These should cause certain metrics to be
+			// emitted, which should be able to be observed through the Metric
+			// Reader.
+			ss.Client.UnaryCall(ctx, request, grpc.UseCompressor(gzip.Name))
+			rm := &metricdata.ResourceMetrics{}
+			reader.Collect(ctx, rm)
+
+			gotMetrics := map[string]metricdata.Metrics{}
+			for _, sm := range rm.ScopeMetrics {
+				for _, m := range sm.Metrics {
+					gotMetrics[m.Name] = m
+				}
+			}
+
+			opts := test.opts
+			opts.Target = ss.Target
+			wantMetrics := itestutils.MetricDataUnary(opts)
+			gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+			itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+		})
+	}
+}
+
+// TestCSMPluginOptionStreaming tests the CSM Plugin Option and labels. It
+// configures the environment for the CSM Plugin Option to read from. It then
+// configures a system with a gRPC Client and gRPC server with the OpenTelemetry
+// Dial and Server Option configured with a CSM Plugin Option with a certain
+// streaming handler set to induce different ways of setting metadata exchange
+// labels, and makes a Streaming RPC. This RPC should cause certain recording
+// for each registered metric observed through a Manual Metrics Reader on the
+// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the
+// plugin option should be attached to the relevant metrics.
+func (s) TestCSMPluginOptionStreaming(t *testing.T) {
+	resourceDetectorEmissions := map[string]string{
+		"cloud.platform":     "gcp_kubernetes_engine",
+		"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+		"cloud.account.id":   "cloud_account_id_val",
+		"k8s.namespace.name": "k8s_namespace_name_val",
+		"k8s.cluster.name":   "k8s_cluster_name_val",
+	}
+	const meshID = "mesh_id"
+	const csmCanonicalServiceName = "csm_canonical_service_name"
+	const csmWorkloadName = "csm_workload_name"
+	setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName)
+
+	attributesWant := map[string]string{
+		"csm.workload_canonical_service": csmCanonicalServiceName, // from env
+		"csm.mesh_id":                    "mesh_id",               // from bootstrap env var
+
+		// No xDS Labels - this happens in a test below.
+
+		"csm.remote_workload_type":              "gcp_kubernetes_engine",
+		"csm.remote_workload_canonical_service": csmCanonicalServiceName,
+		"csm.remote_workload_project_id":        "cloud_account_id_val",
+		"csm.remote_workload_cluster_name":      "k8s_cluster_name_val",
+		"csm.remote_workload_namespace_name":    "k8s_namespace_name_val",
+		"csm.remote_workload_location":          "cloud_region_val",
+		"csm.remote_workload_name":              csmWorkloadName,
+	}
+
+	var csmLabels []attribute.KeyValue
+	for k, v := range attributesWant {
+		csmLabels = append(csmLabels, attribute.String(k, v))
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	streamingCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}))
+	tests := []struct {
+		name string
+		// To test the different operations for Streaming RPC's from the
+		// interceptor level that can plumb metadata exchange header in.
+		streamingCallFunc func(stream testgrpc.TestService_FullDuplexCallServer) error
+		opts              itestutils.MetricDataOptions
+	}{
+		{
+			name: "trailers-only",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "set-header",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.SetHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "send-header",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.SendHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "send-msg",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.Send(&testpb.StreamingOutputCallResponse{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}})
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                      csmLabels,
+				StreamingCompressedMessageSize: streamingCompressedSize,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			reader := metric.NewManualReader()
+			provider := metric.NewMeterProvider(metric.WithReader(reader))
+			ss := &stubserver.StubServer{FullDuplexCallF: test.streamingCallFunc}
+			po := newPluginOption(ctx)
+			sopts := []grpc.ServerOption{
+				serverOptionWithCSMPluginOption(opentelemetry.Options{
+					MetricsOptions: opentelemetry.MetricsOptions{
+						MeterProvider: provider,
+						Metrics:       opentelemetry.DefaultMetrics(),
+					}}, po),
+			}
+			dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{
+				MetricsOptions: opentelemetry.MetricsOptions{
+					MeterProvider:  provider,
+					Metrics:        opentelemetry.DefaultMetrics(),
+					OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"},
+				},
+			}, po)}
+			if err := ss.Start(sopts, dopts...); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			stream, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name))
+			if err != nil {
+				t.Fatalf("ss.Client.FullDuplexCall failed: %f", err)
+			}
+
+			if test.opts.StreamingCompressedMessageSize != 0 {
+				if err := stream.Send(&testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}}); err != nil {
+					t.Fatalf("stream.Send failed")
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("stream.Recv failed with error: %v", err)
+				}
+			}
+
+			stream.CloseSend()
+			if _, err = stream.Recv(); err != io.EOF {
+				t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+			}
+
+			rm := &metricdata.ResourceMetrics{}
+			reader.Collect(ctx, rm)
+
+			gotMetrics := map[string]metricdata.Metrics{}
+			for _, sm := range rm.ScopeMetrics {
+				for _, m := range sm.Metrics {
+					gotMetrics[m.Name] = m
+				}
+			}
+
+			opts := test.opts
+			opts.Target = ss.Target
+			wantMetrics := itestutils.MetricDataStreaming(opts)
+			gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+			itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+		})
+	}
+}
+
+// TestXDSLabels tests that xDS Labels get emitted from OpenTelemetry metrics.
+// This test configures OpenTelemetry with the CSM Plugin Option, and xDS
+// Optional Labels turned on. It then configures a test balancer that updates
+// labels, simulating the cluster_impl picker. It then makes a unary RPC, and
+// expects xDS Labels labels to be attached to emitted relevant metrics. Full
+// xDS System alongside OpenTelemetry will be tested with interop. (there is a
+// test for xDS -> Stats handler and this tests -> OTel -> emission). It also
+// tests the optional per call locality label in the same manner.
+func (s) TestXDSLabels(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	balancerName := "o11y_balancer_xds_labels"
+	balancer.Register(base.NewBalancerBuilder(balancerName, &o11yPickerBuilder{}, base.Config{}))
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Payload: &testpb.Payload{
+				Body: make([]byte, len(in.GetPayload().GetBody())),
+			}}, nil
+		},
+	}
+
+	po := newPluginOption(ctx)
+	sc := fmt.Sprintf(`{"loadBalancingConfig": [{"%s": {}}]}`, balancerName)
+	dopts := []grpc.DialOption{dialOptionSetCSM(opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider:  provider,
+			Metrics:        opentelemetry.DefaultMetrics(),
+			OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name", "grpc.lb.locality", "grpc.lb.backend_service", "grpc.client.call.custom"},
+		},
+	}, po), grpc.WithDefaultServiceConfig(sc)}
+	if err := ss.Start(nil, dopts...); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+
+	defer ss.Stop()
+	ss.Client.UnaryCall(estats.NewContextWithCustomLabel(ctx, "my-custom-label"), &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}, grpc.UseCompressor(gzip.Name))
+
+	rm := &metricdata.ResourceMetrics{}
+	reader.Collect(ctx, rm)
+
+	gotMetrics := map[string]metricdata.Metrics{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			gotMetrics[m.Name] = m
+		}
+	}
+
+	unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall")
+	targetAttr := attribute.String("grpc.target", ss.Target)
+	unaryStatusAttr := attribute.String("grpc.status", "OK")
+
+	serviceNameAttr := attribute.String("csm.service_name", "service_name_val")
+	serviceNamespaceAttr := attribute.String("csm.service_namespace_name", "service_namespace_val")
+	localityAttr := attribute.String("grpc.lb.locality", "grpc.lb.locality_val")
+	backendServiceAttr := attribute.String("grpc.lb.backend_service", "grpc.lb.backend_service_val")
+	meshIDAttr := attribute.String("csm.mesh_id", "unknown")
+	workloadCanonicalServiceAttr := attribute.String("csm.workload_canonical_service", "unknown")
+	remoteWorkloadTypeAttr := attribute.String("csm.remote_workload_type", "unknown")
+	remoteWorkloadCanonicalServiceAttr := attribute.String("csm.remote_workload_canonical_service", "unknown")
+	customLabelAttr := attribute.String("grpc.client.call.custom", "my-custom-label")
+
+	unaryMethodClientSideEnd := []attribute.KeyValue{
+		unaryMethodAttr,
+		targetAttr,
+		unaryStatusAttr,
+		serviceNameAttr,
+		serviceNamespaceAttr,
+		localityAttr,
+		backendServiceAttr,
+		meshIDAttr,
+		workloadCanonicalServiceAttr,
+		remoteWorkloadTypeAttr,
+		remoteWorkloadCanonicalServiceAttr,
+		customLabelAttr,
+	}
+
+	unaryCompressedBytesSentRecv := int64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{
+		Payload: &testpb.Payload{
+			Body: make([]byte, 10000),
+		},
+	})) // Fixed 10000 bytes with gzip.
+	unaryBucketCounts := []uint64{0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}
+	unaryExtrema := metricdata.NewExtrema(unaryCompressedBytesSentRecv)
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.client.attempt.started",
+			Description: "Number of client call attempts started.",
+			Unit:        "{attempt}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, customLabelAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		}, // Doesn't have xDS Labels, CSM Labels start from header or trailer from server, whichever comes first, so this doesn't need it
+		{
+			Name:        "grpc.client.attempt.duration",
+			Description: "End-to-end time taken to complete a client call attempt.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:      1,
+						Bounds:     itestutils.DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.sent_total_compressed_message_size",
+			Description: "Compressed message bytes sent per client call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       itestutils.DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.rcvd_total_compressed_message_size",
+			Description: "Compressed message bytes received per call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       itestutils.DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.call.duration",
+			Description: "Time taken by gRPC to complete an RPC from application's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr, customLabelAttr),
+						Count:      1,
+						Bounds:     itestutils.DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+	}
+
+	gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+	itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+}
+
+// TestObservability tests that Observability global function compiles and runs
+// without error. The actual functionality of this function will be verified in
+// interop tests.
+func (s) TestObservability(*testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	cleanup := EnableObservability(ctx, opentelemetry.Options{})
+	cleanup()
+}
+
+type o11yPickerBuilder struct{}
+
+func (*o11yPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker {
+	return &o11yPicker{info: info}
+}
+
+type o11yPicker struct {
+	info base.PickerBuildInfo
+}
+
+func (p *o11yPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// This simulates the behavior of the cluster_impl picker.
+	// It runs after TagRPC has already registered the default telemetry
+	// callback.
+	istats.UpdateLabels(info.Ctx, map[string]string{
+		"csm.service_name":           "service_name_val",
+		"csm.service_namespace_name": "service_namespace_val",
+
+		"grpc.lb.locality":        "grpc.lb.locality_val",
+		"grpc.lb.backend_service": "grpc.lb.backend_service_val",
+	})
+
+	for sc := range p.info.ReadySCs {
+		return balancer.PickResult{SubConn: sc}, nil
+	}
+	return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+}

stats/opentelemetry/csm/pluginoption.go

@@ -0,0 +1,279 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package csm contains utilities for Google Cloud Service Mesh observability.
+package csm
+
+import (
+	"context"
+	"encoding/base64"
+	"net/url"
+	"os"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats/opentelemetry/internal"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	"go.opentelemetry.io/contrib/detectors/gcp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+var logger = grpclog.Component("csm-observability-plugin")
+
+// pluginOption emits CSM Labels from the environment and metadata exchange
+// for csm channels and all servers.
+//
+// Do not use this directly; use newPluginOption instead.
+type pluginOption struct {
+	// localLabels are the labels that identify the local environment a binary
+	// is run in, and will be emitted from the CSM Plugin Option.
+	localLabels map[string]string
+	// metadataExchangeLabelsEncoded are the metadata exchange labels to be sent
+	// as the value of metadata key "x-envoy-peer-metadata" in proto wire format
+	// and base 64 encoded. This gets sent out from all the servers running in
+	// this process and for csm channels.
+	metadataExchangeLabelsEncoded string
+}
+
+// newPluginOption returns a new pluginOption with local labels and metadata
+// exchange labels derived from the environment.
+func newPluginOption(ctx context.Context) internal.PluginOption {
+	localLabels, metadataExchangeLabelsEncoded := constructMetadataFromEnv(ctx)
+
+	return &pluginOption{
+		localLabels:                   localLabels,
+		metadataExchangeLabelsEncoded: metadataExchangeLabelsEncoded,
+	}
+}
+
+// NewLabelsMD returns a metadata.MD with the CSM labels as an encoded protobuf
+// Struct as the value of "x-envoy-peer-metadata".
+func (cpo *pluginOption) GetMetadata() metadata.MD {
+	return metadata.New(map[string]string{
+		metadataExchangeKey: cpo.metadataExchangeLabelsEncoded,
+	})
+}
+
+// GetLabels gets the CSM peer labels from the metadata provided. It returns
+// "unknown" for labels not found. Labels returned depend on the remote type.
+// Additionally, local labels determined at initialization time are appended to
+// labels returned, in addition to the optionalLabels provided.
+func (cpo *pluginOption) GetLabels(md metadata.MD) map[string]string {
+	labels := map[string]string{ // Remote labels if type is unknown (i.e. unset or error processing x-envoy-peer-metadata)
+		"csm.remote_workload_type":              "unknown",
+		"csm.remote_workload_canonical_service": "unknown",
+	}
+	// Append the local labels.
+	for k, v := range cpo.localLabels {
+		labels[k] = v
+	}
+
+	val := md.Get("x-envoy-peer-metadata")
+	// This can't happen if corresponding csm client because of proto wire
+	// format encoding, but since it is arbitrary off the wire be safe.
+	if len(val) != 1 {
+		logger.Warningf("length of md values of \"x-envoy-peer-metadata\" is not 1, is %v", len(val))
+		return labels
+	}
+
+	protoWireFormat, err := base64.RawStdEncoding.DecodeString(val[0])
+	if err != nil {
+		logger.Warningf("error base 64 decoding value of \"x-envoy-peer-metadata\": %v", err)
+		return labels
+	}
+
+	spb := &structpb.Struct{}
+	if err := proto.Unmarshal(protoWireFormat, spb); err != nil {
+		logger.Warningf("error unmarshalling value of \"x-envoy-peer-metadata\" into proto: %v", err)
+		return labels
+	}
+
+	fields := spb.GetFields()
+
+	labels["csm.remote_workload_type"] = getFromMetadata("type", fields)
+	// The value of “csm.remote_workload_canonical_service” comes from
+	// MetadataExchange with the key “canonical_service”. (Note that this should
+	// be read even if the remote type is unknown.)
+	labels["csm.remote_workload_canonical_service"] = getFromMetadata("canonical_service", fields)
+
+	// Unset/unknown types, and types that aren't GKE or GCP return early with
+	// just local labels, remote_workload_type and
+	// remote_workload_canonical_service labels.
+	workloadType := labels["csm.remote_workload_type"]
+	if workloadType != "gcp_kubernetes_engine" && workloadType != "gcp_compute_engine" {
+		return labels
+	}
+	// GKE and GCE labels.
+	labels["csm.remote_workload_project_id"] = getFromMetadata("project_id", fields)
+	labels["csm.remote_workload_location"] = getFromMetadata("location", fields)
+	labels["csm.remote_workload_name"] = getFromMetadata("workload_name", fields)
+	if workloadType == "gcp_compute_engine" {
+		return labels
+	}
+
+	// GKE only labels.
+	labels["csm.remote_workload_cluster_name"] = getFromMetadata("cluster_name", fields)
+	labels["csm.remote_workload_namespace_name"] = getFromMetadata("namespace_name", fields)
+	return labels
+}
+
+// getFromMetadata gets the value for the metadata key from the protobuf
+// metadata. Returns "unknown" if the metadata is not found in the protobuf
+// metadata, or if the value is not a string value. Returns the string value
+// otherwise.
+func getFromMetadata(metadataKey string, metadata map[string]*structpb.Value) string {
+	if metadata != nil {
+		if metadataVal, ok := metadata[metadataKey]; ok {
+			if _, ok := metadataVal.GetKind().(*structpb.Value_StringValue); ok {
+				return metadataVal.GetStringValue()
+			}
+		}
+	}
+	return "unknown"
+}
+
+// getFromResource gets the value for the resource key from the attribute set.
+// Returns "unknown" if the resourceKey is not found in the attribute set or is
+// not a string value, the string value otherwise.
+func getFromResource(resourceKey attribute.Key, set *attribute.Set) string {
+	if set != nil {
+		if resourceVal, ok := set.Value(resourceKey); ok && resourceVal.Type() == attribute.STRING {
+			return resourceVal.AsString()
+		}
+	}
+	return "unknown"
+}
+
+// getEnv returns "unknown" if environment variable is unset, the environment
+// variable otherwise.
+func getEnv(name string) string {
+	if val, ok := os.LookupEnv(name); ok {
+		return val
+	}
+	return "unknown"
+}
+
+var (
+	// This function will be overridden in unit tests.
+	getAttrSetFromResourceDetector = func(ctx context.Context) *attribute.Set {
+		r, err := resource.New(ctx, resource.WithFromEnv(), resource.WithDetectors(gcp.NewDetector()))
+		if err != nil {
+			logger.Warningf("error reading OpenTelemetry resource: %v", err)
+		}
+		if r != nil {
+			// It's possible for resource.New to return partial data alongside
+			// an error. In this case, use partial data and set "unknown" for
+			// labels missing.
+			return r.Set()
+		}
+		return nil
+	}
+)
+
+// constructMetadataFromEnv creates local labels and labels to send to the peer
+// using metadata exchange based off resource detection and environment
+// variables.
+//
+// Returns local labels, and base 64 encoded protobuf.Struct containing metadata
+// exchange labels.
+func constructMetadataFromEnv(ctx context.Context) (map[string]string, string) {
+	set := getAttrSetFromResourceDetector(ctx)
+
+	labels := make(map[string]string)
+	labels["type"] = getFromResource("cloud.platform", set)
+	labels["canonical_service"] = getEnv("CSM_CANONICAL_SERVICE_NAME")
+
+	// If type is not GCE or GKE only metadata exchange labels are "type" and
+	// "canonical_service".
+	cloudPlatformVal := labels["type"]
+	if cloudPlatformVal != "gcp_kubernetes_engine" && cloudPlatformVal != "gcp_compute_engine" {
+		return initializeLocalAndMetadataLabels(labels)
+	}
+
+	// GCE and GKE labels:
+	labels["workload_name"] = getEnv("CSM_WORKLOAD_NAME")
+
+	locationVal := "unknown"
+	if resourceVal, ok := set.Value("cloud.availability_zone"); ok && resourceVal.Type() == attribute.STRING {
+		locationVal = resourceVal.AsString()
+	} else if resourceVal, ok = set.Value("cloud.region"); ok && resourceVal.Type() == attribute.STRING {
+		locationVal = resourceVal.AsString()
+	}
+	labels["location"] = locationVal
+
+	labels["project_id"] = getFromResource("cloud.account.id", set)
+	if cloudPlatformVal == "gcp_compute_engine" {
+		return initializeLocalAndMetadataLabels(labels)
+	}
+
+	// GKE specific labels:
+	labels["namespace_name"] = getFromResource("k8s.namespace.name", set)
+	labels["cluster_name"] = getFromResource("k8s.cluster.name", set)
+	return initializeLocalAndMetadataLabels(labels)
+}
+
+// initializeLocalAndMetadataLabels csm local labels for a CSM Plugin Option to
+// record. It also builds out a base 64 encoded protobuf.Struct containing the
+// metadata exchange labels to be sent as part of metadata exchange from a CSM
+// Plugin Option.
+func initializeLocalAndMetadataLabels(labels map[string]string) (map[string]string, string) {
+	// The value of “csm.workload_canonical_service” comes from
+	// “CSM_CANONICAL_SERVICE_NAME” env var, “unknown” if unset.
+	val := labels["canonical_service"]
+	localLabels := make(map[string]string)
+	localLabels["csm.workload_canonical_service"] = val
+	localLabels["csm.mesh_id"] = getEnv("CSM_MESH_ID")
+
+	// Metadata exchange labels - can go ahead and encode into proto, and then
+	// base64.
+	pbLabels := &structpb.Struct{
+		Fields: map[string]*structpb.Value{},
+	}
+	for k, v := range labels {
+		pbLabels.Fields[k] = structpb.NewStringValue(v)
+	}
+	protoWireFormat, err := proto.Marshal(pbLabels)
+	metadataExchangeLabelsEncoded := ""
+	if err == nil {
+		metadataExchangeLabelsEncoded = base64.RawStdEncoding.EncodeToString(protoWireFormat)
+	}
+	// else - This behavior triggers server side to reply (if sent from a gRPC
+	// Client within this binary) with the metadata exchange labels. Even if
+	// client side has a problem marshaling proto into wire format, it can
+	// still use server labels so send an empty string as the value of
+	// x-envoy-peer-metadata. The presence of this metadata exchange header
+	// will cause server side to respond with metadata exchange labels.
+
+	return localLabels, metadataExchangeLabelsEncoded
+}
+
+// metadataExchangeKey is the key for HTTP metadata exchange.
+const metadataExchangeKey = "x-envoy-peer-metadata"
+
+func determineTargetCSM(parsedTarget *url.URL) bool {
+	// On the client-side, the channel target is used to determine if a channel is a
+	// CSM channel or not. CSM channels need to have an “xds” scheme and a
+	// "traffic-director-global.xds.googleapis.com" authority. In the cases where no
+	// authority is mentioned, the authority is assumed to be CSM. MetadataExchange
+	// is performed only for CSM channels. Non-metadata exchange labels are detected
+	// as described below.
+	return parsedTarget.Scheme == "xds" && (parsedTarget.Host == "" || parsedTarget.Host == "traffic-director-global.xds.googleapis.com")
+}

stats/opentelemetry/csm/pluginoption_test.go

@@ -0,0 +1,490 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/google/go-cmp/cmp"
+	"go.opentelemetry.io/otel/attribute"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var defaultTestTimeout = 5 * time.Second
+
+// clearEnv unsets all the environment variables relevant to the csm
+// pluginOption.
+func clearEnv() {
+	os.Unsetenv(envconfig.XDSBootstrapFileContentEnv)
+	os.Unsetenv(envconfig.XDSBootstrapFileNameEnv)
+
+	os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+	os.Unsetenv("CSM_WORKLOAD_NAME")
+}
+
+func (s) TestGetLabels(t *testing.T) {
+	clearEnv()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cpo := newPluginOption(ctx)
+
+	tests := []struct {
+		name                   string
+		unsetHeader            bool // Should trigger "unknown" labels
+		twoValues              bool // Should trigger "unknown" labels
+		metadataExchangeLabels map[string]string
+		labelsWant             map[string]string
+	}{
+		{
+			name:                   "unset-labels",
+			metadataExchangeLabels: nil,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "metadata-partially-set",
+			metadataExchangeLabels: map[string]string{
+				"type":        "not-gce-or-gke",
+				"ignore-this": "ignore-this",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "not-gce-or-gke",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "google-compute-engine",
+			metadataExchangeLabels: map[string]string{ // All of these labels get emitted when type is "gcp_compute_engine".
+				"type":              "gcp_compute_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_compute_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "workload_name_val",
+			},
+		},
+		// unset should go to unknown, ignore GKE labels that are not relevant
+		// to GCE.
+		{
+			name: "google-compute-engine-labels-partially-set-with-extra",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"location":          "us-east",
+				// "workload_name": "", unset workload name - should become "unknown"
+				"namespace_name": "should-be-ignored",
+				"cluster_name":   "should-be-ignored",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_compute_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "unknown",
+			},
+		},
+		{
+			name: "google-kubernetes-engine",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_kubernetes_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_cluster_name":      "cluster_name_val",
+				"csm.remote_workload_namespace_name":    "namespace_name_val",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "workload_name_val",
+			},
+		},
+		{
+			name: "google-kubernetes-engine-labels-partially-set",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				// "cluster_name": "", cluster_name unset, should become "unknown"
+				"location": "us-east",
+				// "workload_name": "", workload_name unset, should become "unknown"
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_kubernetes_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_cluster_name":      "unknown",
+				"csm.remote_workload_namespace_name":    "namespace_name_val",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "unknown",
+			},
+		},
+		{
+			name: "unset-header",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			unsetHeader: true,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "two-header-values",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			twoValues: true,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pbLabels := &structpb.Struct{
+				Fields: map[string]*structpb.Value{},
+			}
+			for k, v := range test.metadataExchangeLabels {
+				pbLabels.Fields[k] = structpb.NewStringValue(v)
+			}
+			protoWireFormat, err := proto.Marshal(pbLabels)
+			if err != nil {
+				t.Fatalf("Error marshaling proto: %v", err)
+			}
+			metadataExchangeLabelsEncoded := base64.RawStdEncoding.EncodeToString(protoWireFormat)
+			md := metadata.New(map[string]string{
+				metadataExchangeKey: metadataExchangeLabelsEncoded,
+			})
+			if test.unsetHeader {
+				md.Delete(metadataExchangeKey)
+			}
+			if test.twoValues {
+				md.Append(metadataExchangeKey, "extra-val")
+			}
+
+			labelsGot := cpo.GetLabels(md)
+			if diff := cmp.Diff(labelsGot, test.labelsWant); diff != "" {
+				t.Fatalf("cpo.GetLabels returned unexpected value (-got, +want): %v", diff)
+			}
+		})
+	}
+}
+
+// TestDetermineTargetCSM tests the helper function that determines whether a
+// target is relevant to CSM or not, based off the rules outlined in design.
+func (s) TestDetermineTargetCSM(t *testing.T) {
+	tests := []struct {
+		name      string
+		target    string
+		targetCSM bool
+	}{
+		{
+			name:      "dns:///localhost",
+			target:    "normal-target-here",
+			targetCSM: false,
+		},
+		{
+			name:      "xds-no-authority",
+			target:    "xds:///localhost",
+			targetCSM: true,
+		},
+		{
+			name:      "xds-traffic-director-authority",
+			target:    "xds://traffic-director-global.xds.googleapis.com/localhost",
+			targetCSM: true,
+		},
+		{
+			name:      "xds-not-traffic-director-authority",
+			target:    "xds://not-traffic-director-authority/localhost",
+			targetCSM: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			parsedTarget, err := url.Parse(test.target)
+			if err != nil {
+				t.Fatalf("test target %v failed to parse: %v", test.target, err)
+			}
+			if got := determineTargetCSM(parsedTarget); got != test.targetCSM {
+				t.Fatalf("cpo.determineTargetCSM(%v): got %v, want %v", test.target, got, test.targetCSM)
+			}
+		})
+	}
+}
+
+// TestSetLabels tests the setting of labels, which snapshots the resource and
+// environment. It mocks the resource and environment, and then calls into
+// labels creation. It verifies to local labels created and metadata exchange
+// labels emitted from the setLabels function.
+func (s) TestSetLabels(t *testing.T) {
+	clearEnv()
+	tests := []struct {
+		name                             string
+		resourceKeyValues                map[string]string
+		csmCanonicalServiceNamePopulated bool
+		csmWorkloadNamePopulated         bool
+		meshIDPopulated                  bool
+		localLabelsWant                  map[string]string
+		metadataExchangeLabelsWant       map[string]string
+	}{
+		{
+			name:                             "no-type",
+			csmCanonicalServiceNamePopulated: true,
+			meshIDPopulated:                  true,
+			resourceKeyValues:                map[string]string{},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "canonical_service_name_val", // env var populated so should be set.
+				"csm.mesh_id":                    "mesh_id",                    // env var populated so should be set.
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "unknown",
+				"canonical_service": "canonical_service_name_val", // env var populated so should be set.
+			},
+		},
+		{
+			name:                     "gce",
+			csmWorkloadNamePopulated: true,
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_compute_engine",
+				// csm workload name is an env var
+				"cloud.availability_zone": "cloud_availability_zone_val",
+				"cloud.region":            "should-be-ignored", // cloud.availability_zone takes precedence
+				"cloud.account.id":        "cloud_account_id_val",
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "workload_name_val",
+				"location":          "cloud_availability_zone_val",
+				"project_id":        "cloud_account_id_val",
+			},
+		},
+		{
+			name: "gce-half-unset",
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_compute_engine",
+				// csm workload name is an env var
+				"cloud.availability_zone": "cloud_availability_zone_val",
+				"cloud.region":            "should-be-ignored", // cloud.availability_zone takes precedence
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_availability_zone_val",
+				"project_id":        "unknown",
+			},
+		},
+		{
+			name: "gke",
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_kubernetes_engine",
+				// csm workload name is an env var
+				"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+				"cloud.account.id":   "cloud_account_id_val",
+				"k8s.namespace.name": "k8s_namespace_name_val",
+				"k8s.cluster.name":   "k8s_cluster_name_val",
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_region_val",
+				"project_id":        "cloud_account_id_val",
+				"namespace_name":    "k8s_namespace_name_val",
+				"cluster_name":      "k8s_cluster_name_val",
+			},
+		},
+		{
+			name: "gke-half-unset",
+			resourceKeyValues: map[string]string{ // unset should become unknown
+				"cloud.platform": "gcp_kubernetes_engine",
+				// csm workload name is an env var
+				"cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location
+				// "cloud.account.id": "", // unset - should become unknown
+				"k8s.namespace.name": "k8s_namespace_name_val",
+				// "k8s.cluster.name": "", // unset - should become unknown
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_region_val",
+				"project_id":        "unknown",
+				"namespace_name":    "k8s_namespace_name_val",
+				"cluster_name":      "unknown",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			func() {
+				if test.csmCanonicalServiceNamePopulated {
+					os.Setenv("CSM_CANONICAL_SERVICE_NAME", "canonical_service_name_val")
+					defer os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+				}
+				if test.csmWorkloadNamePopulated {
+					os.Setenv("CSM_WORKLOAD_NAME", "workload_name_val")
+					defer os.Unsetenv("CSM_WORKLOAD_NAME")
+				}
+				if test.meshIDPopulated {
+					os.Setenv("CSM_MESH_ID", "mesh_id")
+					defer os.Unsetenv("CSM_MESH_ID")
+				}
+				var attributes []attribute.KeyValue
+				for k, v := range test.resourceKeyValues {
+					attributes = append(attributes, attribute.String(k, v))
+				}
+				// Return the attributes configured as part of the test in place
+				// of reading from resource.
+				attrSet := attribute.NewSet(attributes...)
+				origGetAttrSet := getAttrSetFromResourceDetector
+				getAttrSetFromResourceDetector = func(context.Context) *attribute.Set {
+					return &attrSet
+				}
+				defer func() { getAttrSetFromResourceDetector = origGetAttrSet }()
+
+				ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+				defer cancel()
+				localLabelsGot, mdEncoded := constructMetadataFromEnv(ctx)
+				if diff := cmp.Diff(localLabelsGot, test.localLabelsWant); diff != "" {
+					t.Fatalf("constructMetadataFromEnv() want: %v, got %v", test.localLabelsWant, localLabelsGot)
+				}
+
+				verifyMetadataExchangeLabels(mdEncoded, test.metadataExchangeLabelsWant)
+			}()
+		})
+	}
+}
+
+func verifyMetadataExchangeLabels(mdEncoded string, mdLabelsWant map[string]string) error {
+	protoWireFormat, err := base64.RawStdEncoding.DecodeString(mdEncoded)
+	if err != nil {
+		return fmt.Errorf("error base 64 decoding metadata val: %v", err)
+	}
+	spb := &structpb.Struct{}
+	if err := proto.Unmarshal(protoWireFormat, spb); err != nil {
+		return fmt.Errorf("error unmarshaling proto wire format: %v", err)
+	}
+	fields := spb.GetFields()
+	for k, v := range mdLabelsWant {
+		if val, ok := fields[k]; !ok {
+			if _, ok := val.GetKind().(*structpb.Value_StringValue); !ok {
+				return fmt.Errorf("struct value for key %v should be string type", k)
+			}
+			if val.GetStringValue() != v {
+				return fmt.Errorf("struct value for key %v got: %v, want %v", k, val.GetStringValue(), v)
+			}
+		}
+	}
+	if len(mdLabelsWant) != len(fields) {
+		return fmt.Errorf("len(mdLabelsWant) = %v, len(mdLabelsGot) = %v", len(mdLabelsWant), len(fields))
+	}
+	return nil
+}

stats/opentelemetry/example_test.go

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry_test
+
+import (
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/stats/opentelemetry"
+
+	"go.opentelemetry.io/otel/sdk/metric"
+)
+
+func ExampleDialOption_basic() {
+	// This is setting default bounds for a view. Setting these bounds through
+	// meter provider from SDK is recommended, as API calls in this module
+	// provide default bounds, but these calls are not guaranteed to be stable
+	// and API implementors are not required to implement bounds. Setting bounds
+	// through SDK ensures the bounds get picked up. The specific fields in
+	// Aggregation take precedence over defaults from API. For any fields unset
+	// in aggregation, defaults get picked up, so can have a mix of fields from
+	// SDK and fields created from API call. The overridden views themselves
+	// also follow same logic, only the specific views being created in the SDK
+	// use SDK information, the rest are created from API call.
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(
+		metric.WithReader(reader),
+		metric.WithView(metric.NewView(metric.Instrument{
+			Name: "grpc.client.call.duration",
+		},
+			metric.Stream{
+				Aggregation: metric.AggregationExplicitBucketHistogram{
+					Boundaries: opentelemetry.DefaultSizeBounds, // The specific fields set in SDK take precedence over API.
+				},
+			},
+		)),
+	)
+
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider: provider,
+			Metrics:       opentelemetry.DefaultMetrics(), // equivalent to unset - distinct from empty
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle err.
+	}
+	defer cc.Close()
+}
+
+func ExampleServerOption_methodFilter() {
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider: provider,
+			// Because Metrics is unset, the user will get default metrics.
+			MethodAttributeFilter: func(str string) bool {
+				// Will allow duplex/any other type of RPC.
+				return str != "/grpc.testing.TestService/UnaryCall"
+			},
+		},
+	}
+	cc, err := grpc.NewClient("some-target", opentelemetry.DialOption(opts), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle err.
+	}
+	defer cc.Close()
+}
+
+func ExampleMetrics_excludeSome() {
+	// To exclude specific metrics, initialize Options as follows:
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			Metrics: opentelemetry.DefaultMetrics().Remove(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName),
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle err.
+	}
+	defer cc.Close()
+}
+
+func ExampleMetrics_disableAll() {
+	// To disable all metrics, initialize Options as follows:
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			Metrics: stats.NewMetricSet(), // Distinct to nil, which creates default metrics. This empty set creates no metrics.
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle err.
+	}
+	defer cc.Close()
+}
+
+func ExampleMetrics_enableSome() {
+	// To only create specific metrics, initialize Options as follows:
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			Metrics: stats.NewMetricSet(opentelemetry.ClientAttemptDurationMetricName, opentelemetry.ClientAttemptRcvdCompressedTotalMessageSizeMetricName), // only create these metrics
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil { // might fail vet
+		// Handle err.
+	}
+	defer cc.Close()
+}
+
+func ExampleOptions_addExperimentalMetrics() {
+	opts := opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			// These are example experimental gRPC metrics, which are disabled
+			// by default and must be explicitly enabled. For the full,
+			// up-to-date list of metrics, see:
+			// https://grpc.io/docs/guides/opentelemetry-metrics/#instruments
+			Metrics: opentelemetry.DefaultMetrics().Add(
+				"grpc.subchannel.connection_attempts_succeeded",
+				"grpc.subchannel.connection_attempts_failed",
+			),
+		},
+	}
+	do := opentelemetry.DialOption(opts)
+	cc, err := grpc.NewClient("<target string>", do, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		// Handle error.
+	}
+	defer cc.Close()
+}

stats/opentelemetry/grpc_trace_bin_propagator.go

@@ -0,0 +1,119 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package opentelemetry
+
+import (
+	"context"
+
+	otelpropagation "go.opentelemetry.io/otel/propagation"
+	oteltrace "go.opentelemetry.io/otel/trace"
+)
+
+// gRPCTraceBinHeaderKey is the gRPC metadata header key `grpc-trace-bin` used
+// to propagate trace context in binary format.
+const grpcTraceBinHeaderKey = "grpc-trace-bin"
+
+// GRPCTraceBinPropagator is an OpenTelemetry TextMapPropagator which is used
+// to extract and inject trace context data from and into headers exchanged by
+// gRPC applications. It propagates trace data in binary format using the
+// `grpc-trace-bin` header.
+type GRPCTraceBinPropagator struct{}
+
+// Inject sets OpenTelemetry span context from the Context into the carrier as
+// a `grpc-trace-bin` header if span context is valid.
+//
+// If span context is not valid, it returns without setting `grpc-trace-bin`
+// header.
+func (GRPCTraceBinPropagator) Inject(ctx context.Context, carrier otelpropagation.TextMapCarrier) {
+	sc := oteltrace.SpanFromContext(ctx)
+	if !sc.SpanContext().IsValid() {
+		return
+	}
+
+	bd := toBinary(sc.SpanContext())
+	carrier.Set(grpcTraceBinHeaderKey, string(bd))
+}
+
+// Extract reads OpenTelemetry span context from the `grpc-trace-bin` header of
+// carrier into the provided context, if present.
+//
+// If a valid span context is retrieved from `grpc-trace-bin`, it returns a new
+// context containing the extracted OpenTelemetry span context marked as
+// remote.
+//
+// If `grpc-trace-bin` header is not present, it returns the context as is.
+func (GRPCTraceBinPropagator) Extract(ctx context.Context, carrier otelpropagation.TextMapCarrier) context.Context {
+	h := carrier.Get(grpcTraceBinHeaderKey)
+	if h == "" {
+		return ctx
+	}
+
+	sc, ok := fromBinary([]byte(h))
+	if !ok {
+		return ctx
+	}
+	return oteltrace.ContextWithRemoteSpanContext(ctx, sc)
+}
+
+// Fields returns the keys whose values are set with Inject.
+//
+// GRPCTraceBinPropagator always returns a slice containing only
+// `grpc-trace-bin` key because it only sets the `grpc-trace-bin` header for
+// propagating trace context.
+func (GRPCTraceBinPropagator) Fields() []string {
+	return []string{grpcTraceBinHeaderKey}
+}
+
+// toBinary returns the binary format representation of a SpanContext.
+//
+// If sc is the zero value, returns nil.
+func toBinary(sc oteltrace.SpanContext) []byte {
+	if sc.Equal(oteltrace.SpanContext{}) {
+		return nil
+	}
+	var b [29]byte
+	traceID := oteltrace.TraceID(sc.TraceID())
+	copy(b[2:18], traceID[:])
+	b[18] = 1
+	spanID := oteltrace.SpanID(sc.SpanID())
+	copy(b[19:27], spanID[:])
+	b[27] = 2
+	b[28] = byte(oteltrace.TraceFlags(sc.TraceFlags()))
+	return b[:]
+}
+
+// fromBinary returns the SpanContext represented by b with Remote set to true.
+//
+// It returns with zero value SpanContext and false, if any of the
+// below condition is not satisfied:
+// - Valid header: len(b) = 29
+// - Valid version: b[0] = 0
+// - Valid traceID prefixed with 0: b[1] = 0
+// - Valid spanID prefixed with 1: b[18] = 1
+// - Valid traceFlags prefixed with 2: b[27] = 2
+func fromBinary(b []byte) (oteltrace.SpanContext, bool) {
+	if len(b) != 29 || b[0] != 0 || b[1] != 0 || b[18] != 1 || b[27] != 2 {
+		return oteltrace.SpanContext{}, false
+	}
+
+	return oteltrace.SpanContext{}.WithTraceID(
+		oteltrace.TraceID(b[2:18])).WithSpanID(
+		oteltrace.SpanID(b[19:27])).WithTraceFlags(
+		oteltrace.TraceFlags(b[28])).WithRemote(true), true
+}

stats/opentelemetry/grpc_trace_bin_propagator_test.go

@@ -0,0 +1,219 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	oteltrace "go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/metadata"
+	itracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing"
+)
+
+var validSpanContext = oteltrace.SpanContext{}.WithTraceID(
+	oteltrace.TraceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}).WithSpanID(
+	oteltrace.SpanID{17, 18, 19, 20, 21, 22, 23, 24}).WithTraceFlags(
+	oteltrace.TraceFlags(1))
+
+// TestInject_ValidSpanContext verifies that the GRPCTraceBinPropagator
+// correctly injects a valid OpenTelemetry span context as `grpc-trace-bin`
+// header in the provided carrier's context metadata.
+//
+// It verifies that if a valid span context is injected, same span context can
+// can be retreived from the carrier's context metadata.
+func (s) TestInject_ValidSpanContext(t *testing.T) {
+	p := GRPCTraceBinPropagator{}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	c := itracing.NewOutgoingCarrier(ctx)
+	ctx = oteltrace.ContextWithSpanContext(ctx, validSpanContext)
+
+	p.Inject(ctx, c)
+
+	md, _ := metadata.FromOutgoingContext(c.Context())
+	gotH := md.Get(grpcTraceBinHeaderKey)
+	if gotH[len(gotH)-1] == "" {
+		t.Fatalf("got empty value from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", validSpanContext)
+	}
+	gotSC, ok := fromBinary([]byte(gotH[len(gotH)-1]))
+	if !ok {
+		t.Fatalf("got invalid span context %v from Carrier's context metadata grpc-trace-bin header, want valid span context: %v", gotSC, validSpanContext)
+	}
+	if cmp.Equal(validSpanContext, gotSC) {
+		t.Fatalf("got span context = %v, want span contexts %v", gotSC, validSpanContext)
+	}
+}
+
+// TestInject_InvalidSpanContext verifies that the GRPCTraceBinPropagator does
+// not inject an invalid OpenTelemetry span context as `grpc-trace-bin` header
+// in the provided carrier's context metadata.
+//
+// If an invalid span context is injected, it verifies that `grpc-trace-bin`
+// header is not set in the carrier's context metadata.
+func (s) TestInject_InvalidSpanContext(t *testing.T) {
+	p := GRPCTraceBinPropagator{}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	c := itracing.NewOutgoingCarrier(ctx)
+	ctx = oteltrace.ContextWithSpanContext(ctx, oteltrace.SpanContext{})
+
+	p.Inject(ctx, c)
+
+	md, _ := metadata.FromOutgoingContext(c.Context())
+	if gotH := md.Get(grpcTraceBinHeaderKey); len(gotH) > 0 {
+		t.Fatalf("got %v value from Carrier's context metadata grpc-trace-bin header, want empty", gotH)
+	}
+}
+
+// TestExtract verifies that the GRPCTraceBinPropagator correctly extracts
+// OpenTelemetry span context data from the provided context using carrier.
+//
+// If a valid span context was injected, it verifies same trace span context
+// is extracted from carrier's metadata for `grpc-trace-bin` header key.
+//
+// If invalid span context was injected, it verifies that valid trace span
+// context is not extracted.
+func (s) TestExtract(t *testing.T) {
+	tests := []struct {
+		name   string
+		wantSC oteltrace.SpanContext // expected span context from carrier
+	}{
+		{
+			name:   "valid OpenTelemetry span context",
+			wantSC: validSpanContext.WithRemote(true),
+		},
+		{
+			name:   "invalid OpenTelemetry span context",
+			wantSC: oteltrace.SpanContext{},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			p := GRPCTraceBinPropagator{}
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			ctx = metadata.NewIncomingContext(ctx, metadata.MD{grpcTraceBinHeaderKey: []string{string(toBinary(test.wantSC))}})
+
+			c := itracing.NewIncomingCarrier(ctx)
+
+			tCtx := p.Extract(ctx, c)
+			got := oteltrace.SpanContextFromContext(tCtx)
+			if !got.Equal(test.wantSC) {
+				t.Fatalf("got span context: %v, want span context: %v", got, test.wantSC)
+			}
+		})
+	}
+}
+
+// TestBinary verifies that the toBinary() function correctly serializes a valid
+// OpenTelemetry span context into its binary format representation. If span
+// context is invalid, it verifies that serialization is nil.
+func (s) TestToBinary(t *testing.T) {
+	tests := []struct {
+		name string
+		sc   oteltrace.SpanContext
+		want []byte
+	}{
+		{
+			name: "valid context",
+			sc:   validSpanContext,
+			want: toBinary(validSpanContext),
+		},
+		{
+			name: "zero value context",
+			sc:   oteltrace.SpanContext{},
+			want: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := toBinary(test.sc); !cmp.Equal(got, test.want) {
+				t.Fatalf("binary() = %v, want %v", got, test.want)
+			}
+		})
+	}
+}
+
+// TestFromBinary verifies that the fromBinary() function correctly
+// deserializes a binary format representation of a valid OpenTelemetry span
+// context into its corresponding span context format. If span context's binary
+// representation is invalid, it verifies that deserialization is zero value
+// span context.
+func (s) TestFromBinary(t *testing.T) {
+	tests := []struct {
+		name string
+		b    []byte
+		want oteltrace.SpanContext
+		ok   bool
+	}{
+		{
+			name: "valid",
+			b:    []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1},
+			want: validSpanContext.WithRemote(true),
+			ok:   true,
+		},
+		{
+			name: "invalid length",
+			b:    []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2},
+			want: oteltrace.SpanContext{},
+			ok:   false,
+		},
+		{
+			name: "invalid version",
+			b:    []byte{1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1},
+			want: oteltrace.SpanContext{},
+			ok:   false,
+		},
+		{
+			name: "invalid traceID field ID",
+			b:    []byte{0, 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1},
+			want: oteltrace.SpanContext{},
+			ok:   false,
+		},
+		{
+			name: "invalid spanID field ID",
+			b:    []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, 17, 18, 19, 20, 21, 22, 23, 24, 2, 1},
+			want: oteltrace.SpanContext{},
+			ok:   false,
+		},
+		{
+			name: "invalid traceFlags field ID",
+			b:    []byte{0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 17, 18, 19, 20, 21, 22, 23, 24, 1, 1},
+			want: oteltrace.SpanContext{},
+			ok:   false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, ok := fromBinary(test.b)
+			if ok != test.ok {
+				t.Fatalf("fromBinary() ok = %v, want %v", ok, test.ok)
+				return
+			}
+			if !got.Equal(test.want) {
+				t.Fatalf("fromBinary() got = %v, want %v", got, test.want)
+			}
+		})
+	}
+}

stats/opentelemetry/internal/pluginoption.go

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package internal defines the PluginOption interface.
+package internal
+
+import (
+	"google.golang.org/grpc/metadata"
+)
+
+// SetPluginOption sets the plugin option on Options.
+var SetPluginOption any // func(*Options, PluginOption)
+
+// PluginOption is the interface which represents a plugin option for the
+// OpenTelemetry instrumentation component. This plugin option emits labels from
+// metadata and also creates metadata containing labels. These labels are
+// intended to be added to applicable OpenTelemetry metrics recorded in the
+// OpenTelemetry instrumentation component.
+//
+// In the future, we hope to stabilize and expose this API to allow plugins to
+// inject labels of their choosing into metrics recorded.
+type PluginOption interface {
+	// GetMetadata creates a MD with metadata exchange labels.
+	GetMetadata() metadata.MD
+	// GetLabels emits labels to be attached to metrics for the RPC that
+	// contains the provided incomingMetadata.
+	GetLabels(incomingMetadata metadata.MD) map[string]string
+}

stats/opentelemetry/internal/testutils/testutils.go

@@ -0,0 +1,853 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package testutils contains helpers for OpenTelemetry tests.
+package testutils
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"fmt"
+	"slices"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+	"google.golang.org/protobuf/proto"
+)
+
+// Redefine default bounds here to avoid a cyclic dependency with top level
+// opentelemetry package. Could define once through internal, but would make
+// external opentelemetry godoc less readable.
+var (
+	// DefaultLatencyBounds are the default bounds for latency metrics.
+	DefaultLatencyBounds = []float64{0, 0.00001, 0.00005, 0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.008, 0.01, 0.013, 0.016, 0.02, 0.025, 0.03, 0.04, 0.05, 0.065, 0.08, 0.1, 0.13, 0.16, 0.2, 0.25, 0.3, 0.4, 0.5, 0.65, 0.8, 1, 2, 5, 10, 20, 50, 100}
+	// DefaultSizeBounds are the default bounds for metrics which record size.
+	DefaultSizeBounds = []float64{0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296}
+)
+
+// waitForServerCompletedRPCs waits until the unary and streaming stats.End
+// calls are finished processing. It does this by waiting for the expected
+// metric triggered by stats.End to appear through the passed in metrics reader.
+//
+// Returns a new gotMetrics map containing the metric data being polled for, or
+// an error if failed to wait for metric.
+func waitForServerCompletedRPCs(ctx context.Context, reader metric.Reader, wantMetric metricdata.Metrics) (map[string]metricdata.Metrics, error) {
+	for ; ctx.Err() == nil; <-time.After(time.Millisecond) {
+		rm := &metricdata.ResourceMetrics{}
+		reader.Collect(ctx, rm)
+		gotMetrics := map[string]metricdata.Metrics{}
+		for _, sm := range rm.ScopeMetrics {
+			for _, m := range sm.Metrics {
+				gotMetrics[m.Name] = m
+			}
+		}
+		val, ok := gotMetrics[wantMetric.Name]
+		if !ok {
+			continue
+		}
+		switch data := val.Data.(type) {
+		case metricdata.Histogram[int64]:
+			if len(wantMetric.Data.(metricdata.Histogram[int64]).DataPoints) > len(data.DataPoints) {
+				continue
+			}
+		case metricdata.Histogram[float64]:
+			if len(wantMetric.Data.(metricdata.Histogram[float64]).DataPoints) > len(data.DataPoints) {
+				continue
+			}
+		}
+		return gotMetrics, nil
+	}
+	return nil, fmt.Errorf("error waiting for metric %v: %v", wantMetric, ctx.Err())
+}
+
+// checkDataPointWithinFiveSeconds checks if the metric passed in contains a
+// histogram with dataPoints that fall within buckets that are <=5. Returns an
+// error if check fails.
+func checkDataPointWithinFiveSeconds(metric metricdata.Metrics) error {
+	histo, ok := metric.Data.(metricdata.Histogram[float64])
+	if !ok {
+		return fmt.Errorf("metric data is not histogram")
+	}
+	for _, dataPoint := range histo.DataPoints {
+		var boundWithFive int
+		for i, bound := range dataPoint.Bounds {
+			if bound >= 5 {
+				boundWithFive = i
+			}
+		}
+		foundPoint := false
+		for i, count := range dataPoint.BucketCounts {
+			if i >= boundWithFive {
+				return fmt.Errorf("data point not found in bucket <=5 seconds")
+			}
+			if count == 1 {
+				foundPoint = true
+				break
+			}
+		}
+		if !foundPoint {
+			return fmt.Errorf("no data point found for metric")
+		}
+	}
+	return nil
+}
+
+// MetricDataOptions are the options used to configure the metricData emissions
+// of expected metrics data from NewMetricData.
+type MetricDataOptions struct {
+	// CSMLabels are the csm labels to attach to metrics which receive csm
+	// labels (all A66 expect client call and started RPC's client and server
+	// side).
+	CSMLabels []attribute.KeyValue
+	// Target is the target of the client and server.
+	Target string
+	// UnaryCallFailed is whether the Unary Call failed, which would trigger
+	// trailers only.
+	UnaryCallFailed bool
+	// UnaryCompressedMessageSize is the compressed message size of the Unary
+	// RPC. This assumes both client and server sent the same message size.
+	UnaryCompressedMessageSize float64
+	// StreamingCompressedMessageSize is the compressed message size of the
+	// Streaming RPC. This assumes both client and server sent the same message
+	// size.
+	StreamingCompressedMessageSize float64
+}
+
+// createBucketCounts creates a list of bucket counts based off the
+// recordingPoints and bounds. Both recordingPoints and bounds are assumed to be
+// in order.
+func createBucketCounts(recordingPoints []float64, bounds []float64) []uint64 {
+	var bucketCounts []uint64
+	var recordingPointIndex int
+	for _, bound := range bounds {
+		var bucketCount uint64
+		if recordingPointIndex >= len(recordingPoints) {
+			bucketCounts = append(bucketCounts, bucketCount)
+			continue
+		}
+		for recordingPoints[recordingPointIndex] <= bound {
+			bucketCount++
+			recordingPointIndex++
+			if recordingPointIndex >= len(recordingPoints) {
+				break
+			}
+		}
+		bucketCounts = append(bucketCounts, bucketCount)
+	}
+	// The rest of the recording points are last bound -> infinity.
+	bucketCounts = append(bucketCounts, uint64(len(recordingPoints)-recordingPointIndex))
+	return bucketCounts
+}
+
+// MetricDataUnary returns a list of expected metrics defined in A66 for a
+// client and server for one unary RPC.
+func MetricDataUnary(options MetricDataOptions) []metricdata.Metrics {
+	methodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall")
+	targetAttr := attribute.String("grpc.target", options.Target)
+	statusAttr := attribute.String("grpc.status", "OK")
+	if options.UnaryCallFailed {
+		statusAttr = attribute.String("grpc.status", "UNKNOWN")
+	}
+	clientSideEnd := []attribute.KeyValue{
+		methodAttr,
+		targetAttr,
+		statusAttr,
+	}
+	serverSideEnd := []attribute.KeyValue{
+		methodAttr,
+		statusAttr,
+	}
+	clientSideEnd = append(clientSideEnd, options.CSMLabels...)
+	serverSideEnd = append(serverSideEnd, options.CSMLabels...)
+	compressedBytesSentRecv := int64(options.UnaryCompressedMessageSize)
+	bucketCounts := createBucketCounts([]float64{options.UnaryCompressedMessageSize}, DefaultSizeBounds)
+	extrema := metricdata.NewExtrema(int64(options.UnaryCompressedMessageSize))
+	return []metricdata.Metrics{
+		{
+			Name:        "grpc.client.attempt.started",
+			Description: "Number of client call attempts started.",
+			Unit:        "{attempt}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(methodAttr, targetAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.duration",
+			Description: "End-to-end time taken to complete a client call attempt.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(clientSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.sent_total_compressed_message_size",
+			Description: "Compressed message bytes sent per client call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(clientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.rcvd_total_compressed_message_size",
+			Description: "Compressed message bytes received per call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(clientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.call.duration",
+			Description: "Time taken by gRPC to complete an RPC from application's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(methodAttr, targetAttr, statusAttr),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.started",
+			Description: "Number of server calls started.",
+			Unit:        "{call}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(methodAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.server.call.sent_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes sent per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(serverSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.rcvd_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes received per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(serverSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.duration",
+			Description: "End-to-end time taken to complete a call from server transport's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(serverSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+	}
+}
+
+// MetricDataStreaming returns a list of expected metrics defined in A66 for a
+// client and server for one streaming RPC.
+func MetricDataStreaming(options MetricDataOptions) []metricdata.Metrics {
+	methodAttr := attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall")
+	targetAttr := attribute.String("grpc.target", options.Target)
+	statusAttr := attribute.String("grpc.status", "OK")
+	clientSideEnd := []attribute.KeyValue{
+		methodAttr,
+		targetAttr,
+		statusAttr,
+	}
+	serverSideEnd := []attribute.KeyValue{
+		methodAttr,
+		statusAttr,
+	}
+	clientSideEnd = append(clientSideEnd, options.CSMLabels...)
+	serverSideEnd = append(serverSideEnd, options.CSMLabels...)
+	compressedBytesSentRecv := int64(options.StreamingCompressedMessageSize)
+	bucketCounts := createBucketCounts([]float64{options.StreamingCompressedMessageSize}, DefaultSizeBounds)
+	extrema := metricdata.NewExtrema(int64(options.StreamingCompressedMessageSize))
+	return []metricdata.Metrics{
+		{
+			Name:        "grpc.client.attempt.started",
+			Description: "Number of client call attempts started.",
+			Unit:        "{attempt}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(methodAttr, targetAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.duration",
+			Description: "End-to-end time taken to complete a client call attempt.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(clientSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.sent_total_compressed_message_size",
+			Description: "Compressed message bytes sent per client call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(clientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.rcvd_total_compressed_message_size",
+			Description: "Compressed message bytes received per call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(clientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.call.duration",
+			Description: "Time taken by gRPC to complete an RPC from application's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(methodAttr, targetAttr, statusAttr),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.started",
+			Description: "Number of server calls started.",
+			Unit:        "{call}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(methodAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.server.call.sent_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes sent per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(serverSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.rcvd_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes received per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(serverSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: bucketCounts,
+						Min:          extrema,
+						Max:          extrema,
+						Sum:          compressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.duration",
+			Description: "End-to-end time taken to complete a call from server transport's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(serverSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+	}
+}
+
+// MetricData returns a metricsDataSlice for A66 metrics for client and server
+// with a unary RPC and streaming RPC with certain compression and message flow
+// sent. If csmAttributes is set to true, the corresponding CSM Metrics (not
+// client side call metrics, or started on client and server side).
+func MetricData(options MetricDataOptions) []metricdata.Metrics {
+	unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall")
+	duplexMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/FullDuplexCall")
+	targetAttr := attribute.String("grpc.target", options.Target)
+	unaryStatusAttr := attribute.String("grpc.status", "OK")
+	streamingStatusAttr := attribute.String("grpc.status", "OK")
+	if options.UnaryCallFailed {
+		unaryStatusAttr = attribute.String("grpc.status", "UNKNOWN")
+	}
+	unaryMethodClientSideEnd := []attribute.KeyValue{
+		unaryMethodAttr,
+		targetAttr,
+		unaryStatusAttr,
+	}
+	streamingMethodClientSideEnd := []attribute.KeyValue{
+		duplexMethodAttr,
+		targetAttr,
+		streamingStatusAttr,
+	}
+	unaryMethodServerSideEnd := []attribute.KeyValue{
+		unaryMethodAttr,
+		unaryStatusAttr,
+	}
+	streamingMethodServerSideEnd := []attribute.KeyValue{
+		duplexMethodAttr,
+		streamingStatusAttr,
+	}
+
+	unaryMethodClientSideEnd = append(unaryMethodClientSideEnd, options.CSMLabels...)
+	streamingMethodClientSideEnd = append(streamingMethodClientSideEnd, options.CSMLabels...)
+	unaryMethodServerSideEnd = append(unaryMethodServerSideEnd, options.CSMLabels...)
+	streamingMethodServerSideEnd = append(streamingMethodServerSideEnd, options.CSMLabels...)
+	unaryCompressedBytesSentRecv := int64(options.UnaryCompressedMessageSize)
+	unaryBucketCounts := createBucketCounts([]float64{options.UnaryCompressedMessageSize}, DefaultSizeBounds)
+	unaryExtrema := metricdata.NewExtrema(int64(options.UnaryCompressedMessageSize))
+
+	streamingCompressedBytesSentRecv := int64(options.StreamingCompressedMessageSize)
+	streamingBucketCounts := createBucketCounts([]float64{options.StreamingCompressedMessageSize}, DefaultSizeBounds)
+	streamingExtrema := metricdata.NewExtrema(int64(options.StreamingCompressedMessageSize))
+
+	return []metricdata.Metrics{
+		{
+			Name:        "grpc.client.attempt.started",
+			Description: "Number of client call attempts started.",
+			Unit:        "{attempt}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr),
+						Value:      1,
+					},
+					{
+						Attributes: attribute.NewSet(duplexMethodAttr, targetAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.duration",
+			Description: "End-to-end time taken to complete a client call attempt.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+					{
+						Attributes: attribute.NewSet(streamingMethodClientSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.sent_total_compressed_message_size",
+			Description: "Compressed message bytes sent per client call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+					{
+						Attributes:   attribute.NewSet(streamingMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: streamingBucketCounts,
+						Min:          streamingExtrema,
+						Max:          streamingExtrema,
+						Sum:          streamingCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.rcvd_total_compressed_message_size",
+			Description: "Compressed message bytes received per call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+					{
+						Attributes:   attribute.NewSet(streamingMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: streamingBucketCounts,
+						Min:          streamingExtrema,
+						Max:          streamingExtrema,
+						Sum:          streamingCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.call.duration",
+			Description: "Time taken by gRPC to complete an RPC from application's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+					{
+						Attributes: attribute.NewSet(duplexMethodAttr, targetAttr, streamingStatusAttr),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.started",
+			Description: "Number of server calls started.",
+			Unit:        "{call}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr),
+						Value:      1,
+					},
+					{
+						Attributes: attribute.NewSet(duplexMethodAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "grpc.server.call.sent_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes sent per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodServerSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+					{
+						Attributes:   attribute.NewSet(streamingMethodServerSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: streamingBucketCounts,
+						Min:          streamingExtrema,
+						Max:          streamingExtrema,
+						Sum:          streamingCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.rcvd_total_compressed_message_size",
+			Unit:        "By",
+			Description: "Compressed message bytes received per server call.",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodServerSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+					{
+						Attributes:   attribute.NewSet(streamingMethodServerSideEnd...),
+						Count:        1,
+						Bounds:       DefaultSizeBounds,
+						BucketCounts: streamingBucketCounts,
+						Min:          streamingExtrema,
+						Max:          streamingExtrema,
+						Sum:          streamingCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.server.call.duration",
+			Description: "End-to-end time taken to complete a call from server transport's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodServerSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+					{
+						Attributes: attribute.NewSet(streamingMethodServerSideEnd...),
+						Count:      1,
+						Bounds:     DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+	}
+}
+
+// GzipCompressedMessageSize returns the byte length of the proto message after
+// gzip compression.
+func GzipCompressedMessageSize(t *testing.T, m proto.Message) int {
+	t.Helper()
+	data, err := proto.Marshal(m)
+	if err != nil {
+		t.Fatalf("proto.Marshal failed: %v", err)
+	}
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	if _, err := zw.Write(data); err != nil {
+		t.Fatalf("gzip.Write failed: %v", err)
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatalf("gzip.Close failed: %v", err)
+	}
+	return buf.Len()
+}
+
+// CompareMetrics asserts wantMetrics are what we expect. For duration metrics
+// makes sure the data point is within possible testing time (five seconds from
+// context timeout).
+func CompareMetrics(t *testing.T, gotMetrics map[string]metricdata.Metrics, wantMetrics []metricdata.Metrics) {
+	for _, metric := range wantMetrics {
+		val, ok := gotMetrics[metric.Name]
+		if !ok {
+			t.Errorf("Metric %v not present in recorded metrics", metric.Name)
+			continue
+		}
+
+		if metric.Name == "grpc.server.call.sent_total_compressed_message_size" || metric.Name == "grpc.server.call.rcvd_total_compressed_message_size" {
+			val := gotMetrics[metric.Name]
+			if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+				t.Errorf("Metrics data type not equal for metric: %v", metric.Name)
+			}
+			continue
+		}
+
+		// If one of the duration metrics, ignore the bucket counts, and make
+		// sure it count falls within a bucket <= 5 seconds (maximum duration of
+		// test due to context).
+		if metric.Name == "grpc.client.attempt.duration" || metric.Name == "grpc.client.call.duration" || metric.Name == "grpc.server.call.duration" {
+			val := gotMetrics[metric.Name]
+			if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars(), metricdatatest.IgnoreValue()) {
+				t.Errorf("Metrics data type not equal for metric: %v", metric.Name)
+			}
+			if err := checkDataPointWithinFiveSeconds(val); err != nil {
+				t.Errorf("Data point not within five seconds for metric %v: %v", metric.Name, err)
+			}
+			continue
+		}
+
+		if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+			t.Errorf("Metrics data type not equal for metric: %v", metric.Name)
+		}
+	}
+}
+
+// WaitForServerMetrics waits for eventual server metrics (not emitted
+// synchronously with client side rpc returning).
+func WaitForServerMetrics(ctx context.Context, t *testing.T, mr *metric.ManualReader, gotMetrics map[string]metricdata.Metrics, wantMetrics []metricdata.Metrics) map[string]metricdata.Metrics {
+	terminalMetrics := []string{
+		"grpc.server.call.sent_total_compressed_message_size",
+		"grpc.server.call.rcvd_total_compressed_message_size",
+		"grpc.client.attempt.duration",
+		"grpc.client.call.duration",
+		"grpc.server.call.duration",
+	}
+	for _, metric := range wantMetrics {
+		if !slices.Contains(terminalMetrics, metric.Name) {
+			continue
+		}
+		// Sync the metric reader to see the event because stats.End is
+		// handled async server side. Thus, poll until metrics created from
+		// stats.End show up.
+		var err error
+		if gotMetrics, err = waitForServerCompletedRPCs(ctx, mr, metric); err != nil { // move to shared helper
+			t.Fatal(err)
+		}
+	}
+
+	return gotMetrics
+}

stats/opentelemetry/internal/tracing/carrier.go

@@ -0,0 +1,131 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package tracing implements the OpenTelemetry carrier for context propagation
+// in gRPC tracing.
+package tracing
+
+import (
+	"context"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/metadata"
+)
+
+var logger = grpclog.Component("otel-plugin")
+
+// IncomingCarrier is a TextMapCarrier that uses incoming `context.Context` to
+// retrieve any propagated key-value pairs in text format.
+type IncomingCarrier struct {
+	ctx context.Context
+}
+
+// NewIncomingCarrier creates a new `IncomingCarrier` with the given context.
+// The incoming carrier should be used with propagator's `Extract()` method in
+// the incoming rpc path.
+func NewIncomingCarrier(ctx context.Context) *IncomingCarrier {
+	return &IncomingCarrier{ctx: ctx}
+}
+
+// Get returns the string value associated with the passed key from the
+// carrier's incoming context metadata.
+//
+// It returns an empty string if the key is not present in the carrier's
+// context or if the value associated with the key is empty.
+//
+// If multiple values are present for a key, it returns the last one.
+func (c *IncomingCarrier) Get(key string) string {
+	values := metadata.ValueFromIncomingContext(c.ctx, key)
+	if len(values) == 0 {
+		return ""
+	}
+	return values[len(values)-1]
+}
+
+// Set just logs an error. It implements the `TextMapCarrier` interface but
+// should not be used with `IncomingCarrier`.
+func (c *IncomingCarrier) Set(string, string) {
+	logger.Error("Set() should not be used with IncomingCarrier.")
+}
+
+// Keys returns the keys stored in the carrier's context metadata. It returns
+// keys from incoming context metadata.
+func (c *IncomingCarrier) Keys() []string {
+	md, ok := metadata.FromIncomingContext(c.ctx)
+	if !ok {
+		return nil
+	}
+	keys := make([]string, 0, len(md))
+	for key := range md {
+		keys = append(keys, key)
+	}
+	return keys
+}
+
+// Context returns the underlying context associated with the
+// `IncomingCarrier“.
+func (c *IncomingCarrier) Context() context.Context {
+	return c.ctx
+}
+
+// OutgoingCarrier is a TextMapCarrier that uses outgoing `context.Context` to
+// store any propagated key-value pairs in text format.
+type OutgoingCarrier struct {
+	ctx context.Context
+}
+
+// NewOutgoingCarrier creates a new Carrier with the given context. The
+// outgoing carrier should be used with propagator's `Inject()` method in the
+// outgoing rpc path.
+func NewOutgoingCarrier(ctx context.Context) *OutgoingCarrier {
+	return &OutgoingCarrier{ctx: ctx}
+}
+
+// Get just logs an error and returns an empty string. It implements the
+// `TextMapCarrier` interface but should not be used with `OutgoingCarrier`.
+func (c *OutgoingCarrier) Get(string) string {
+	logger.Error("Get() should not be used with `OutgoingCarrier`")
+	return ""
+}
+
+// Set stores the key-value pair in the carrier's outgoing context metadata.
+//
+// If the key already exists, given value is appended to the last.
+func (c *OutgoingCarrier) Set(key, value string) {
+	c.ctx = metadata.AppendToOutgoingContext(c.ctx, key, value)
+}
+
+// Keys returns the keys stored in the carrier's context metadata. It returns
+// keys from outgoing context metadata.
+func (c *OutgoingCarrier) Keys() []string {
+	md, ok := metadata.FromOutgoingContext(c.ctx)
+	if !ok {
+		return nil
+	}
+	keys := make([]string, 0, len(md))
+	for key := range md {
+		keys = append(keys, key)
+	}
+	return keys
+}
+
+// Context returns the underlying context associated with the
+// `OutgoingCarrier“.
+func (c *OutgoingCarrier) Context() context.Context {
+	return c.ctx
+}

stats/opentelemetry/internal/tracing/carrier_test.go

@@ -0,0 +1,190 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     htestp://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package tracing
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/metadata"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestIncomingCarrier verifies that `IncomingCarrier.Get()` returns correct
+// value for the corresponding key in the carrier's context metadata, if key is
+// present. If key is not present, it verifies that empty string is returned.
+//
+// If multiple values are present for a key, it verifies that last value is
+// returned.
+//
+// If key ends with `-bin`, it verifies that a correct binary value is returned
+// in the string format for the binary header.
+func (s) TestIncomingCarrier(t *testing.T) {
+	tests := []struct {
+		name     string
+		md       metadata.MD
+		key      string
+		want     string
+		wantKeys []string
+	}{
+		{
+			name:     "existing key",
+			md:       metadata.Pairs("key1", "value1"),
+			key:      "key1",
+			want:     "value1",
+			wantKeys: []string{"key1"},
+		},
+		{
+			name:     "non-existing key",
+			md:       metadata.Pairs("key1", "value1"),
+			key:      "key2",
+			want:     "",
+			wantKeys: []string{"key1"},
+		},
+		{
+			name:     "empty key",
+			md:       metadata.MD{},
+			key:      "key1",
+			want:     "",
+			wantKeys: []string{},
+		},
+		{
+			name:     "more than one key/value pair",
+			md:       metadata.MD{"key1": []string{"value1"}, "key2": []string{"value2"}},
+			key:      "key2",
+			want:     "value2",
+			wantKeys: []string{"key1", "key2"},
+		},
+		{
+			name:     "more than one value for a key",
+			md:       metadata.MD{"key1": []string{"value1", "value2"}},
+			key:      "key1",
+			want:     "value2",
+			wantKeys: []string{"key1"},
+		},
+		{
+			name:     "grpc-trace-bin key",
+			md:       metadata.Pairs("grpc-trace-bin", string([]byte{0x01, 0x02, 0x03})),
+			key:      "grpc-trace-bin",
+			want:     string([]byte{0x01, 0x02, 0x03}),
+			wantKeys: []string{"grpc-trace-bin"},
+		},
+		{
+			name:     "grpc-trace-bin key with another string key",
+			md:       metadata.MD{"key1": []string{"value1"}, "grpc-trace-bin": []string{string([]byte{0x01, 0x02, 0x03})}},
+			key:      "grpc-trace-bin",
+			want:     string([]byte{0x01, 0x02, 0x03}),
+			wantKeys: []string{"key1", "grpc-trace-bin"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			c := NewIncomingCarrier(metadata.NewIncomingContext(ctx, test.md))
+			got := c.Get(test.key)
+			if got != test.want {
+				t.Fatalf("c.Get() = %s, want %s", got, test.want)
+			}
+			if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) {
+				t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys)
+			}
+		})
+	}
+}
+
+// TestOutgoingCarrier verifies that a key-value pair is set in carrier's
+// context metadata using `OutgoingCarrier.Set()`. If key is not present, it
+// verifies that key-value pair is insterted. If key is already present, it
+// verifies that new value is appended at the end of list for the existing key.
+//
+// If key ends with `-bin`, it verifies that a binary value is set for
+// `-bin` header in string format.
+//
+// It also verifies that both existing and newly inserted keys are present in
+// the carrier's context using `Carrier.Keys()`.
+func (s) TestOutgoingCarrier(t *testing.T) {
+	tests := []struct {
+		name      string
+		initialMD metadata.MD
+		setKey    string
+		setValue  string
+		wantValue string // expected value of the set key
+		wantKeys  []string
+	}{
+		{
+			name:      "new key",
+			initialMD: metadata.MD{},
+			setKey:    "key1",
+			setValue:  "value1",
+			wantValue: "value1",
+			wantKeys:  []string{"key1"},
+		},
+		{
+			name:      "add to existing key",
+			initialMD: metadata.MD{"key1": []string{"oldvalue"}},
+			setKey:    "key1",
+			setValue:  "newvalue",
+			wantValue: "newvalue",
+			wantKeys:  []string{"key1"},
+		},
+		{
+			name:      "new key with different existing key",
+			initialMD: metadata.MD{"key2": []string{"value2"}},
+			setKey:    "key1",
+			setValue:  "value1",
+			wantValue: "value1",
+			wantKeys:  []string{"key2", "key1"},
+		},
+		{
+			name:      "grpc-trace-bin binary key",
+			initialMD: metadata.MD{"key1": []string{"value1"}},
+			setKey:    "grpc-trace-bin",
+			setValue:  string([]byte{0x01, 0x02, 0x03}),
+			wantValue: string([]byte{0x01, 0x02, 0x03}),
+			wantKeys:  []string{"key1", "grpc-trace-bin"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+			c := NewOutgoingCarrier(metadata.NewOutgoingContext(ctx, test.initialMD))
+			c.Set(test.setKey, test.setValue)
+			if gotKeys := c.Keys(); !cmp.Equal(test.wantKeys, gotKeys, cmpopts.SortSlices(func(a, b string) bool { return a < b })) {
+				t.Fatalf("c.Keys() = keys %v, want %v", gotKeys, test.wantKeys)
+			}
+			if md, ok := metadata.FromOutgoingContext(c.Context()); ok && md.Get(test.setKey)[len(md.Get(test.setKey))-1] != test.wantValue {
+				t.Fatalf("got value %s, want %s, for key %s", md.Get(test.setKey)[len(md.Get(test.setKey))-1], test.wantValue, test.setKey)
+			}
+		})
+	}
+}

stats/opentelemetry/metricsregistry_test.go

@@ -0,0 +1,341 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/internal/grpctest"
+
+	"go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest"
+)
+
+var defaultTestTimeout = 5 * time.Second
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+type metricsRecorderForTest interface {
+	estats.MetricsRecorder
+	initializeMetrics()
+}
+
+func newClientStatsHandler(options MetricsOptions) metricsRecorderForTest {
+	return &clientMetricsHandler{options: Options{MetricsOptions: options}}
+}
+
+func newServerStatsHandler(options MetricsOptions) metricsRecorderForTest {
+	return &serverMetricsHandler{options: Options{MetricsOptions: options}}
+}
+
+// TestMetricsRegistryMetrics tests the OpenTelemetry behavior with respect to
+// registered metrics. It registers metrics in the metrics registry. It then
+// creates an OpenTelemetry client and server stats handler This test then makes
+// measurements on those instruments using one of the stats handlers, then tests
+// the expected metrics emissions, which includes default metrics and optional
+// label assertions.
+func (s) TestMetricsRegistryMetrics(t *testing.T) {
+	cleanup := internal.SnapshotMetricRegistryForTesting()
+	defer cleanup()
+
+	intCountHandle1 := estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "int-counter-1",
+		Description:    "Sum of calls from test",
+		Unit:           "int",
+		Labels:         []string{"int counter 1 label key"},
+		OptionalLabels: []string{"int counter 1 optional label key"},
+		Default:        true,
+	})
+	// A non default metric. If not specified in OpenTelemetry constructor, this
+	// will become a no-op, so measurements recorded on it won't show up in
+	// emitted metrics.
+	intCountHandle2 := estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "int-counter-2",
+		Description:    "Sum of calls from test",
+		Unit:           "int",
+		Labels:         []string{"int counter 2 label key"},
+		OptionalLabels: []string{"int counter 2 optional label key"},
+		Default:        false,
+	})
+	// Register another non default metric. This will get added to the default
+	// metrics set in the OpenTelemetry constructor options, so metrics recorded
+	// on this should show up in metrics emissions.
+	intCountHandle3 := estats.RegisterInt64Count(estats.MetricDescriptor{
+		Name:           "int-counter-3",
+		Description:    "sum of calls from test",
+		Unit:           "int",
+		Labels:         []string{"int counter 3 label key"},
+		OptionalLabels: []string{"int counter 3 optional label key"},
+		Default:        false,
+	})
+	floatCountHandle := estats.RegisterFloat64Count(estats.MetricDescriptor{
+		Name:           "float-counter",
+		Description:    "sum of calls from test",
+		Unit:           "float",
+		Labels:         []string{"float counter label key"},
+		OptionalLabels: []string{"float counter optional label key"},
+		Default:        true,
+	})
+	bounds := []float64{0, 5, 10}
+	intHistoHandle := estats.RegisterInt64Histo(estats.MetricDescriptor{
+		Name:           "int-histo",
+		Description:    "histogram of call values from tests",
+		Unit:           "int",
+		Labels:         []string{"int histo label key"},
+		OptionalLabels: []string{"int histo optional label key"},
+		Default:        true,
+		Bounds:         bounds,
+	})
+	floatHistoHandle := estats.RegisterFloat64Histo(estats.MetricDescriptor{
+		Name:           "float-histo",
+		Description:    "histogram of call values from tests",
+		Unit:           "float",
+		Labels:         []string{"float histo label key"},
+		OptionalLabels: []string{"float histo optional label key"},
+		Default:        true,
+		Bounds:         bounds,
+	})
+	intGaugeHandle := estats.RegisterInt64Gauge(estats.MetricDescriptor{
+		Name:           "simple-gauge",
+		Description:    "the most recent int emitted by test",
+		Unit:           "int",
+		Labels:         []string{"int gauge label key"},
+		OptionalLabels: []string{"int gauge optional label key"},
+		Default:        true,
+	})
+	intAsyncHandle := estats.RegisterInt64AsyncGauge(estats.MetricDescriptor{
+		Name:           "async-gauge",
+		Description:    "async gauge value from test",
+		Unit:           "int",
+		Labels:         []string{"async label key"},
+		OptionalLabels: []string{"async optional label key"},
+		Default:        true,
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Only float optional labels are configured, so only float optional labels should show up.
+	// All required labels should show up.
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "int-counter-1",
+			Description: "Sum of calls from test",
+			Unit:        "int",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("int counter 1 label key", "int counter 1 label value")), // No optional label, not float.
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "int-counter-3",
+			Description: "sum of calls from test",
+			Unit:        "int",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("int counter 3 label key", "int counter 3 label value")), // No optional label, not float.
+						Value:      4,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "float-counter",
+			Description: "sum of calls from test",
+			Unit:        "float",
+			Data: metricdata.Sum[float64]{
+				DataPoints: []metricdata.DataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("float counter label key", "float counter label value"), attribute.String("float counter optional label key", "float counter optional label value")),
+						Value:      1.2,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		},
+		{
+			Name:        "int-histo",
+			Description: "histogram of call values from tests",
+			Unit:        "int",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(attribute.String("int histo label key", "int histo label value")), // No optional label, not float.
+						Count:        1,
+						Bounds:       bounds,
+						BucketCounts: []uint64{0, 1, 0, 0},
+						Min:          metricdata.NewExtrema(int64(3)),
+						Max:          metricdata.NewExtrema(int64(3)),
+						Sum:          3,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "float-histo",
+			Description: "histogram of call values from tests",
+			Unit:        "float",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes:   attribute.NewSet(attribute.String("float histo label key", "float histo label value"), attribute.String("float histo optional label key", "float histo optional label value")),
+						Count:        1,
+						Bounds:       bounds,
+						BucketCounts: []uint64{0, 1, 0, 0},
+						Min:          metricdata.NewExtrema(float64(4.3)),
+						Max:          metricdata.NewExtrema(float64(4.3)),
+						Sum:          4.3,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "simple-gauge",
+			Description: "the most recent int emitted by test",
+			Unit:        "int",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(attribute.String("int gauge label key", "int gauge label value")), // No optional label, not float.
+						Value:      8,
+					},
+				},
+			},
+		},
+		{
+			Name:        "async-gauge",
+			Description: "async gauge value from test",
+			Unit:        "int",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						// Note: Only the required label is expected because optional labels
+						// for "async optional label key" are not enabled in MetricsOptions below.
+						Attributes: attribute.NewSet(attribute.String("async label key", "async label value")),
+						Value:      999,
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range []struct {
+		name        string
+		constructor func(options MetricsOptions) metricsRecorderForTest
+	}{
+		{
+			name:        "client stats handler",
+			constructor: newClientStatsHandler,
+		},
+		{
+			name:        "server stats handler",
+			constructor: newServerStatsHandler,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			reader := otelmetric.NewManualReader()
+			provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader))
+
+			// This configures the defaults alongside int counter 3. All the instruments
+			// registered except int counter 2 and 3 are default, so all measurements
+			// recorded should show up in reader collected metrics except those for int
+			// counter 2.
+			// This also only toggles the float count and float histo optional labels,
+			// so only those should show up in metrics emissions. All the required
+			// labels should show up in metrics emissions.
+			mo := MetricsOptions{
+				Metrics:        DefaultMetrics().Add("int-counter-3"),
+				OptionalLabels: []string{"float counter optional label key", "float histo optional label key"},
+				MeterProvider:  provider,
+			}
+			mr := test.constructor(mo)
+			mr.initializeMetrics()
+			// These Record calls are guaranteed at a layer underneath OpenTelemetry for
+			// labels emitted to match the length of labels + optional labels.
+			intCountHandle1.Record(mr, 1, []string{"int counter 1 label value", "int counter 1 optional label value"}...)
+			// int-counter-2 is not part of metrics specified (not default), so this
+			// record call shouldn't show up in the reader.
+			intCountHandle2.Record(mr, 2, []string{"int counter 2 label value", "int counter 2 optional label value"}...)
+			// int-counter-3 is part of metrics specified, so this call should show up
+			// in the reader.
+			intCountHandle3.Record(mr, 4, []string{"int counter 3 label value", "int counter 3 optional label value"}...)
+
+			// All future recording points should show up in emissions as all of these are defaults.
+			floatCountHandle.Record(mr, 1.2, []string{"float counter label value", "float counter optional label value"}...)
+			intHistoHandle.Record(mr, 3, []string{"int histo label value", "int histo optional label value"}...)
+			floatHistoHandle.Record(mr, 4.3, []string{"float histo label value", "float histo optional label value"}...)
+			intGaugeHandle.Record(mr, 7, []string{"int gauge label value", "int gauge optional label value"}...)
+			// This second gauge call should take the place of the previous gauge call.
+			intGaugeHandle.Record(mr, 8, []string{"int gauge label value", "int gauge optional label value"}...)
+			reporter := estats.AsyncMetricReporterFunc(func(r estats.AsyncMetricsRecorder) error {
+				// We record value 999.
+				// Note: We pass both required and optional labels, but the expectation (Step 2)
+				// knows that the optional one will be dropped based on 'mo' config.
+				intAsyncHandle.Record(r, 999, "async label value", "async optional label value")
+				return nil
+			})
+			mr.RegisterAsyncReporter(reporter, intAsyncHandle)
+			rm := &metricdata.ResourceMetrics{}
+			reader.Collect(ctx, rm)
+			gotMetrics := map[string]metricdata.Metrics{}
+			for _, sm := range rm.ScopeMetrics {
+				for _, m := range sm.Metrics {
+					gotMetrics[m.Name] = m
+				}
+			}
+			for _, metric := range wantMetrics {
+				val, ok := gotMetrics[metric.Name]
+				if !ok {
+					t.Fatalf("Metric %v not present in recorded metrics", metric.Name)
+				}
+				if !metricdatatest.AssertEqual(t, metric, val, metricdatatest.IgnoreTimestamp(), metricdatatest.IgnoreExemplars()) {
+					t.Fatalf("Metrics data type not equal for metric: %v", metric.Name)
+				}
+			}
+
+			// int-counter-2 is not a default metric and wasn't specified in
+			// constructor, so emissions should not show up.
+			if _, ok := gotMetrics["int-counter-2"]; ok {
+				t.Fatalf("Metric int-counter-2 present in recorded metrics, was not configured")
+			}
+		})
+	}
+}

stats/opentelemetry/opentelemetry.go

@@ -0,0 +1,626 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package opentelemetry implements opentelemetry instrumentation code for
+// gRPC-Go clients and servers.
+//
+// For details on configuring opentelemetry and various instruments that this
+// package creates, see
+// [gRPC OpenTelemetry Metrics](https://grpc.io/docs/guides/opentelemetry-metrics/).
+package opentelemetry
+
+import (
+	"context"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	otelattribute "go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
+	"go.opentelemetry.io/otel/metric/noop"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	experimental "google.golang.org/grpc/experimental/opentelemetry"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats"
+	otelinternal "google.golang.org/grpc/stats/opentelemetry/internal"
+)
+
+func init() {
+	otelinternal.SetPluginOption = func(o *Options, po otelinternal.PluginOption) {
+		o.MetricsOptions.pluginOption = po
+		// Log an error if one of the options is missing.
+		if (o.TraceOptions.TextMapPropagator == nil) != (o.TraceOptions.TracerProvider == nil) {
+			logger.Warning("Tracing will not be recorded because traceOptions are not set properly: one of TextMapPropagator or TracerProvider is missing")
+		}
+	}
+}
+
+var (
+	logger          = grpclog.Component("otel-plugin")
+	canonicalString = internal.CanonicalString.(func(codes.Code) string)
+	joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption)
+)
+
+// Options are the options for OpenTelemetry instrumentation.
+type Options struct {
+	// MetricsOptions are the metrics options for OpenTelemetry instrumentation.
+	MetricsOptions MetricsOptions
+	// TraceOptions are the tracing options for OpenTelemetry instrumentation.
+	TraceOptions experimental.TraceOptions
+}
+
+func (o *Options) isMetricsEnabled() bool {
+	return o.MetricsOptions.MeterProvider != nil
+}
+
+func (o *Options) isTracingEnabled() bool {
+	return o.TraceOptions.TracerProvider != nil
+}
+
+// MetricsOptions are the metrics options for OpenTelemetry instrumentation.
+type MetricsOptions struct {
+	// MeterProvider is the MeterProvider instance that will be used to create
+	// instruments. To enable metrics collection, set a meter provider. If
+	// unset, no metrics will be recorded.
+	MeterProvider otelmetric.MeterProvider
+
+	// Metrics are the metrics to instrument. Will create instrument and record telemetry
+	// for corresponding metric supported by the client and server
+	// instrumentation components if applicable. If not set, the default metrics
+	// will be recorded.
+	Metrics *stats.MetricSet
+
+	// MethodAttributeFilter is a function that determines whether to record the
+	// method name of RPCs as an attribute, or to bucket into "other". Take care
+	// to limit the values allowed, as allowing too many will increase
+	// cardinality and could cause severe memory or performance problems.
+	//
+	// This only applies for server-side metrics.  For clients, to record the
+	// method name in the attributes, pass grpc.StaticMethodCallOption to Invoke
+	// or NewStream. Note that when using protobuf generated clients, this
+	// CallOption is included automatically.
+	MethodAttributeFilter func(string) bool
+
+	// OptionalLabels specifies a list of optional labels to enable on any
+	// metrics that support them.
+	OptionalLabels []string
+
+	// pluginOption is used to get labels to attach to certain metrics, if set.
+	pluginOption otelinternal.PluginOption
+}
+
+// DialOption returns a dial option which enables OpenTelemetry instrumentation
+// code for a grpc.ClientConn.
+//
+// Client applications interested in instrumenting their grpc.ClientConn should
+// pass the dial option returned from this function as a dial option to
+// grpc.NewClient().
+//
+// For the metrics supported by this instrumentation code, specify the client
+// metrics to record in metrics options. Also provide an implementation of a
+// MeterProvider. If the passed in Meter Provider does not have the view
+// configured for an individual metric turned on, the API call in this component
+// will create a default view for that metric.
+//
+// For the traces supported by this instrumentation code, provide an
+// implementation of a TextMapPropagator and OpenTelemetry TracerProvider.
+func DialOption(o Options) grpc.DialOption {
+	var metricsOpts, tracingOpts []grpc.DialOption
+
+	if o.isMetricsEnabled() {
+		metricsHandler := &clientMetricsHandler{options: o}
+		metricsHandler.initializeMetrics()
+		metricsOpts = append(metricsOpts, grpc.WithChainUnaryInterceptor(metricsHandler.unaryInterceptor), grpc.WithChainStreamInterceptor(metricsHandler.streamInterceptor), grpc.WithStatsHandler(metricsHandler))
+	}
+	if o.isTracingEnabled() {
+		tracingHandler := &clientTracingHandler{options: o}
+		tracingHandler.initializeTraces()
+		tracingOpts = append(tracingOpts, grpc.WithChainUnaryInterceptor(tracingHandler.unaryInterceptor), grpc.WithChainStreamInterceptor(tracingHandler.streamInterceptor), grpc.WithStatsHandler(tracingHandler))
+	}
+	return joinDialOptions(append(metricsOpts, tracingOpts...)...)
+}
+
+var joinServerOptions = internal.JoinServerOptions.(func(...grpc.ServerOption) grpc.ServerOption)
+
+// ServerOption returns a server option which enables OpenTelemetry
+// instrumentation code for a grpc.Server.
+//
+// Server applications interested in instrumenting their grpc.Server should pass
+// the server option returned from this function as an argument to
+// grpc.NewServer().
+//
+// For the metrics supported by this instrumentation code, specify the server
+// metrics to record in metrics options. Also provide an implementation of a
+// MeterProvider. If the passed in Meter Provider does not have the view
+// configured for an individual metric turned on, the API call in this component
+// will create a default view for that metric.
+//
+// For the traces supported by this instrumentation code, provide an
+// implementation of a TextMapPropagator and OpenTelemetry TracerProvider.
+func ServerOption(o Options) grpc.ServerOption {
+	var metricsOpts, tracingOpts []grpc.ServerOption
+
+	if o.isMetricsEnabled() {
+		metricsHandler := &serverMetricsHandler{options: o}
+		metricsHandler.initializeMetrics()
+		metricsOpts = append(metricsOpts, grpc.ChainUnaryInterceptor(metricsHandler.unaryInterceptor), grpc.ChainStreamInterceptor(metricsHandler.streamInterceptor), grpc.StatsHandler(metricsHandler))
+	}
+	if o.isTracingEnabled() {
+		tracingHandler := &serverTracingHandler{options: o}
+		tracingHandler.initializeTraces()
+		tracingOpts = append(tracingOpts, grpc.StatsHandler(tracingHandler))
+	}
+	return joinServerOptions(append(metricsOpts, tracingOpts...)...)
+}
+
+// callInfo is information pertaining to the lifespan of the RPC client side.
+type callInfo struct {
+	target string
+
+	method string
+
+	// nameResolutionEventAdded is set when the resolver delay trace event
+	// is added. Prevents duplicate events, since it is reported per-attempt.
+	nameResolutionEventAdded atomic.Bool
+	// previousRPCAttempts holds the count of RPC attempts that have happened
+	// before current attempt. Transparent retries are excluded.
+	previousRPCAttempts atomic.Uint32
+}
+
+type callInfoKey struct{}
+
+// getCallInfo returns the callInfo stored in the context, or nil
+// if there isn't one.
+func getCallInfo(ctx context.Context) *callInfo {
+	ci, _ := ctx.Value(callInfoKey{}).(*callInfo)
+	return ci
+}
+
+// rpcInfo is RPC information scoped to the RPC attempt life span client side,
+// and the RPC life span server side.
+type rpcInfo struct {
+	ai *attemptInfo
+}
+
+type clientRPCInfoKey struct{}
+type serverRPCInfoKey struct{}
+
+// clientRPCInfo returns the rpcInfo stored in the context for client, or nil
+// if there isn't one.
+func clientRPCInfo(ctx context.Context) *rpcInfo {
+	ri, _ := ctx.Value(clientRPCInfoKey{}).(*rpcInfo)
+	return ri
+}
+
+// serverRPCInfo returns the rpcInfo stored in the context for server, or nil
+// if there isn't one.
+func serverRPCInfo(ctx context.Context) *rpcInfo {
+	ri, _ := ctx.Value(serverRPCInfoKey{}).(*rpcInfo)
+	return ri
+}
+
+func getOrCreateClientRPCInfo(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *rpcInfo) {
+	ri := clientRPCInfo(ctx)
+	if ri != nil {
+		return ctx, ri
+	}
+	ri = &rpcInfo{
+		ai: &attemptInfo{
+			startTime: time.Now(),
+			method:    removeLeadingSlash(info.FullMethodName),
+			xdsLabels: map[string]string{
+				// The defaults for all the per call labels from a plugin that
+				// executes on the callpath that this OpenTelemetry component
+				// currently supports.
+				"grpc.lb.locality":        "",
+				"grpc.lb.backend_service": "",
+			},
+		},
+	}
+	return context.WithValue(ctx, clientRPCInfoKey{}, ri), ri
+}
+
+func getOrCreateServerRPCInfo(ctx context.Context, info *stats.RPCTagInfo, options MetricsOptions) (context.Context, *rpcInfo) {
+	ri := serverRPCInfo(ctx)
+	if ri != nil {
+		return ctx, ri
+	}
+	method := info.FullMethodName
+	if options.MethodAttributeFilter != nil {
+		if !options.MethodAttributeFilter(method) {
+			method = "other"
+		}
+	}
+	server := internal.ServerFromContext.(func(context.Context) *grpc.Server)(ctx)
+	if server == nil { // Shouldn't happen, defensive programming.
+		logger.Error("ctx passed into server side stats handler has no grpc server ref")
+		method = "other"
+	} else {
+		isRegisteredMethod := internal.IsRegisteredMethod.(func(*grpc.Server, string) bool)
+		if !isRegisteredMethod(server, method) {
+			method = "other"
+		}
+	}
+
+	ri = &rpcInfo{
+		ai: &attemptInfo{
+			startTime: time.Now(),
+			method:    removeLeadingSlash(method),
+		},
+	}
+	return context.WithValue(ctx, serverRPCInfoKey{}, ri), ri
+}
+
+func removeLeadingSlash(mn string) string {
+	return strings.TrimLeft(mn, "/")
+}
+
+// attemptInfo is RPC information scoped to the RPC attempt life span client
+// side, and the RPC life span server side.
+type attemptInfo struct {
+	// access these counts atomically for hedging in the future:
+	// number of bytes after compression (within each message) from side (client
+	// || server).
+	sentCompressedBytes int64
+	// number of compressed bytes received (within each message) received on
+	// side (client || server).
+	recvCompressedBytes int64
+
+	startTime time.Time
+	method    string
+
+	pluginOptionLabels map[string]string // pluginOptionLabels to attach to metrics emitted
+	xdsLabels          map[string]string
+
+	// traceSpan is data used for recording traces.
+	traceSpan trace.Span
+	// message counters for sent and received messages (used for
+	// generating message IDs), and the number of previous RPC attempts for the
+	// associated call.
+	countSentMsg uint32
+	countRecvMsg uint32
+}
+
+type clientMetrics struct {
+	// "grpc.client.attempt.started"
+	attemptStarted otelmetric.Int64Counter
+	// "grpc.client.attempt.duration"
+	attemptDuration otelmetric.Float64Histogram
+	// "grpc.client.attempt.sent_total_compressed_message_size"
+	attemptSentTotalCompressedMessageSize otelmetric.Int64Histogram
+	// "grpc.client.attempt.rcvd_total_compressed_message_size"
+	attemptRcvdTotalCompressedMessageSize otelmetric.Int64Histogram
+	// "grpc.client.call.duration"
+	callDuration otelmetric.Float64Histogram
+}
+
+type serverMetrics struct {
+	// "grpc.server.call.started"
+	callStarted otelmetric.Int64Counter
+	// "grpc.server.call.sent_total_compressed_message_size"
+	callSentTotalCompressedMessageSize otelmetric.Int64Histogram
+	// "grpc.server.call.rcvd_total_compressed_message_size"
+	callRcvdTotalCompressedMessageSize otelmetric.Int64Histogram
+	// "grpc.server.call.duration"
+	callDuration otelmetric.Float64Histogram
+}
+
+func createInt64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64CounterOption) otelmetric.Int64Counter {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Int64Counter{}
+	}
+	ret, err := meter.Int64Counter(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Int64Counter{}
+	}
+	return ret
+}
+
+func createInt64UpDownCounter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64UpDownCounterOption) otelmetric.Int64UpDownCounter {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Int64UpDownCounter{}
+	}
+	ret, err := meter.Int64UpDownCounter(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("Failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Int64UpDownCounter{}
+	}
+	return ret
+}
+
+func createFloat64Counter(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64CounterOption) otelmetric.Float64Counter {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Float64Counter{}
+	}
+	ret, err := meter.Float64Counter(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Float64Counter{}
+	}
+	return ret
+}
+
+func createInt64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64HistogramOption) otelmetric.Int64Histogram {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Int64Histogram{}
+	}
+	ret, err := meter.Int64Histogram(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Int64Histogram{}
+	}
+	return ret
+}
+
+func createFloat64Histogram(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Float64HistogramOption) otelmetric.Float64Histogram {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Float64Histogram{}
+	}
+	ret, err := meter.Float64Histogram(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Float64Histogram{}
+	}
+	return ret
+}
+
+func createInt64Gauge(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64GaugeOption) otelmetric.Int64Gauge {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		return noop.Int64Gauge{}
+	}
+	ret, err := meter.Int64Gauge(string(metricName), options...)
+	if err != nil {
+		logger.Errorf("failed to register metric \"%v\", will not record: %v", metricName, err)
+		return noop.Int64Gauge{}
+	}
+	return ret
+}
+
+// createInt64ObservableGauge initializes an OTel Int64ObservableGauge if the metric is enabled.
+func createInt64ObservableGauge(setOfMetrics map[string]bool, metricName string, meter otelmetric.Meter, options ...otelmetric.Int64ObservableGaugeOption) otelmetric.Int64ObservableGauge {
+	if _, ok := setOfMetrics[metricName]; !ok {
+		n, _ := noop.NewMeterProvider().Meter("noop").Int64ObservableGauge("noop")
+		return n
+	}
+	ret, err := meter.Int64ObservableGauge(metricName, options...)
+	if err != nil {
+		logger.Errorf("failed to register metric %q, will not record: %v", metricName, err)
+		n, _ := noop.NewMeterProvider().Meter("noop").Int64ObservableGauge("noop")
+		return n
+	}
+	return ret
+}
+
+func optionFromLabels(labelKeys []string, optionalLabelKeys []string, optionalLabels []string, labelVals ...string) otelmetric.MeasurementOption {
+	var attributes []otelattribute.KeyValue
+
+	// Once it hits here lower level has guaranteed length of labelVals matches
+	// labelKeys + optionalLabelKeys.
+	for i, label := range labelKeys {
+		attributes = append(attributes, otelattribute.String(label, labelVals[i]))
+	}
+
+	for i, label := range optionalLabelKeys {
+		for _, optLabel := range optionalLabels { // o(n) could build out a set but n is currently capped at < 5
+			if label == optLabel {
+				attributes = append(attributes, otelattribute.String(label, labelVals[i+len(labelKeys)]))
+			}
+		}
+	}
+	return otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))
+}
+
+// registryMetrics implements MetricsRecorder for the client and server stats
+// handlers.
+type registryMetrics struct {
+	internal.EnforceMetricsRecorderEmbedding
+	intCounts       map[*estats.MetricDescriptor]otelmetric.Int64Counter
+	floatCounts     map[*estats.MetricDescriptor]otelmetric.Float64Counter
+	intHistos       map[*estats.MetricDescriptor]otelmetric.Int64Histogram
+	floatHistos     map[*estats.MetricDescriptor]otelmetric.Float64Histogram
+	intGauges       map[*estats.MetricDescriptor]otelmetric.Int64Gauge
+	intUpDownCounts map[*estats.MetricDescriptor]otelmetric.Int64UpDownCounter
+
+	// Asynchronous (Observable) Instruments
+	intObservableGauges map[*estats.MetricDescriptor]otelmetric.Int64ObservableGauge
+
+	meter          otelmetric.Meter
+	optionalLabels []string
+}
+
+func (rm *registryMetrics) registerMetrics(metrics *stats.MetricSet, meter otelmetric.Meter) {
+	rm.meter = meter
+	rm.intCounts = make(map[*estats.MetricDescriptor]otelmetric.Int64Counter)
+	rm.floatCounts = make(map[*estats.MetricDescriptor]otelmetric.Float64Counter)
+	rm.intHistos = make(map[*estats.MetricDescriptor]otelmetric.Int64Histogram)
+	rm.floatHistos = make(map[*estats.MetricDescriptor]otelmetric.Float64Histogram)
+	rm.intGauges = make(map[*estats.MetricDescriptor]otelmetric.Int64Gauge)
+	rm.intUpDownCounts = make(map[*estats.MetricDescriptor]otelmetric.Int64UpDownCounter)
+	rm.intObservableGauges = make(map[*estats.MetricDescriptor]otelmetric.Int64ObservableGauge)
+
+	for metric := range metrics.Metrics() {
+		desc := estats.DescriptorForMetric(metric)
+		if desc == nil {
+			// Either the metric was per call or the metric is not registered.
+			// Thus, if this component ever receives the desc as a handle in
+			// record it will be a no-op.
+			continue
+		}
+		switch desc.Type {
+		case estats.MetricTypeIntCount:
+			rm.intCounts[desc] = createInt64Counter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description))
+		case estats.MetricTypeFloatCount:
+			rm.floatCounts[desc] = createFloat64Counter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description))
+		case estats.MetricTypeIntHisto:
+			rm.intHistos[desc] = createInt64Histogram(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description), otelmetric.WithExplicitBucketBoundaries(desc.Bounds...))
+		case estats.MetricTypeFloatHisto:
+			rm.floatHistos[desc] = createFloat64Histogram(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description), otelmetric.WithExplicitBucketBoundaries(desc.Bounds...))
+		case estats.MetricTypeIntGauge:
+			rm.intGauges[desc] = createInt64Gauge(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description))
+		case estats.MetricTypeIntUpDownCount:
+			rm.intUpDownCounts[desc] = createInt64UpDownCounter(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description))
+		case estats.MetricTypeIntAsyncGauge:
+			rm.intObservableGauges[desc] = createInt64ObservableGauge(metrics.Metrics(), desc.Name, meter, otelmetric.WithUnit(desc.Unit), otelmetric.WithDescription(desc.Description))
+		}
+	}
+}
+
+func (rm *registryMetrics) RecordInt64Count(handle *estats.Int64CountHandle, incr int64, labels ...string) {
+	desc := handle.Descriptor()
+	if ic, ok := rm.intCounts[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		ic.Add(context.TODO(), incr, ao)
+	}
+}
+
+func (rm *registryMetrics) RecordInt64UpDownCount(handle *estats.Int64UpDownCountHandle, incr int64, labels ...string) {
+	desc := handle.Descriptor()
+	if ic, ok := rm.intUpDownCounts[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		ic.Add(context.TODO(), incr, ao)
+	}
+}
+
+func (rm *registryMetrics) RecordFloat64Count(handle *estats.Float64CountHandle, incr float64, labels ...string) {
+	desc := handle.Descriptor()
+	if fc, ok := rm.floatCounts[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		fc.Add(context.TODO(), incr, ao)
+	}
+}
+
+func (rm *registryMetrics) RecordInt64Histo(handle *estats.Int64HistoHandle, incr int64, labels ...string) {
+	desc := handle.Descriptor()
+	if ih, ok := rm.intHistos[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		ih.Record(context.TODO(), incr, ao)
+	}
+}
+
+func (rm *registryMetrics) RecordFloat64Histo(handle *estats.Float64HistoHandle, incr float64, labels ...string) {
+	desc := handle.Descriptor()
+	if fh, ok := rm.floatHistos[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		fh.Record(context.TODO(), incr, ao)
+	}
+}
+
+func (rm *registryMetrics) RecordInt64Gauge(handle *estats.Int64GaugeHandle, incr int64, labels ...string) {
+	desc := handle.Descriptor()
+	if ig, ok := rm.intGauges[desc]; ok {
+		ao := optionFromLabels(desc.Labels, desc.OptionalLabels, rm.optionalLabels, labels...)
+		ig.Record(context.TODO(), incr, ao)
+	}
+}
+
+// RegisterAsyncReporter will register a callback with the underlying OpenTelemetry
+// Meter for the provided descriptors.
+//
+// It will map the provided descriptors to their corresponding OTel Observable
+// instruments. If no instruments match the descriptors, registration is
+// skipped.
+//
+// The returned cleanup function unregisters the callback from the Meter.
+// RegisterAsyncReporter registers a callback with the OpenTelemetry Meter.
+func (rm *registryMetrics) RegisterAsyncReporter(reporter estats.AsyncMetricReporter, metrics ...estats.AsyncMetric) func() {
+	observables := make([]otelmetric.Observable, 0, len(metrics))
+	observableMap := make(map[*estats.MetricDescriptor]otelmetric.Observable, len(metrics))
+
+	for _, m := range metrics {
+		d := m.Descriptor()
+		if inst, ok := rm.intObservableGauges[d]; ok {
+			observables = append(observables, inst)
+			observableMap[d] = inst
+		}
+	}
+
+	if len(observables) == 0 {
+		return func() {}
+	}
+
+	cbWrapper := func(_ context.Context, o otelmetric.Observer) error {
+		adapter := &observerAdapter{
+			observableMap:  observableMap,
+			optionalLabels: rm.optionalLabels,
+			delegate:       o,
+		}
+		reporter.Report(adapter)
+		return nil
+	}
+
+	reg, err := rm.meter.RegisterCallback(cbWrapper, observables...)
+	if err != nil {
+		logger.Warningf("grpc: failed to register callback for async metrics: %v", err)
+		return func() {}
+	}
+
+	return func() {
+		err = reg.Unregister()
+		if err != nil {
+			logger.Errorf("grpc: failed to unregister callback for async metrics: %v", err)
+		}
+	}
+}
+
+// Users of this component should use these bucket boundaries as part of their
+// SDK MeterProvider passed in. This component sends this as "advice" to the
+// API, which works, however this stability is not guaranteed, so for safety the
+// SDK Meter Provider provided should set these bounds for corresponding
+// metrics.
+var (
+	// DefaultLatencyBounds are the default bounds for latency metrics.
+	DefaultLatencyBounds = []float64{0, 0.00001, 0.00005, 0.0001, 0.0003, 0.0006, 0.0008, 0.001, 0.002, 0.003, 0.004, 0.005, 0.006, 0.008, 0.01, 0.013, 0.016, 0.02, 0.025, 0.03, 0.04, 0.05, 0.065, 0.08, 0.1, 0.13, 0.16, 0.2, 0.25, 0.3, 0.4, 0.5, 0.65, 0.8, 1, 2, 5, 10, 20, 50, 100} // provide "advice" through API, SDK should set this too
+	// DefaultSizeBounds are the default bounds for metrics which record size.
+	DefaultSizeBounds = []float64{0, 1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296}
+	// defaultPerCallMetrics are the default metrics provided by this module.
+	defaultPerCallMetrics = stats.NewMetricSet(ClientAttemptStartedMetricName, ClientAttemptDurationMetricName, ClientAttemptSentCompressedTotalMessageSizeMetricName, ClientAttemptRcvdCompressedTotalMessageSizeMetricName, ClientCallDurationMetricName, ServerCallStartedMetricName, ServerCallSentCompressedTotalMessageSizeMetricName, ServerCallRcvdCompressedTotalMessageSizeMetricName, ServerCallDurationMetricName)
+)
+
+// DefaultMetrics returns a set of default OpenTelemetry metrics.
+//
+// This should only be invoked after init time.
+func DefaultMetrics() *stats.MetricSet {
+	return defaultPerCallMetrics.Join(estats.DefaultMetrics)
+}
+
+type observerAdapter struct {
+	observableMap  map[*estats.MetricDescriptor]otelmetric.Observable
+	optionalLabels []string
+	delegate       otelmetric.Observer
+}
+
+// RecordInt64AsyncGauge records the measurement alongside labels on the int
+// gauge associated with the provided handle.
+func (a *observerAdapter) RecordInt64AsyncGauge(handle *estats.Int64AsyncGaugeHandle, val int64, labels ...string) {
+	desc := handle.Descriptor()
+	observable, ok := a.observableMap[desc]
+	if !ok {
+		return
+	}
+
+	ao := optionFromLabels(desc.Labels, desc.OptionalLabels, a.optionalLabels, labels...)
+
+	switch obs := observable.(type) {
+	case otelmetric.Int64ObservableGauge:
+		a.delegate.ObserveInt64(obs, val, ao)
+	default:
+	}
+}

stats/opentelemetry/server_metrics.go

@@ -0,0 +1,254 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	otelattribute "go.opentelemetry.io/otel/attribute"
+	otelmetric "go.opentelemetry.io/otel/metric"
+
+	"google.golang.org/grpc"
+	estats "google.golang.org/grpc/experimental/stats"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+type serverMetricsHandler struct {
+	estats.MetricsRecorder
+	options       Options
+	serverMetrics serverMetrics
+}
+
+func (h *serverMetricsHandler) initializeMetrics() {
+	// Will set no metrics to record, logically making this stats handler a
+	// no-op.
+	if h.options.MetricsOptions.MeterProvider == nil {
+		return
+	}
+
+	meter := h.options.MetricsOptions.MeterProvider.Meter("grpc-go", otelmetric.WithInstrumentationVersion(grpc.Version))
+	if meter == nil {
+		return
+	}
+	metrics := h.options.MetricsOptions.Metrics
+	if metrics == nil {
+		metrics = DefaultMetrics()
+	}
+
+	h.serverMetrics.callStarted = createInt64Counter(metrics.Metrics(), "grpc.server.call.started", meter, otelmetric.WithUnit("{call}"), otelmetric.WithDescription("Number of server calls started."))
+	h.serverMetrics.callSentTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.server.call.sent_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes sent per server call."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...))
+	h.serverMetrics.callRcvdTotalCompressedMessageSize = createInt64Histogram(metrics.Metrics(), "grpc.server.call.rcvd_total_compressed_message_size", meter, otelmetric.WithUnit("By"), otelmetric.WithDescription("Compressed message bytes received per server call."), otelmetric.WithExplicitBucketBoundaries(DefaultSizeBounds...))
+	h.serverMetrics.callDuration = createFloat64Histogram(metrics.Metrics(), "grpc.server.call.duration", meter, otelmetric.WithUnit("s"), otelmetric.WithDescription("End-to-end time taken to complete a call from server transport's perspective."), otelmetric.WithExplicitBucketBoundaries(DefaultLatencyBounds...))
+
+	rm := &registryMetrics{
+		optionalLabels: h.options.MetricsOptions.OptionalLabels,
+	}
+	h.MetricsRecorder = rm
+	rm.registerMetrics(metrics, meter)
+}
+
+// attachLabelsTransportStream intercepts SetHeader and SendHeader calls of the
+// underlying ServerTransportStream to attach metadataExchangeLabels.
+type attachLabelsTransportStream struct {
+	grpc.ServerTransportStream
+
+	attachedLabels         atomic.Bool
+	metadataExchangeLabels metadata.MD
+}
+
+func (s *attachLabelsTransportStream) SetHeader(md metadata.MD) error {
+	if !s.attachedLabels.Swap(true) {
+		s.ServerTransportStream.SetHeader(s.metadataExchangeLabels)
+	}
+	return s.ServerTransportStream.SetHeader(md)
+}
+
+func (s *attachLabelsTransportStream) SendHeader(md metadata.MD) error {
+	if !s.attachedLabels.Swap(true) {
+		s.ServerTransportStream.SetHeader(s.metadataExchangeLabels)
+	}
+
+	return s.ServerTransportStream.SendHeader(md)
+}
+
+func (h *serverMetricsHandler) unaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+	var metadataExchangeLabels metadata.MD
+	if h.options.MetricsOptions.pluginOption != nil {
+		metadataExchangeLabels = h.options.MetricsOptions.pluginOption.GetMetadata()
+	}
+
+	sts := grpc.ServerTransportStreamFromContext(ctx)
+
+	alts := &attachLabelsTransportStream{
+		ServerTransportStream:  sts,
+		metadataExchangeLabels: metadataExchangeLabels,
+	}
+	ctx = grpc.NewContextWithServerTransportStream(ctx, alts)
+
+	res, err := handler(ctx, req)
+	if err != nil { // maybe trailers-only if headers haven't already been sent
+		if !alts.attachedLabels.Swap(true) {
+			alts.SetTrailer(alts.metadataExchangeLabels)
+		}
+	} else { // headers will be written; a message was sent
+		if !alts.attachedLabels.Swap(true) {
+			alts.SetHeader(alts.metadataExchangeLabels)
+		}
+	}
+
+	return res, err
+}
+
+// attachLabelsStream embeds a grpc.ServerStream, and intercepts the
+// SetHeader/SendHeader/SendMsg/SendTrailer call to attach metadata exchange
+// labels.
+type attachLabelsStream struct {
+	grpc.ServerStream
+
+	attachedLabels         atomic.Bool
+	metadataExchangeLabels metadata.MD
+}
+
+func (s *attachLabelsStream) SetHeader(md metadata.MD) error {
+	if !s.attachedLabels.Swap(true) {
+		s.ServerStream.SetHeader(s.metadataExchangeLabels)
+	}
+
+	return s.ServerStream.SetHeader(md)
+}
+
+func (s *attachLabelsStream) SendHeader(md metadata.MD) error {
+	if !s.attachedLabels.Swap(true) {
+		s.ServerStream.SetHeader(s.metadataExchangeLabels)
+	}
+
+	return s.ServerStream.SendHeader(md)
+}
+
+func (s *attachLabelsStream) SendMsg(m any) error {
+	if !s.attachedLabels.Swap(true) {
+		s.ServerStream.SetHeader(s.metadataExchangeLabels)
+	}
+	return s.ServerStream.SendMsg(m)
+}
+
+func (h *serverMetricsHandler) streamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	var metadataExchangeLabels metadata.MD
+	if h.options.MetricsOptions.pluginOption != nil {
+		metadataExchangeLabels = h.options.MetricsOptions.pluginOption.GetMetadata()
+	}
+	als := &attachLabelsStream{
+		ServerStream:           ss,
+		metadataExchangeLabels: metadataExchangeLabels,
+	}
+	err := handler(srv, als)
+
+	// Add metadata exchange labels to trailers if never sent in headers,
+	// irrespective of whether or not RPC failed.
+	if !als.attachedLabels.Load() {
+		als.SetTrailer(als.metadataExchangeLabels)
+	}
+	return err
+}
+
+// TagConn exists to satisfy stats.Handler.
+func (h *serverMetricsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+// HandleConn exists to satisfy stats.Handler.
+func (h *serverMetricsHandler) HandleConn(context.Context, stats.ConnStats) {}
+
+// TagRPC implements per RPC context management for metrics.
+func (h *serverMetricsHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	ctx, _ = getOrCreateServerRPCInfo(ctx, info, h.options.MetricsOptions)
+	return ctx
+}
+
+// HandleRPC handles per RPC stats implementation.
+func (h *serverMetricsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	ri := serverRPCInfo(ctx)
+	if ri == nil {
+		logger.Error("ctx passed into server side stats handler metrics event handling has no server call data present")
+		return
+	}
+	h.processRPCData(ctx, rs, ri.ai)
+}
+
+func (h *serverMetricsHandler) processRPCData(ctx context.Context, s stats.RPCStats, ai *attemptInfo) {
+	switch st := s.(type) {
+	case *stats.InHeader:
+		if ai.pluginOptionLabels == nil && h.options.MetricsOptions.pluginOption != nil {
+			labels := h.options.MetricsOptions.pluginOption.GetLabels(st.Header)
+			if labels == nil {
+				labels = map[string]string{} // Shouldn't return a nil map. Make it empty if so to ignore future Get Calls for this Attempt.
+			}
+			ai.pluginOptionLabels = labels
+		}
+		attrs := otelmetric.WithAttributeSet(otelattribute.NewSet(
+			otelattribute.String("grpc.method", ai.method),
+		))
+		h.serverMetrics.callStarted.Add(ctx, 1, attrs)
+	case *stats.OutPayload:
+		atomic.AddInt64(&ai.sentCompressedBytes, int64(st.CompressedLength))
+	case *stats.InPayload:
+		atomic.AddInt64(&ai.recvCompressedBytes, int64(st.CompressedLength))
+	case *stats.End:
+		h.processRPCEnd(ctx, ai, st)
+	default:
+	}
+}
+
+func (h *serverMetricsHandler) processRPCEnd(ctx context.Context, ai *attemptInfo, e *stats.End) {
+	latency := float64(time.Since(ai.startTime)) / float64(time.Second)
+	st := "OK"
+	if e.Error != nil {
+		s, _ := status.FromError(e.Error)
+		st = canonicalString(s.Code())
+	}
+	attributes := []otelattribute.KeyValue{
+		otelattribute.String("grpc.method", ai.method),
+		otelattribute.String("grpc.status", st),
+	}
+	for k, v := range ai.pluginOptionLabels {
+		attributes = append(attributes, otelattribute.String(k, v))
+	}
+
+	// Allocate vararg slice once.
+	opts := []otelmetric.RecordOption{otelmetric.WithAttributeSet(otelattribute.NewSet(attributes...))}
+	h.serverMetrics.callDuration.Record(ctx, latency, opts...)
+	h.serverMetrics.callSentTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.sentCompressedBytes), opts...)
+	h.serverMetrics.callRcvdTotalCompressedMessageSize.Record(ctx, atomic.LoadInt64(&ai.recvCompressedBytes), opts...)
+}
+
+const (
+	// ServerCallStartedMetricName is the number of server calls started.
+	ServerCallStartedMetricName string = "grpc.server.call.started"
+	// ServerCallSentCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes sent per server call.
+	ServerCallSentCompressedTotalMessageSizeMetricName string = "grpc.server.call.sent_total_compressed_message_size"
+	// ServerCallRcvdCompressedTotalMessageSizeMetricName is the compressed
+	// message bytes received per server call.
+	ServerCallRcvdCompressedTotalMessageSizeMetricName string = "grpc.server.call.rcvd_total_compressed_message_size"
+	// ServerCallDurationMetricName is the end-to-end time taken to complete a
+	// call from server transport's perspective.
+	ServerCallDurationMetricName string = "grpc.server.call.duration"
+)

stats/opentelemetry/server_tracing.go

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"context"
+	"log"
+	"strings"
+
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/stats"
+	otelinternaltracing "google.golang.org/grpc/stats/opentelemetry/internal/tracing"
+)
+
+type serverTracingHandler struct {
+	options Options
+}
+
+func (h *serverTracingHandler) initializeTraces() {
+	if h.options.TraceOptions.TracerProvider == nil {
+		log.Printf("TracerProvider is not provided in server TraceOptions")
+		return
+	}
+}
+
+// TagRPC implements per RPC attempt context management for traces.
+func (h *serverTracingHandler) TagRPC(ctx context.Context, info *stats.RPCTagInfo) context.Context {
+	ctx, ri := getOrCreateServerRPCInfo(ctx, info, h.options.MetricsOptions)
+	ctx, _ = h.traceTagRPC(ctx, ri.ai)
+	return ctx
+}
+
+// traceTagRPC populates context with new span data using the TextMapPropagator
+// supplied in trace options and internal itracing.Carrier. It creates a new
+// incoming carrier which extracts an existing span context (if present) by
+// deserializing from provided context. If valid span context is extracted, it
+// is set as parent of the new span otherwise new span remains the root span.
+// If TextMapPropagator is not provided in the trace options, it returns context
+// as is.
+func (h *serverTracingHandler) traceTagRPC(ctx context.Context, ai *attemptInfo) (context.Context, *attemptInfo) {
+	mn := "Recv." + strings.Replace(ai.method, "/", ".", -1)
+	var span trace.Span
+	tracer := h.options.TraceOptions.TracerProvider.Tracer(tracerName, trace.WithInstrumentationVersion(grpc.Version))
+	ctx = h.options.TraceOptions.TextMapPropagator.Extract(ctx, otelinternaltracing.NewIncomingCarrier(ctx))
+	// If the context.Context provided in `ctx` to tracer.Start(), contains a
+	// span then the newly-created Span will be a child of that span,
+	// otherwise it will be a root span.
+	ctx, span = tracer.Start(ctx, mn, trace.WithSpanKind(trace.SpanKindServer))
+	ai.traceSpan = span
+	return ctx, ai
+}
+
+// HandleRPC handles per RPC tracing implementation.
+func (h *serverTracingHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) {
+	ri := serverRPCInfo(ctx)
+	if ri == nil {
+		logger.Error("ctx passed into server side tracing handler trace event handling has no server call data present")
+		return
+	}
+	populateSpan(rs, ri.ai)
+}
+
+// TagConn exists to satisfy stats.Handler for tracing.
+func (h *serverTracingHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context {
+	return ctx
+}
+
+// HandleConn exists to satisfy stats.Handler for tracing.
+func (h *serverTracingHandler) HandleConn(context.Context, stats.ConnStats) {}

stats/opentelemetry/trace.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opentelemetry
+
+import (
+	"go.opentelemetry.io/otel/attribute"
+	otelcodes "go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"google.golang.org/grpc/stats"
+	"google.golang.org/grpc/status"
+)
+
+// populateSpan populates span information based on stats passed in, representing
+// invariants of the RPC lifecycle. It ends the span, triggering its export.
+// This function handles attempt spans on the client-side and call spans on the
+// server-side.
+func populateSpan(rs stats.RPCStats, ai *attemptInfo) {
+	if ai == nil || ai.traceSpan == nil {
+		// Shouldn't happen, tagRPC call comes before this function gets called
+		// which populates this information.
+		logger.Error("ctx passed into stats handler tracing event handling has no traceSpan present")
+		return
+	}
+	span := ai.traceSpan
+
+	switch rs := rs.(type) {
+	case *stats.DelayedPickComplete:
+		span.AddEvent("Delayed LB pick complete")
+	case *stats.InPayload:
+		// message id - "must be calculated as two different counters starting
+		// from one for sent messages and one for received messages."
+		attrs := []attribute.KeyValue{
+			attribute.Int64("sequence-number", int64(ai.countRecvMsg)),
+			attribute.Int64("message-size", int64(rs.Length)),
+		}
+		if rs.CompressedLength != rs.Length {
+			attrs = append(attrs, attribute.Int64("message-size-compressed", int64(rs.CompressedLength)))
+		}
+		span.AddEvent("Inbound message", trace.WithAttributes(attrs...))
+		ai.countRecvMsg++
+	case *stats.OutPayload:
+		attrs := []attribute.KeyValue{
+			attribute.Int64("sequence-number", int64(ai.countSentMsg)),
+			attribute.Int64("message-size", int64(rs.Length)),
+		}
+		if rs.CompressedLength != rs.Length {
+			attrs = append(attrs, attribute.Int64("message-size-compressed", int64(rs.CompressedLength)))
+		}
+		span.AddEvent("Outbound message", trace.WithAttributes(attrs...))
+		ai.countSentMsg++
+	case *stats.End:
+		if rs.Error != nil {
+			s := status.Convert(rs.Error)
+			span.SetStatus(otelcodes.Error, s.Message())
+		} else {
+			span.SetStatus(otelcodes.Ok, "Ok")
+		}
+		span.End()
+	}
+}