snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

stats/opentelemetry/csm/observability.go

@@ -0,0 +1,84 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"net/url"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/internal"
+	"google.golang.org/grpc/stats/opentelemetry"
+	otelinternal "google.golang.org/grpc/stats/opentelemetry/internal"
+)
+
+// EnableObservability sets up CSM Observability for the binary globally.
+//
+// The CSM Stats Plugin is instantiated with local labels and metadata exchange
+// labels pulled from the environment, and emits metadata exchange labels from
+// the peer and local labels. Context timeouts do not trigger an error, but set
+// certain labels to "unknown".
+//
+// This function is not thread safe, and should only be invoked once in main
+// before any channels or servers are created. Returns a cleanup function to be
+// deferred in main.
+func EnableObservability(ctx context.Context, options opentelemetry.Options) func() {
+	csmPluginOption := newPluginOption(ctx)
+	clientSideOTelWithCSM := dialOptionWithCSMPluginOption(options, csmPluginOption)
+	clientSideOTel := opentelemetry.DialOption(options)
+	internal.AddGlobalPerTargetDialOptions.(func(opt any))(&perTargetDialOption{
+		clientSideOTelWithCSM: clientSideOTelWithCSM,
+		clientSideOTel:        clientSideOTel,
+	})
+
+	serverSideOTelWithCSM := serverOptionWithCSMPluginOption(options, csmPluginOption)
+	internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(serverSideOTelWithCSM)
+
+	return func() {
+		internal.ClearGlobalServerOptions()
+		internal.ClearGlobalPerTargetDialOptions()
+	}
+}
+
+type perTargetDialOption struct {
+	clientSideOTelWithCSM grpc.DialOption
+	clientSideOTel        grpc.DialOption
+}
+
+func (o *perTargetDialOption) DialOptionForTarget(parsedTarget url.URL) grpc.DialOption {
+	if determineTargetCSM(&parsedTarget) {
+		return o.clientSideOTelWithCSM
+	}
+	return o.clientSideOTel
+}
+
+func dialOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption {
+	options.MetricsOptions.OptionalLabels = []string{"csm.service_name", "csm.service_namespace_name"} // Attach the two xDS Optional Labels for this component to not filter out.
+	return dialOptionSetCSM(options, po)
+}
+
+func dialOptionSetCSM(options opentelemetry.Options, po otelinternal.PluginOption) grpc.DialOption {
+	otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po)
+	return opentelemetry.DialOption(options)
+}
+
+func serverOptionWithCSMPluginOption(options opentelemetry.Options, po otelinternal.PluginOption) grpc.ServerOption {
+	otelinternal.SetPluginOption.(func(options *opentelemetry.Options, po otelinternal.PluginOption))(&options, po)
+	return opentelemetry.ServerOption(options)
+}

stats/opentelemetry/csm/observability_test.go

@@ -0,0 +1,649 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/base"
+	"google.golang.org/grpc/encoding/gzip"
+	estats "google.golang.org/grpc/experimental/stats"
+	istats "google.golang.org/grpc/internal/stats"
+	"google.golang.org/grpc/internal/stubserver"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats/opentelemetry"
+	itestutils "google.golang.org/grpc/stats/opentelemetry/internal/testutils"
+)
+
+// setupEnv configures the environment for CSM Observability Testing. It sets
+// the bootstrap env var to a bootstrap file with a nodeID provided. It sets CSM
+// Env Vars as well, and mocks the resource detector's returned attribute set to
+// simulate the environment. It registers a cleanup function on the provided t
+// to restore the environment to its original state.
+func setupEnv(t *testing.T, resourceDetectorEmissions map[string]string, meshID, csmCanonicalServiceName, csmWorkloadName string) {
+	oldCSMMeshID, csmMeshIDPresent := os.LookupEnv("CSM_MESH_ID")
+	oldCSMCanonicalServiceName, csmCanonicalServiceNamePresent := os.LookupEnv("CSM_CANONICAL_SERVICE_NAME")
+	oldCSMWorkloadName, csmWorkloadNamePresent := os.LookupEnv("CSM_WORKLOAD_NAME")
+	os.Setenv("CSM_MESH_ID", meshID)
+	os.Setenv("CSM_CANONICAL_SERVICE_NAME", csmCanonicalServiceName)
+	os.Setenv("CSM_WORKLOAD_NAME", csmWorkloadName)
+
+	var attributes []attribute.KeyValue
+	for k, v := range resourceDetectorEmissions {
+		attributes = append(attributes, attribute.String(k, v))
+	}
+	// Return the attributes configured as part of the test in place
+	// of reading from resource.
+	attrSet := attribute.NewSet(attributes...)
+	origGetAttrSet := getAttrSetFromResourceDetector
+	getAttrSetFromResourceDetector = func(context.Context) *attribute.Set {
+		return &attrSet
+	}
+	t.Cleanup(func() {
+		if csmMeshIDPresent {
+			os.Setenv("CSM_MESH_ID", oldCSMMeshID)
+		} else {
+			os.Unsetenv("CSM_MESH_ID")
+		}
+		if csmCanonicalServiceNamePresent {
+			os.Setenv("CSM_CANONICAL_SERVICE_NAME", oldCSMCanonicalServiceName)
+		} else {
+			os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+		}
+		if csmWorkloadNamePresent {
+			os.Setenv("CSM_WORKLOAD_NAME", oldCSMWorkloadName)
+		} else {
+			os.Unsetenv("CSM_WORKLOAD_NAME")
+		}
+
+		getAttrSetFromResourceDetector = origGetAttrSet
+	})
+}
+
+// TestCSMPluginOptionUnary tests the CSM Plugin Option and labels. It
+// configures the environment for the CSM Plugin Option to read from. It then
+// configures a system with a gRPC Client and gRPC server with the OpenTelemetry
+// Dial and Server Option configured with a CSM Plugin Option with a certain
+// unary handler set to induce different ways of setting metadata exchange
+// labels, and makes a Unary RPC. This RPC should cause certain recording for
+// each registered metric observed through a Manual Metrics Reader on the
+// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the
+// plugin option should be attached to the relevant metrics.
+func (s) TestCSMPluginOptionUnary(t *testing.T) {
+	resourceDetectorEmissions := map[string]string{
+		"cloud.platform":     "gcp_kubernetes_engine",
+		"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+		"cloud.account.id":   "cloud_account_id_val",
+		"k8s.namespace.name": "k8s_namespace_name_val",
+		"k8s.cluster.name":   "k8s_cluster_name_val",
+	}
+	const meshID = "mesh_id"
+	const csmCanonicalServiceName = "csm_canonical_service_name"
+	const csmWorkloadName = "csm_workload_name"
+	setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName)
+
+	attributesWant := map[string]string{
+		"csm.workload_canonical_service": csmCanonicalServiceName, // from env
+		"csm.mesh_id":                    "mesh_id",               // from bootstrap env var
+
+		// No xDS Labels - this happens in a test below.
+
+		"csm.remote_workload_type":              "gcp_kubernetes_engine",
+		"csm.remote_workload_canonical_service": csmCanonicalServiceName,
+		"csm.remote_workload_project_id":        "cloud_account_id_val",
+		"csm.remote_workload_cluster_name":      "k8s_cluster_name_val",
+		"csm.remote_workload_namespace_name":    "k8s_namespace_name_val",
+		"csm.remote_workload_location":          "cloud_region_val",
+		"csm.remote_workload_name":              csmWorkloadName,
+	}
+
+	var csmLabels []attribute.KeyValue
+	for k, v := range attributesWant {
+		csmLabels = append(csmLabels, attribute.String(k, v))
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	unaryCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}))
+	tests := []struct {
+		name string
+		// To test the different operations for Unary RPC's from the interceptor
+		// level that can plumb metadata exchange header in.
+		unaryCallFunc func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error)
+		opts          itestutils.MetricDataOptions
+	}{
+		{
+			name: "normal-flow",
+			unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "trailers-only",
+			unaryCallFunc: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return nil, errors.New("some error") // return an error and no message - this triggers trailers only - no messages or headers sent
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:       csmLabels,
+				UnaryCallFailed: true,
+			},
+		},
+		{
+			name: "set-header",
+			unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				grpc.SetHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "send-header",
+			unaryCallFunc: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				grpc.SendHeader(ctx, metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+		{
+			name: "send-msg",
+			unaryCallFunc: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+				return &testpb.SimpleResponse{Payload: &testpb.Payload{
+					Body: make([]byte, len(in.GetPayload().GetBody())),
+				}}, nil
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                  csmLabels,
+				UnaryCompressedMessageSize: unaryCompressedSize,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			reader := metric.NewManualReader()
+			provider := metric.NewMeterProvider(metric.WithReader(reader))
+			ss := &stubserver.StubServer{UnaryCallF: test.unaryCallFunc}
+			po := newPluginOption(ctx)
+			sopts := []grpc.ServerOption{
+				serverOptionWithCSMPluginOption(opentelemetry.Options{
+					MetricsOptions: opentelemetry.MetricsOptions{
+						MeterProvider: provider,
+						Metrics:       opentelemetry.DefaultMetrics(),
+					}}, po),
+			}
+			dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{
+				MetricsOptions: opentelemetry.MetricsOptions{
+					MeterProvider:  provider,
+					Metrics:        opentelemetry.DefaultMetrics(),
+					OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"},
+				},
+			}, po)}
+			if err := ss.Start(sopts, dopts...); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			var request *testpb.SimpleRequest
+			if test.opts.UnaryCompressedMessageSize != 0 {
+				request = &testpb.SimpleRequest{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}}
+			}
+			// Make a Unary RPC. These should cause certain metrics to be
+			// emitted, which should be able to be observed through the Metric
+			// Reader.
+			ss.Client.UnaryCall(ctx, request, grpc.UseCompressor(gzip.Name))
+			rm := &metricdata.ResourceMetrics{}
+			reader.Collect(ctx, rm)
+
+			gotMetrics := map[string]metricdata.Metrics{}
+			for _, sm := range rm.ScopeMetrics {
+				for _, m := range sm.Metrics {
+					gotMetrics[m.Name] = m
+				}
+			}
+
+			opts := test.opts
+			opts.Target = ss.Target
+			wantMetrics := itestutils.MetricDataUnary(opts)
+			gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+			itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+		})
+	}
+}
+
+// TestCSMPluginOptionStreaming tests the CSM Plugin Option and labels. It
+// configures the environment for the CSM Plugin Option to read from. It then
+// configures a system with a gRPC Client and gRPC server with the OpenTelemetry
+// Dial and Server Option configured with a CSM Plugin Option with a certain
+// streaming handler set to induce different ways of setting metadata exchange
+// labels, and makes a Streaming RPC. This RPC should cause certain recording
+// for each registered metric observed through a Manual Metrics Reader on the
+// provided OpenTelemetry SDK's Meter Provider. The CSM Labels emitted from the
+// plugin option should be attached to the relevant metrics.
+func (s) TestCSMPluginOptionStreaming(t *testing.T) {
+	resourceDetectorEmissions := map[string]string{
+		"cloud.platform":     "gcp_kubernetes_engine",
+		"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+		"cloud.account.id":   "cloud_account_id_val",
+		"k8s.namespace.name": "k8s_namespace_name_val",
+		"k8s.cluster.name":   "k8s_cluster_name_val",
+	}
+	const meshID = "mesh_id"
+	const csmCanonicalServiceName = "csm_canonical_service_name"
+	const csmWorkloadName = "csm_workload_name"
+	setupEnv(t, resourceDetectorEmissions, meshID, csmCanonicalServiceName, csmWorkloadName)
+
+	attributesWant := map[string]string{
+		"csm.workload_canonical_service": csmCanonicalServiceName, // from env
+		"csm.mesh_id":                    "mesh_id",               // from bootstrap env var
+
+		// No xDS Labels - this happens in a test below.
+
+		"csm.remote_workload_type":              "gcp_kubernetes_engine",
+		"csm.remote_workload_canonical_service": csmCanonicalServiceName,
+		"csm.remote_workload_project_id":        "cloud_account_id_val",
+		"csm.remote_workload_cluster_name":      "k8s_cluster_name_val",
+		"csm.remote_workload_namespace_name":    "k8s_namespace_name_val",
+		"csm.remote_workload_location":          "cloud_region_val",
+		"csm.remote_workload_name":              csmWorkloadName,
+	}
+
+	var csmLabels []attribute.KeyValue
+	for k, v := range attributesWant {
+		csmLabels = append(csmLabels, attribute.String(k, v))
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	streamingCompressedSize := float64(itestutils.GzipCompressedMessageSize(t, &testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: make([]byte, 10000)}}))
+	tests := []struct {
+		name string
+		// To test the different operations for Streaming RPC's from the
+		// interceptor level that can plumb metadata exchange header in.
+		streamingCallFunc func(stream testgrpc.TestService_FullDuplexCallServer) error
+		opts              itestutils.MetricDataOptions
+	}{
+		{
+			name: "trailers-only",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "set-header",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.SetHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "send-header",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.SendHeader(metadata.New(map[string]string{"some-metadata": "some-metadata-val"}))
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels: csmLabels,
+			},
+		},
+		{
+			name: "send-msg",
+			streamingCallFunc: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+				stream.Send(&testpb.StreamingOutputCallResponse{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}})
+				for {
+					if _, err := stream.Recv(); err == io.EOF {
+						return nil
+					}
+				}
+			},
+			opts: itestutils.MetricDataOptions{
+				CSMLabels:                      csmLabels,
+				StreamingCompressedMessageSize: streamingCompressedSize,
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			reader := metric.NewManualReader()
+			provider := metric.NewMeterProvider(metric.WithReader(reader))
+			ss := &stubserver.StubServer{FullDuplexCallF: test.streamingCallFunc}
+			po := newPluginOption(ctx)
+			sopts := []grpc.ServerOption{
+				serverOptionWithCSMPluginOption(opentelemetry.Options{
+					MetricsOptions: opentelemetry.MetricsOptions{
+						MeterProvider: provider,
+						Metrics:       opentelemetry.DefaultMetrics(),
+					}}, po),
+			}
+			dopts := []grpc.DialOption{dialOptionWithCSMPluginOption(opentelemetry.Options{
+				MetricsOptions: opentelemetry.MetricsOptions{
+					MeterProvider:  provider,
+					Metrics:        opentelemetry.DefaultMetrics(),
+					OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name"},
+				},
+			}, po)}
+			if err := ss.Start(sopts, dopts...); err != nil {
+				t.Fatalf("Error starting endpoint server: %v", err)
+			}
+			defer ss.Stop()
+
+			stream, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name))
+			if err != nil {
+				t.Fatalf("ss.Client.FullDuplexCall failed: %f", err)
+			}
+
+			if test.opts.StreamingCompressedMessageSize != 0 {
+				if err := stream.Send(&testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{
+					Body: make([]byte, 10000),
+				}}); err != nil {
+					t.Fatalf("stream.Send failed")
+				}
+				if _, err := stream.Recv(); err != nil {
+					t.Fatalf("stream.Recv failed with error: %v", err)
+				}
+			}
+
+			stream.CloseSend()
+			if _, err = stream.Recv(); err != io.EOF {
+				t.Fatalf("stream.Recv received an unexpected error: %v, expected an EOF error", err)
+			}
+
+			rm := &metricdata.ResourceMetrics{}
+			reader.Collect(ctx, rm)
+
+			gotMetrics := map[string]metricdata.Metrics{}
+			for _, sm := range rm.ScopeMetrics {
+				for _, m := range sm.Metrics {
+					gotMetrics[m.Name] = m
+				}
+			}
+
+			opts := test.opts
+			opts.Target = ss.Target
+			wantMetrics := itestutils.MetricDataStreaming(opts)
+			gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+			itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+		})
+	}
+}
+
+// TestXDSLabels tests that xDS Labels get emitted from OpenTelemetry metrics.
+// This test configures OpenTelemetry with the CSM Plugin Option, and xDS
+// Optional Labels turned on. It then configures a test balancer that updates
+// labels, simulating the cluster_impl picker. It then makes a unary RPC, and
+// expects xDS Labels labels to be attached to emitted relevant metrics. Full
+// xDS System alongside OpenTelemetry will be tested with interop. (there is a
+// test for xDS -> Stats handler and this tests -> OTel -> emission). It also
+// tests the optional per call locality label in the same manner.
+func (s) TestXDSLabels(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	balancerName := "o11y_balancer_xds_labels"
+	balancer.Register(base.NewBalancerBuilder(balancerName, &o11yPickerBuilder{}, base.Config{}))
+	reader := metric.NewManualReader()
+	provider := metric.NewMeterProvider(metric.WithReader(reader))
+	ss := &stubserver.StubServer{
+		UnaryCallF: func(_ context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			return &testpb.SimpleResponse{Payload: &testpb.Payload{
+				Body: make([]byte, len(in.GetPayload().GetBody())),
+			}}, nil
+		},
+	}
+
+	po := newPluginOption(ctx)
+	sc := fmt.Sprintf(`{"loadBalancingConfig": [{"%s": {}}]}`, balancerName)
+	dopts := []grpc.DialOption{dialOptionSetCSM(opentelemetry.Options{
+		MetricsOptions: opentelemetry.MetricsOptions{
+			MeterProvider:  provider,
+			Metrics:        opentelemetry.DefaultMetrics(),
+			OptionalLabels: []string{"csm.service_name", "csm.service_namespace_name", "grpc.lb.locality", "grpc.lb.backend_service", "grpc.client.call.custom"},
+		},
+	}, po), grpc.WithDefaultServiceConfig(sc)}
+	if err := ss.Start(nil, dopts...); err != nil {
+		t.Fatalf("Error starting endpoint server: %v", err)
+	}
+
+	defer ss.Stop()
+	ss.Client.UnaryCall(estats.NewContextWithCustomLabel(ctx, "my-custom-label"), &testpb.SimpleRequest{Payload: &testpb.Payload{
+		Body: make([]byte, 10000),
+	}}, grpc.UseCompressor(gzip.Name))
+
+	rm := &metricdata.ResourceMetrics{}
+	reader.Collect(ctx, rm)
+
+	gotMetrics := map[string]metricdata.Metrics{}
+	for _, sm := range rm.ScopeMetrics {
+		for _, m := range sm.Metrics {
+			gotMetrics[m.Name] = m
+		}
+	}
+
+	unaryMethodAttr := attribute.String("grpc.method", "grpc.testing.TestService/UnaryCall")
+	targetAttr := attribute.String("grpc.target", ss.Target)
+	unaryStatusAttr := attribute.String("grpc.status", "OK")
+
+	serviceNameAttr := attribute.String("csm.service_name", "service_name_val")
+	serviceNamespaceAttr := attribute.String("csm.service_namespace_name", "service_namespace_val")
+	localityAttr := attribute.String("grpc.lb.locality", "grpc.lb.locality_val")
+	backendServiceAttr := attribute.String("grpc.lb.backend_service", "grpc.lb.backend_service_val")
+	meshIDAttr := attribute.String("csm.mesh_id", "unknown")
+	workloadCanonicalServiceAttr := attribute.String("csm.workload_canonical_service", "unknown")
+	remoteWorkloadTypeAttr := attribute.String("csm.remote_workload_type", "unknown")
+	remoteWorkloadCanonicalServiceAttr := attribute.String("csm.remote_workload_canonical_service", "unknown")
+	customLabelAttr := attribute.String("grpc.client.call.custom", "my-custom-label")
+
+	unaryMethodClientSideEnd := []attribute.KeyValue{
+		unaryMethodAttr,
+		targetAttr,
+		unaryStatusAttr,
+		serviceNameAttr,
+		serviceNamespaceAttr,
+		localityAttr,
+		backendServiceAttr,
+		meshIDAttr,
+		workloadCanonicalServiceAttr,
+		remoteWorkloadTypeAttr,
+		remoteWorkloadCanonicalServiceAttr,
+		customLabelAttr,
+	}
+
+	unaryCompressedBytesSentRecv := int64(itestutils.GzipCompressedMessageSize(t, &testpb.SimpleRequest{
+		Payload: &testpb.Payload{
+			Body: make([]byte, 10000),
+		},
+	})) // Fixed 10000 bytes with gzip.
+	unaryBucketCounts := []uint64{0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}
+	unaryExtrema := metricdata.NewExtrema(unaryCompressedBytesSentRecv)
+	wantMetrics := []metricdata.Metrics{
+		{
+			Name:        "grpc.client.attempt.started",
+			Description: "Number of client call attempts started.",
+			Unit:        "{attempt}",
+			Data: metricdata.Sum[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, customLabelAttr),
+						Value:      1,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+			},
+		}, // Doesn't have xDS Labels, CSM Labels start from header or trailer from server, whichever comes first, so this doesn't need it
+		{
+			Name:        "grpc.client.attempt.duration",
+			Description: "End-to-end time taken to complete a client call attempt.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:      1,
+						Bounds:     itestutils.DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.sent_total_compressed_message_size",
+			Description: "Compressed message bytes sent per client call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       itestutils.DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.attempt.rcvd_total_compressed_message_size",
+			Description: "Compressed message bytes received per call attempt.",
+			Unit:        "By",
+			Data: metricdata.Histogram[int64]{
+				DataPoints: []metricdata.HistogramDataPoint[int64]{
+					{
+						Attributes:   attribute.NewSet(unaryMethodClientSideEnd...),
+						Count:        1,
+						Bounds:       itestutils.DefaultSizeBounds,
+						BucketCounts: unaryBucketCounts,
+						Min:          unaryExtrema,
+						Max:          unaryExtrema,
+						Sum:          unaryCompressedBytesSentRecv,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+		{
+			Name:        "grpc.client.call.duration",
+			Description: "Time taken by gRPC to complete an RPC from application's perspective.",
+			Unit:        "s",
+			Data: metricdata.Histogram[float64]{
+				DataPoints: []metricdata.HistogramDataPoint[float64]{
+					{
+						Attributes: attribute.NewSet(unaryMethodAttr, targetAttr, unaryStatusAttr, customLabelAttr),
+						Count:      1,
+						Bounds:     itestutils.DefaultLatencyBounds,
+					},
+				},
+				Temporality: metricdata.CumulativeTemporality,
+			},
+		},
+	}
+
+	gotMetrics = itestutils.WaitForServerMetrics(ctx, t, reader, gotMetrics, wantMetrics)
+	itestutils.CompareMetrics(t, gotMetrics, wantMetrics)
+}
+
+// TestObservability tests that Observability global function compiles and runs
+// without error. The actual functionality of this function will be verified in
+// interop tests.
+func (s) TestObservability(*testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	cleanup := EnableObservability(ctx, opentelemetry.Options{})
+	cleanup()
+}
+
+type o11yPickerBuilder struct{}
+
+func (*o11yPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker {
+	return &o11yPicker{info: info}
+}
+
+type o11yPicker struct {
+	info base.PickerBuildInfo
+}
+
+func (p *o11yPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) {
+	// This simulates the behavior of the cluster_impl picker.
+	// It runs after TagRPC has already registered the default telemetry
+	// callback.
+	istats.UpdateLabels(info.Ctx, map[string]string{
+		"csm.service_name":           "service_name_val",
+		"csm.service_namespace_name": "service_namespace_val",
+
+		"grpc.lb.locality":        "grpc.lb.locality_val",
+		"grpc.lb.backend_service": "grpc.lb.backend_service_val",
+	})
+
+	for sc := range p.info.ReadySCs {
+		return balancer.PickResult{SubConn: sc}, nil
+	}
+	return balancer.PickResult{}, balancer.ErrNoSubConnAvailable
+}

stats/opentelemetry/csm/pluginoption.go

@@ -0,0 +1,279 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package csm contains utilities for Google Cloud Service Mesh observability.
+package csm
+
+import (
+	"context"
+	"encoding/base64"
+	"net/url"
+	"os"
+
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/stats/opentelemetry/internal"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+
+	"go.opentelemetry.io/contrib/detectors/gcp"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+)
+
+var logger = grpclog.Component("csm-observability-plugin")
+
+// pluginOption emits CSM Labels from the environment and metadata exchange
+// for csm channels and all servers.
+//
+// Do not use this directly; use newPluginOption instead.
+type pluginOption struct {
+	// localLabels are the labels that identify the local environment a binary
+	// is run in, and will be emitted from the CSM Plugin Option.
+	localLabels map[string]string
+	// metadataExchangeLabelsEncoded are the metadata exchange labels to be sent
+	// as the value of metadata key "x-envoy-peer-metadata" in proto wire format
+	// and base 64 encoded. This gets sent out from all the servers running in
+	// this process and for csm channels.
+	metadataExchangeLabelsEncoded string
+}
+
+// newPluginOption returns a new pluginOption with local labels and metadata
+// exchange labels derived from the environment.
+func newPluginOption(ctx context.Context) internal.PluginOption {
+	localLabels, metadataExchangeLabelsEncoded := constructMetadataFromEnv(ctx)
+
+	return &pluginOption{
+		localLabels:                   localLabels,
+		metadataExchangeLabelsEncoded: metadataExchangeLabelsEncoded,
+	}
+}
+
+// NewLabelsMD returns a metadata.MD with the CSM labels as an encoded protobuf
+// Struct as the value of "x-envoy-peer-metadata".
+func (cpo *pluginOption) GetMetadata() metadata.MD {
+	return metadata.New(map[string]string{
+		metadataExchangeKey: cpo.metadataExchangeLabelsEncoded,
+	})
+}
+
+// GetLabels gets the CSM peer labels from the metadata provided. It returns
+// "unknown" for labels not found. Labels returned depend on the remote type.
+// Additionally, local labels determined at initialization time are appended to
+// labels returned, in addition to the optionalLabels provided.
+func (cpo *pluginOption) GetLabels(md metadata.MD) map[string]string {
+	labels := map[string]string{ // Remote labels if type is unknown (i.e. unset or error processing x-envoy-peer-metadata)
+		"csm.remote_workload_type":              "unknown",
+		"csm.remote_workload_canonical_service": "unknown",
+	}
+	// Append the local labels.
+	for k, v := range cpo.localLabels {
+		labels[k] = v
+	}
+
+	val := md.Get("x-envoy-peer-metadata")
+	// This can't happen if corresponding csm client because of proto wire
+	// format encoding, but since it is arbitrary off the wire be safe.
+	if len(val) != 1 {
+		logger.Warningf("length of md values of \"x-envoy-peer-metadata\" is not 1, is %v", len(val))
+		return labels
+	}
+
+	protoWireFormat, err := base64.RawStdEncoding.DecodeString(val[0])
+	if err != nil {
+		logger.Warningf("error base 64 decoding value of \"x-envoy-peer-metadata\": %v", err)
+		return labels
+	}
+
+	spb := &structpb.Struct{}
+	if err := proto.Unmarshal(protoWireFormat, spb); err != nil {
+		logger.Warningf("error unmarshalling value of \"x-envoy-peer-metadata\" into proto: %v", err)
+		return labels
+	}
+
+	fields := spb.GetFields()
+
+	labels["csm.remote_workload_type"] = getFromMetadata("type", fields)
+	// The value of “csm.remote_workload_canonical_service” comes from
+	// MetadataExchange with the key “canonical_service”. (Note that this should
+	// be read even if the remote type is unknown.)
+	labels["csm.remote_workload_canonical_service"] = getFromMetadata("canonical_service", fields)
+
+	// Unset/unknown types, and types that aren't GKE or GCP return early with
+	// just local labels, remote_workload_type and
+	// remote_workload_canonical_service labels.
+	workloadType := labels["csm.remote_workload_type"]
+	if workloadType != "gcp_kubernetes_engine" && workloadType != "gcp_compute_engine" {
+		return labels
+	}
+	// GKE and GCE labels.
+	labels["csm.remote_workload_project_id"] = getFromMetadata("project_id", fields)
+	labels["csm.remote_workload_location"] = getFromMetadata("location", fields)
+	labels["csm.remote_workload_name"] = getFromMetadata("workload_name", fields)
+	if workloadType == "gcp_compute_engine" {
+		return labels
+	}
+
+	// GKE only labels.
+	labels["csm.remote_workload_cluster_name"] = getFromMetadata("cluster_name", fields)
+	labels["csm.remote_workload_namespace_name"] = getFromMetadata("namespace_name", fields)
+	return labels
+}
+
+// getFromMetadata gets the value for the metadata key from the protobuf
+// metadata. Returns "unknown" if the metadata is not found in the protobuf
+// metadata, or if the value is not a string value. Returns the string value
+// otherwise.
+func getFromMetadata(metadataKey string, metadata map[string]*structpb.Value) string {
+	if metadata != nil {
+		if metadataVal, ok := metadata[metadataKey]; ok {
+			if _, ok := metadataVal.GetKind().(*structpb.Value_StringValue); ok {
+				return metadataVal.GetStringValue()
+			}
+		}
+	}
+	return "unknown"
+}
+
+// getFromResource gets the value for the resource key from the attribute set.
+// Returns "unknown" if the resourceKey is not found in the attribute set or is
+// not a string value, the string value otherwise.
+func getFromResource(resourceKey attribute.Key, set *attribute.Set) string {
+	if set != nil {
+		if resourceVal, ok := set.Value(resourceKey); ok && resourceVal.Type() == attribute.STRING {
+			return resourceVal.AsString()
+		}
+	}
+	return "unknown"
+}
+
+// getEnv returns "unknown" if environment variable is unset, the environment
+// variable otherwise.
+func getEnv(name string) string {
+	if val, ok := os.LookupEnv(name); ok {
+		return val
+	}
+	return "unknown"
+}
+
+var (
+	// This function will be overridden in unit tests.
+	getAttrSetFromResourceDetector = func(ctx context.Context) *attribute.Set {
+		r, err := resource.New(ctx, resource.WithFromEnv(), resource.WithDetectors(gcp.NewDetector()))
+		if err != nil {
+			logger.Warningf("error reading OpenTelemetry resource: %v", err)
+		}
+		if r != nil {
+			// It's possible for resource.New to return partial data alongside
+			// an error. In this case, use partial data and set "unknown" for
+			// labels missing.
+			return r.Set()
+		}
+		return nil
+	}
+)
+
+// constructMetadataFromEnv creates local labels and labels to send to the peer
+// using metadata exchange based off resource detection and environment
+// variables.
+//
+// Returns local labels, and base 64 encoded protobuf.Struct containing metadata
+// exchange labels.
+func constructMetadataFromEnv(ctx context.Context) (map[string]string, string) {
+	set := getAttrSetFromResourceDetector(ctx)
+
+	labels := make(map[string]string)
+	labels["type"] = getFromResource("cloud.platform", set)
+	labels["canonical_service"] = getEnv("CSM_CANONICAL_SERVICE_NAME")
+
+	// If type is not GCE or GKE only metadata exchange labels are "type" and
+	// "canonical_service".
+	cloudPlatformVal := labels["type"]
+	if cloudPlatformVal != "gcp_kubernetes_engine" && cloudPlatformVal != "gcp_compute_engine" {
+		return initializeLocalAndMetadataLabels(labels)
+	}
+
+	// GCE and GKE labels:
+	labels["workload_name"] = getEnv("CSM_WORKLOAD_NAME")
+
+	locationVal := "unknown"
+	if resourceVal, ok := set.Value("cloud.availability_zone"); ok && resourceVal.Type() == attribute.STRING {
+		locationVal = resourceVal.AsString()
+	} else if resourceVal, ok = set.Value("cloud.region"); ok && resourceVal.Type() == attribute.STRING {
+		locationVal = resourceVal.AsString()
+	}
+	labels["location"] = locationVal
+
+	labels["project_id"] = getFromResource("cloud.account.id", set)
+	if cloudPlatformVal == "gcp_compute_engine" {
+		return initializeLocalAndMetadataLabels(labels)
+	}
+
+	// GKE specific labels:
+	labels["namespace_name"] = getFromResource("k8s.namespace.name", set)
+	labels["cluster_name"] = getFromResource("k8s.cluster.name", set)
+	return initializeLocalAndMetadataLabels(labels)
+}
+
+// initializeLocalAndMetadataLabels csm local labels for a CSM Plugin Option to
+// record. It also builds out a base 64 encoded protobuf.Struct containing the
+// metadata exchange labels to be sent as part of metadata exchange from a CSM
+// Plugin Option.
+func initializeLocalAndMetadataLabels(labels map[string]string) (map[string]string, string) {
+	// The value of “csm.workload_canonical_service” comes from
+	// “CSM_CANONICAL_SERVICE_NAME” env var, “unknown” if unset.
+	val := labels["canonical_service"]
+	localLabels := make(map[string]string)
+	localLabels["csm.workload_canonical_service"] = val
+	localLabels["csm.mesh_id"] = getEnv("CSM_MESH_ID")
+
+	// Metadata exchange labels - can go ahead and encode into proto, and then
+	// base64.
+	pbLabels := &structpb.Struct{
+		Fields: map[string]*structpb.Value{},
+	}
+	for k, v := range labels {
+		pbLabels.Fields[k] = structpb.NewStringValue(v)
+	}
+	protoWireFormat, err := proto.Marshal(pbLabels)
+	metadataExchangeLabelsEncoded := ""
+	if err == nil {
+		metadataExchangeLabelsEncoded = base64.RawStdEncoding.EncodeToString(protoWireFormat)
+	}
+	// else - This behavior triggers server side to reply (if sent from a gRPC
+	// Client within this binary) with the metadata exchange labels. Even if
+	// client side has a problem marshaling proto into wire format, it can
+	// still use server labels so send an empty string as the value of
+	// x-envoy-peer-metadata. The presence of this metadata exchange header
+	// will cause server side to respond with metadata exchange labels.
+
+	return localLabels, metadataExchangeLabelsEncoded
+}
+
+// metadataExchangeKey is the key for HTTP metadata exchange.
+const metadataExchangeKey = "x-envoy-peer-metadata"
+
+func determineTargetCSM(parsedTarget *url.URL) bool {
+	// On the client-side, the channel target is used to determine if a channel is a
+	// CSM channel or not. CSM channels need to have an “xds” scheme and a
+	// "traffic-director-global.xds.googleapis.com" authority. In the cases where no
+	// authority is mentioned, the authority is assumed to be CSM. MetadataExchange
+	// is performed only for CSM channels. Non-metadata exchange labels are detected
+	// as described below.
+	return parsedTarget.Scheme == "xds" && (parsedTarget.Host == "" || parsedTarget.Host == "traffic-director-global.xds.googleapis.com")
+}

stats/opentelemetry/csm/pluginoption_test.go

@@ -0,0 +1,490 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package csm
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/url"
+	"os"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/envconfig"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/metadata"
+
+	"github.com/google/go-cmp/cmp"
+	"go.opentelemetry.io/otel/attribute"
+
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+var defaultTestTimeout = 5 * time.Second
+
+// clearEnv unsets all the environment variables relevant to the csm
+// pluginOption.
+func clearEnv() {
+	os.Unsetenv(envconfig.XDSBootstrapFileContentEnv)
+	os.Unsetenv(envconfig.XDSBootstrapFileNameEnv)
+
+	os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+	os.Unsetenv("CSM_WORKLOAD_NAME")
+}
+
+func (s) TestGetLabels(t *testing.T) {
+	clearEnv()
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	cpo := newPluginOption(ctx)
+
+	tests := []struct {
+		name                   string
+		unsetHeader            bool // Should trigger "unknown" labels
+		twoValues              bool // Should trigger "unknown" labels
+		metadataExchangeLabels map[string]string
+		labelsWant             map[string]string
+	}{
+		{
+			name:                   "unset-labels",
+			metadataExchangeLabels: nil,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "metadata-partially-set",
+			metadataExchangeLabels: map[string]string{
+				"type":        "not-gce-or-gke",
+				"ignore-this": "ignore-this",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "not-gce-or-gke",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "google-compute-engine",
+			metadataExchangeLabels: map[string]string{ // All of these labels get emitted when type is "gcp_compute_engine".
+				"type":              "gcp_compute_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_compute_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "workload_name_val",
+			},
+		},
+		// unset should go to unknown, ignore GKE labels that are not relevant
+		// to GCE.
+		{
+			name: "google-compute-engine-labels-partially-set-with-extra",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"location":          "us-east",
+				// "workload_name": "", unset workload name - should become "unknown"
+				"namespace_name": "should-be-ignored",
+				"cluster_name":   "should-be-ignored",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_compute_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "unknown",
+			},
+		},
+		{
+			name: "google-kubernetes-engine",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_kubernetes_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_cluster_name":      "cluster_name_val",
+				"csm.remote_workload_namespace_name":    "namespace_name_val",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "workload_name_val",
+			},
+		},
+		{
+			name: "google-kubernetes-engine-labels-partially-set",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				// "cluster_name": "", cluster_name unset, should become "unknown"
+				"location": "us-east",
+				// "workload_name": "", workload_name unset, should become "unknown"
+			},
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "gcp_kubernetes_engine",
+				"csm.remote_workload_canonical_service": "canonical_service_val",
+				"csm.remote_workload_project_id":        "unique-id",
+				"csm.remote_workload_cluster_name":      "unknown",
+				"csm.remote_workload_namespace_name":    "namespace_name_val",
+				"csm.remote_workload_location":          "us-east",
+				"csm.remote_workload_name":              "unknown",
+			},
+		},
+		{
+			name: "unset-header",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			unsetHeader: true,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+		{
+			name: "two-header-values",
+			metadataExchangeLabels: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "canonical_service_val",
+				"project_id":        "unique-id",
+				"namespace_name":    "namespace_name_val",
+				"cluster_name":      "cluster_name_val",
+				"location":          "us-east",
+				"workload_name":     "workload_name_val",
+			},
+			twoValues: true,
+			labelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+
+				"csm.remote_workload_type":              "unknown",
+				"csm.remote_workload_canonical_service": "unknown",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pbLabels := &structpb.Struct{
+				Fields: map[string]*structpb.Value{},
+			}
+			for k, v := range test.metadataExchangeLabels {
+				pbLabels.Fields[k] = structpb.NewStringValue(v)
+			}
+			protoWireFormat, err := proto.Marshal(pbLabels)
+			if err != nil {
+				t.Fatalf("Error marshaling proto: %v", err)
+			}
+			metadataExchangeLabelsEncoded := base64.RawStdEncoding.EncodeToString(protoWireFormat)
+			md := metadata.New(map[string]string{
+				metadataExchangeKey: metadataExchangeLabelsEncoded,
+			})
+			if test.unsetHeader {
+				md.Delete(metadataExchangeKey)
+			}
+			if test.twoValues {
+				md.Append(metadataExchangeKey, "extra-val")
+			}
+
+			labelsGot := cpo.GetLabels(md)
+			if diff := cmp.Diff(labelsGot, test.labelsWant); diff != "" {
+				t.Fatalf("cpo.GetLabels returned unexpected value (-got, +want): %v", diff)
+			}
+		})
+	}
+}
+
+// TestDetermineTargetCSM tests the helper function that determines whether a
+// target is relevant to CSM or not, based off the rules outlined in design.
+func (s) TestDetermineTargetCSM(t *testing.T) {
+	tests := []struct {
+		name      string
+		target    string
+		targetCSM bool
+	}{
+		{
+			name:      "dns:///localhost",
+			target:    "normal-target-here",
+			targetCSM: false,
+		},
+		{
+			name:      "xds-no-authority",
+			target:    "xds:///localhost",
+			targetCSM: true,
+		},
+		{
+			name:      "xds-traffic-director-authority",
+			target:    "xds://traffic-director-global.xds.googleapis.com/localhost",
+			targetCSM: true,
+		},
+		{
+			name:      "xds-not-traffic-director-authority",
+			target:    "xds://not-traffic-director-authority/localhost",
+			targetCSM: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			parsedTarget, err := url.Parse(test.target)
+			if err != nil {
+				t.Fatalf("test target %v failed to parse: %v", test.target, err)
+			}
+			if got := determineTargetCSM(parsedTarget); got != test.targetCSM {
+				t.Fatalf("cpo.determineTargetCSM(%v): got %v, want %v", test.target, got, test.targetCSM)
+			}
+		})
+	}
+}
+
+// TestSetLabels tests the setting of labels, which snapshots the resource and
+// environment. It mocks the resource and environment, and then calls into
+// labels creation. It verifies to local labels created and metadata exchange
+// labels emitted from the setLabels function.
+func (s) TestSetLabels(t *testing.T) {
+	clearEnv()
+	tests := []struct {
+		name                             string
+		resourceKeyValues                map[string]string
+		csmCanonicalServiceNamePopulated bool
+		csmWorkloadNamePopulated         bool
+		meshIDPopulated                  bool
+		localLabelsWant                  map[string]string
+		metadataExchangeLabelsWant       map[string]string
+	}{
+		{
+			name:                             "no-type",
+			csmCanonicalServiceNamePopulated: true,
+			meshIDPopulated:                  true,
+			resourceKeyValues:                map[string]string{},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "canonical_service_name_val", // env var populated so should be set.
+				"csm.mesh_id":                    "mesh_id",                    // env var populated so should be set.
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "unknown",
+				"canonical_service": "canonical_service_name_val", // env var populated so should be set.
+			},
+		},
+		{
+			name:                     "gce",
+			csmWorkloadNamePopulated: true,
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_compute_engine",
+				// csm workload name is an env var
+				"cloud.availability_zone": "cloud_availability_zone_val",
+				"cloud.region":            "should-be-ignored", // cloud.availability_zone takes precedence
+				"cloud.account.id":        "cloud_account_id_val",
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "workload_name_val",
+				"location":          "cloud_availability_zone_val",
+				"project_id":        "cloud_account_id_val",
+			},
+		},
+		{
+			name: "gce-half-unset",
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_compute_engine",
+				// csm workload name is an env var
+				"cloud.availability_zone": "cloud_availability_zone_val",
+				"cloud.region":            "should-be-ignored", // cloud.availability_zone takes precedence
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_compute_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_availability_zone_val",
+				"project_id":        "unknown",
+			},
+		},
+		{
+			name: "gke",
+			resourceKeyValues: map[string]string{
+				"cloud.platform": "gcp_kubernetes_engine",
+				// csm workload name is an env var
+				"cloud.region":       "cloud_region_val", // availability_zone isn't present, so this should become location
+				"cloud.account.id":   "cloud_account_id_val",
+				"k8s.namespace.name": "k8s_namespace_name_val",
+				"k8s.cluster.name":   "k8s_cluster_name_val",
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_region_val",
+				"project_id":        "cloud_account_id_val",
+				"namespace_name":    "k8s_namespace_name_val",
+				"cluster_name":      "k8s_cluster_name_val",
+			},
+		},
+		{
+			name: "gke-half-unset",
+			resourceKeyValues: map[string]string{ // unset should become unknown
+				"cloud.platform": "gcp_kubernetes_engine",
+				// csm workload name is an env var
+				"cloud.region": "cloud_region_val", // availability_zone isn't present, so this should become location
+				// "cloud.account.id": "", // unset - should become unknown
+				"k8s.namespace.name": "k8s_namespace_name_val",
+				// "k8s.cluster.name": "", // unset - should become unknown
+			},
+			localLabelsWant: map[string]string{
+				"csm.workload_canonical_service": "unknown",
+				"csm.mesh_id":                    "unknown",
+			},
+			metadataExchangeLabelsWant: map[string]string{
+				"type":              "gcp_kubernetes_engine",
+				"canonical_service": "unknown",
+				"workload_name":     "unknown",
+				"location":          "cloud_region_val",
+				"project_id":        "unknown",
+				"namespace_name":    "k8s_namespace_name_val",
+				"cluster_name":      "unknown",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			func() {
+				if test.csmCanonicalServiceNamePopulated {
+					os.Setenv("CSM_CANONICAL_SERVICE_NAME", "canonical_service_name_val")
+					defer os.Unsetenv("CSM_CANONICAL_SERVICE_NAME")
+				}
+				if test.csmWorkloadNamePopulated {
+					os.Setenv("CSM_WORKLOAD_NAME", "workload_name_val")
+					defer os.Unsetenv("CSM_WORKLOAD_NAME")
+				}
+				if test.meshIDPopulated {
+					os.Setenv("CSM_MESH_ID", "mesh_id")
+					defer os.Unsetenv("CSM_MESH_ID")
+				}
+				var attributes []attribute.KeyValue
+				for k, v := range test.resourceKeyValues {
+					attributes = append(attributes, attribute.String(k, v))
+				}
+				// Return the attributes configured as part of the test in place
+				// of reading from resource.
+				attrSet := attribute.NewSet(attributes...)
+				origGetAttrSet := getAttrSetFromResourceDetector
+				getAttrSetFromResourceDetector = func(context.Context) *attribute.Set {
+					return &attrSet
+				}
+				defer func() { getAttrSetFromResourceDetector = origGetAttrSet }()
+
+				ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+				defer cancel()
+				localLabelsGot, mdEncoded := constructMetadataFromEnv(ctx)
+				if diff := cmp.Diff(localLabelsGot, test.localLabelsWant); diff != "" {
+					t.Fatalf("constructMetadataFromEnv() want: %v, got %v", test.localLabelsWant, localLabelsGot)
+				}
+
+				verifyMetadataExchangeLabels(mdEncoded, test.metadataExchangeLabelsWant)
+			}()
+		})
+	}
+}
+
+func verifyMetadataExchangeLabels(mdEncoded string, mdLabelsWant map[string]string) error {
+	protoWireFormat, err := base64.RawStdEncoding.DecodeString(mdEncoded)
+	if err != nil {
+		return fmt.Errorf("error base 64 decoding metadata val: %v", err)
+	}
+	spb := &structpb.Struct{}
+	if err := proto.Unmarshal(protoWireFormat, spb); err != nil {
+		return fmt.Errorf("error unmarshaling proto wire format: %v", err)
+	}
+	fields := spb.GetFields()
+	for k, v := range mdLabelsWant {
+		if val, ok := fields[k]; !ok {
+			if _, ok := val.GetKind().(*structpb.Value_StringValue); !ok {
+				return fmt.Errorf("struct value for key %v should be string type", k)
+			}
+			if val.GetStringValue() != v {
+				return fmt.Errorf("struct value for key %v got: %v, want %v", k, val.GetStringValue(), v)
+			}
+		}
+	}
+	if len(mdLabelsWant) != len(fields) {
+		return fmt.Errorf("len(mdLabelsWant) = %v, len(mdLabelsGot) = %v", len(mdLabelsWant), len(fields))
+	}
+	return nil
+}