snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

orca/call_metrics.go

@@ -0,0 +1,196 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca
+
+import (
+	"context"
+	"sync"
+
+	"google.golang.org/grpc"
+	grpcinternal "google.golang.org/grpc/internal"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/protobuf/proto"
+)
+
+// CallMetricsRecorder allows a service method handler to record per-RPC
+// metrics.  It contains all utilization-based metrics from
+// ServerMetricsRecorder as well as additional request cost metrics.
+type CallMetricsRecorder interface {
+	ServerMetricsRecorder
+
+	// SetRequestCost sets the relevant server metric.
+	SetRequestCost(name string, val float64)
+	// DeleteRequestCost deletes the relevant server metric to prevent it
+	// from being sent.
+	DeleteRequestCost(name string)
+
+	// SetNamedMetric sets the relevant server metric.
+	SetNamedMetric(name string, val float64)
+	// DeleteNamedMetric deletes the relevant server metric to prevent it
+	// from being sent.
+	DeleteNamedMetric(name string)
+}
+
+type callMetricsRecorderCtxKey struct{}
+
+// CallMetricsRecorderFromContext returns the RPC-specific custom metrics
+// recorder embedded in the provided RPC context.
+//
+// Returns nil if no custom metrics recorder is found in the provided context,
+// which will be the case when custom metrics reporting is not enabled.
+func CallMetricsRecorderFromContext(ctx context.Context) CallMetricsRecorder {
+	rw, ok := ctx.Value(callMetricsRecorderCtxKey{}).(*recorderWrapper)
+	if !ok {
+		return nil
+	}
+	return rw.recorder()
+}
+
+// recorderWrapper is a wrapper around a CallMetricsRecorder to ensure that
+// concurrent calls to CallMetricsRecorderFromContext() results in only one
+// allocation of the underlying metrics recorder, while also allowing for lazy
+// initialization of the recorder itself.
+type recorderWrapper struct {
+	once sync.Once
+	r    CallMetricsRecorder
+	smp  ServerMetricsProvider
+}
+
+func (rw *recorderWrapper) recorder() CallMetricsRecorder {
+	rw.once.Do(func() {
+		rw.r = newServerMetricsRecorder()
+	})
+	return rw.r
+}
+
+// setTrailerMetadata adds a trailer metadata entry with key being set to
+// `internal.TrailerMetadataKey` and value being set to the binary-encoded
+// orca.OrcaLoadReport protobuf message.
+//
+// This function is called from the unary and streaming interceptors defined
+// above. Any errors encountered here are not propagated to the caller because
+// they are ignored there. Hence we simply log any errors encountered here at
+// warning level, and return nothing.
+func (rw *recorderWrapper) setTrailerMetadata(ctx context.Context) {
+	var sm *ServerMetrics
+	if rw.smp != nil {
+		sm = rw.smp.ServerMetrics()
+		sm.merge(rw.r.ServerMetrics())
+	} else {
+		sm = rw.r.ServerMetrics()
+	}
+
+	b, err := proto.Marshal(sm.toLoadReportProto())
+	if err != nil {
+		logger.Warningf("Failed to marshal load report: %v", err)
+		return
+	}
+	if err := grpc.SetTrailer(ctx, metadata.Pairs(internal.TrailerMetadataKey, string(b))); err != nil {
+		logger.Warningf("Failed to set trailer metadata: %v", err)
+	}
+}
+
+var joinServerOptions = grpcinternal.JoinServerOptions.(func(...grpc.ServerOption) grpc.ServerOption)
+
+// CallMetricsServerOption returns a server option which enables the reporting
+// of per-RPC custom backend metrics for unary and streaming RPCs.
+//
+// Server applications interested in injecting custom backend metrics should
+// pass the server option returned from this function as the first argument to
+// grpc.NewServer().
+//
+// Subsequently, server RPC handlers can retrieve a reference to the RPC
+// specific custom metrics recorder [CallMetricsRecorder] to be used, via a call
+// to CallMetricsRecorderFromContext(), and inject custom metrics at any time
+// during the RPC lifecycle.
+//
+// The injected custom metrics will be sent as part of trailer metadata, as a
+// binary-encoded [ORCA LoadReport] protobuf message, with the metadata key
+// being set be "endpoint-load-metrics-bin".
+//
+// If a non-nil ServerMetricsProvider is provided, the gRPC server will
+// transmit the metrics it provides, overwritten by any per-RPC metrics given
+// to the CallMetricsRecorder.  A ServerMetricsProvider is typically obtained
+// by calling NewServerMetricsRecorder.
+//
+// [ORCA LoadReport]: https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15
+func CallMetricsServerOption(smp ServerMetricsProvider) grpc.ServerOption {
+	return joinServerOptions(grpc.ChainUnaryInterceptor(unaryInt(smp)), grpc.ChainStreamInterceptor(streamInt(smp)))
+}
+
+func unaryInt(smp ServerMetricsProvider) func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+	return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) {
+		// We don't allocate the metric recorder here. It will be allocated the
+		// first time the user calls CallMetricsRecorderFromContext().
+		rw := &recorderWrapper{smp: smp}
+		ctxWithRecorder := newContextWithRecorderWrapper(ctx, rw)
+
+		resp, err := handler(ctxWithRecorder, req)
+
+		// It is safe to access the underlying metric recorder inside the wrapper at
+		// this point, as the user's RPC handler is done executing, and therefore
+		// there will be no more calls to CallMetricsRecorderFromContext(), which is
+		// where the metric recorder is lazy allocated.
+		if rw.r != nil {
+			rw.setTrailerMetadata(ctx)
+		}
+		return resp, err
+	}
+}
+
+func streamInt(smp ServerMetricsProvider) func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+	return func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		// We don't allocate the metric recorder here. It will be allocated the
+		// first time the user calls CallMetricsRecorderFromContext().
+		rw := &recorderWrapper{smp: smp}
+		ws := &wrappedStream{
+			ServerStream: ss,
+			ctx:          newContextWithRecorderWrapper(ss.Context(), rw),
+		}
+
+		err := handler(srv, ws)
+
+		// It is safe to access the underlying metric recorder inside the wrapper at
+		// this point, as the user's RPC handler is done executing, and therefore
+		// there will be no more calls to CallMetricsRecorderFromContext(), which is
+		// where the metric recorder is lazy allocated.
+		if rw.r != nil {
+			rw.setTrailerMetadata(ss.Context())
+		}
+		return err
+	}
+}
+
+func newContextWithRecorderWrapper(ctx context.Context, r *recorderWrapper) context.Context {
+	return context.WithValue(ctx, callMetricsRecorderCtxKey{}, r)
+}
+
+// wrappedStream wraps the grpc.ServerStream received by the streaming
+// interceptor. Overrides only the Context() method to return a context which
+// contains a reference to the CallMetricsRecorder corresponding to this
+// stream.
+type wrappedStream struct {
+	grpc.ServerStream
+	ctx context.Context
+}
+
+func (w *wrappedStream) Context() context.Context {
+	return w.ctx
+}

orca/call_metrics_test.go

@@ -0,0 +1,287 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/orca"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/protobuf/proto"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+// TestE2ECallMetricsUnary tests the injection of custom backend metrics from
+// the server application for a unary RPC, and verifies that expected load
+// reports are received at the client.
+func (s) TestE2ECallMetricsUnary(t *testing.T) {
+	tests := []struct {
+		desc          string
+		injectMetrics bool
+		wantProto     *v3orcapb.OrcaLoadReport
+	}{
+		{
+			desc:          "with custom backend metrics",
+			injectMetrics: true,
+			wantProto: &v3orcapb.OrcaLoadReport{
+				CpuUtilization: 1.0,
+				MemUtilization: 0.9,
+				RequestCost:    map[string]float64{"queryCost": 25.0},
+				Utilization:    map[string]float64{"queueSize": 0.75},
+			},
+		},
+		{
+			desc:          "with no custom backend metrics",
+			injectMetrics: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			// A server option to enable reporting of per-call backend metrics.
+			smr := orca.NewServerMetricsRecorder()
+			callMetricsServerOption := orca.CallMetricsServerOption(smr)
+			smr.SetCPUUtilization(1.0)
+
+			// An interceptor to injects custom backend metrics, added only when
+			// the injectMetrics field in the test is set.
+			injectingInterceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) {
+				recorder := orca.CallMetricsRecorderFromContext(ctx)
+				if recorder == nil {
+					err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context")
+					t.Error(err)
+					return nil, err
+				}
+				recorder.SetMemoryUtilization(0.9)
+				// This value will be overwritten by a write to the same metric
+				// from the server handler.
+				recorder.SetNamedUtilization("queueSize", 1.0)
+				return handler(ctx, req)
+			}
+
+			// A stub server whose unary handler injects custom metrics, if the
+			// injectMetrics field in the test is set. It overwrites one of the
+			// values injected above, by the interceptor.
+			srv := stubserver.StubServer{
+				EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
+					if !test.injectMetrics {
+						return &testpb.Empty{}, nil
+					}
+					recorder := orca.CallMetricsRecorderFromContext(ctx)
+					if recorder == nil {
+						err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context")
+						t.Error(err)
+						return nil, err
+					}
+					recorder.SetRequestCost("queryCost", 25.0)
+					recorder.SetNamedUtilization("queueSize", 0.75)
+					return &testpb.Empty{}, nil
+				},
+			}
+
+			// Start the stub server with the appropriate server options.
+			sopts := []grpc.ServerOption{callMetricsServerOption}
+			if test.injectMetrics {
+				sopts = append(sopts, grpc.ChainUnaryInterceptor(injectingInterceptor))
+			}
+			if err := srv.StartServer(sopts...); err != nil {
+				t.Fatalf("Failed to start server: %v", err)
+			}
+			defer srv.Stop()
+
+			// Dial the stub server.
+			cc, err := grpc.NewClient(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("grpc.NewClient(%s) failed: %v", srv.Address, err)
+			}
+			defer cc.Close()
+
+			// Make a unary RPC and expect the trailer metadata to contain the custom
+			// backend metrics as an ORCA LoadReport protobuf message.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			client := testgrpc.NewTestServiceClient(cc)
+			trailer := metadata.MD{}
+			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Trailer(&trailer)); err != nil {
+				t.Fatalf("EmptyCall failed: %v", err)
+			}
+
+			gotProto, err := internal.ToLoadReport(trailer)
+			if err != nil {
+				t.Fatalf("When retrieving load report, got error: %v, want: <nil>", err)
+			}
+			if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) {
+				t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto))
+			}
+		})
+	}
+}
+
+// TestE2ECallMetricsStreaming tests the injection of custom backend metrics
+// from the server application for a streaming RPC, and verifies that expected
+// load reports are received at the client.
+func (s) TestE2ECallMetricsStreaming(t *testing.T) {
+	tests := []struct {
+		desc          string
+		injectMetrics bool
+		wantProto     *v3orcapb.OrcaLoadReport
+	}{
+		{
+			desc:          "with custom backend metrics",
+			injectMetrics: true,
+			wantProto: &v3orcapb.OrcaLoadReport{
+				CpuUtilization: 1.0,
+				MemUtilization: 0.5,
+				RequestCost:    map[string]float64{"queryCost": 0.25},
+				Utilization:    map[string]float64{"queueSize": 0.75},
+			},
+		},
+		{
+			desc:          "with no custom backend metrics",
+			injectMetrics: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			// A server option to enable reporting of per-call backend metrics.
+			smr := orca.NewServerMetricsRecorder()
+			callMetricsServerOption := orca.CallMetricsServerOption(smr)
+			smr.SetCPUUtilization(1.0)
+
+			// An interceptor which injects custom backend metrics, added only
+			// when the injectMetrics field in the test is set.
+			injectingInterceptor := func(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+				recorder := orca.CallMetricsRecorderFromContext(ss.Context())
+				if recorder == nil {
+					err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context")
+					t.Error(err)
+					return err
+				}
+				recorder.SetMemoryUtilization(0.5)
+				// This value will be overwritten by a write to the same metric
+				// from the server handler.
+				recorder.SetNamedUtilization("queueSize", 1.0)
+				return handler(srv, ss)
+			}
+
+			// A stub server whose streaming handler injects custom metrics, if
+			// the injectMetrics field in the test is set. It overwrites one of
+			// the values injected above, by the interceptor.
+			srv := stubserver.StubServer{
+				FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
+					if test.injectMetrics {
+						recorder := orca.CallMetricsRecorderFromContext(stream.Context())
+						if recorder == nil {
+							err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context")
+							t.Error(err)
+							return err
+						}
+						recorder.SetRequestCost("queryCost", 0.25)
+						recorder.SetNamedUtilization("queueSize", 0.75)
+					}
+
+					// Streaming implementation replies with a dummy response until the
+					// client closes the stream (in which case it will see an io.EOF),
+					// or an error occurs while reading/writing messages.
+					for {
+						_, err := stream.Recv()
+						if err == io.EOF {
+							return nil
+						}
+						if err != nil {
+							return err
+						}
+						payload := &testpb.Payload{Body: make([]byte, 32)}
+						if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: payload}); err != nil {
+							return err
+						}
+					}
+				},
+			}
+
+			// Start the stub server with the appropriate server options.
+			sopts := []grpc.ServerOption{callMetricsServerOption}
+			if test.injectMetrics {
+				sopts = append(sopts, grpc.ChainStreamInterceptor(injectingInterceptor))
+			}
+			if err := srv.StartServer(sopts...); err != nil {
+				t.Fatalf("Failed to start server: %v", err)
+			}
+			defer srv.Stop()
+
+			// Dial the stub server.
+			cc, err := grpc.NewClient(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
+			if err != nil {
+				t.Fatalf("grpc.NewClient(%s) failed: %v", srv.Address, err)
+			}
+			defer cc.Close()
+
+			// Start the full duplex streaming RPC.
+			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+			defer cancel()
+			tc := testgrpc.NewTestServiceClient(cc)
+			stream, err := tc.FullDuplexCall(ctx)
+			if err != nil {
+				t.Fatalf("FullDuplexCall failed: %v", err)
+			}
+
+			// Send one request to the server.
+			payload := &testpb.Payload{Body: make([]byte, 32)}
+			req := &testpb.StreamingOutputCallRequest{Payload: payload}
+			if err := stream.Send(req); err != nil {
+				t.Fatalf("stream.Send() failed: %v", err)
+			}
+			// Read one reply from the server.
+			if _, err := stream.Recv(); err != nil {
+				t.Fatalf("stream.Recv() failed: %v", err)
+			}
+			// Close the sending side.
+			if err := stream.CloseSend(); err != nil {
+				t.Fatalf("stream.CloseSend() failed: %v", err)
+			}
+			// Make sure it is safe to read the trailer.
+			for {
+				if _, err := stream.Recv(); err != nil {
+					break
+				}
+			}
+
+			gotProto, err := internal.ToLoadReport(stream.Trailer())
+			if err != nil {
+				t.Fatalf("When retrieving load report, got error: %v, want: <nil>", err)
+			}
+			if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) {
+				t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto))
+			}
+		})
+	}
+}

orca/internal/internal.go

@@ -0,0 +1,71 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package internal contains orca-internal code, for testing purposes and to
+// avoid polluting the godoc of the top-level orca package.
+package internal
+
+import (
+	"errors"
+	"fmt"
+
+	ibackoff "google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/protobuf/proto"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+)
+
+// AllowAnyMinReportingInterval prevents clamping of the MinReportingInterval
+// configured via ServiceOptions, to a minimum of 30s.
+//
+// For testing purposes only.
+var AllowAnyMinReportingInterval any // func(*ServiceOptions)
+
+// DefaultBackoffFunc is used by the producer to control its backoff behavior.
+//
+// For testing purposes only.
+var DefaultBackoffFunc = ibackoff.DefaultExponential.Backoff
+
+// TrailerMetadataKey is the key in which the per-call backend metrics are
+// transmitted.
+const TrailerMetadataKey = "endpoint-load-metrics-bin"
+
+// ToLoadReport unmarshals a binary encoded [ORCA LoadReport] protobuf message
+// from md and returns the corresponding struct. The load report is expected to
+// be stored as the value for key "endpoint-load-metrics-bin".
+//
+// If no load report was found in the provided metadata, if multiple load
+// reports are found, or if the load report found cannot be parsed, an error is
+// returned.
+//
+// [ORCA LoadReport]: (https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15)
+func ToLoadReport(md metadata.MD) (*v3orcapb.OrcaLoadReport, error) {
+	vs := md.Get(TrailerMetadataKey)
+	if len(vs) == 0 {
+		return nil, nil
+	}
+	if len(vs) != 1 {
+		return nil, errors.New("multiple orca load reports found in provided metadata")
+	}
+	ret := new(v3orcapb.OrcaLoadReport)
+	if err := proto.Unmarshal([]byte(vs[0]), ret); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal load report found in metadata: %v", err)
+	}
+	return ret, nil
+}

orca/orca.go

@@ -0,0 +1,57 @@
+/*
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package orca implements Open Request Cost Aggregation, which is an open
+// standard for request cost aggregation and reporting by backends and the
+// corresponding aggregation of such reports by L7 load balancers (such as
+// Envoy) on the data plane. In a proxyless world with gRPC enabled
+// applications, aggregation of such reports will be done by the gRPC client.
+//
+// # Experimental
+//
+// Notice: All APIs is this package are EXPERIMENTAL and may be changed or
+// removed in a later release.
+package orca
+
+import (
+	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/balancerload"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/orca/internal"
+)
+
+var logger = grpclog.Component("orca-backend-metrics")
+
+// loadParser implements the Parser interface defined in `internal/balancerload`
+// package. This interface is used by the client stream to parse load reports
+// sent by the server in trailer metadata. The parsed loads are then sent to
+// balancers via balancer.DoneInfo.
+//
+// The grpc package cannot directly call toLoadReport() as that would cause an
+// import cycle. Hence this roundabout method is used.
+type loadParser struct{}
+
+func (loadParser) Parse(md metadata.MD) any {
+	lr, err := internal.ToLoadReport(md)
+	if err != nil {
+		logger.Infof("Parse failed: %v", err)
+	}
+	return lr
+}
+
+func init() {
+	balancerload.SetParser(loadParser{})
+}

orca/orca_test.go

@@ -0,0 +1,102 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/protobuf/proto"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+const defaultTestTimeout = 5 * time.Second
+
+func (s) TestToLoadReport(t *testing.T) {
+	goodReport := &v3orcapb.OrcaLoadReport{
+		CpuUtilization: 1.0,
+		MemUtilization: 50.0,
+		RequestCost:    map[string]float64{"queryCost": 25.0},
+		Utilization:    map[string]float64{"queueSize": 75.0},
+	}
+	tests := []struct {
+		name    string
+		md      metadata.MD
+		want    *v3orcapb.OrcaLoadReport
+		wantErr bool
+	}{
+		{
+			name:    "no load report in metadata",
+			md:      metadata.MD{},
+			wantErr: false,
+		},
+		{
+			name: "badly marshaled load report",
+			md: func() metadata.MD {
+				return metadata.Pairs("endpoint-load-metrics-bin", string("foo-bar"))
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "multiple load reports",
+			md: func() metadata.MD {
+				b, _ := proto.Marshal(goodReport)
+				return metadata.Pairs("endpoint-load-metrics-bin", string(b), "endpoint-load-metrics-bin", string(b))
+			}(),
+			wantErr: true,
+		},
+		{
+			name: "good load report",
+			md: func() metadata.MD {
+				b, _ := proto.Marshal(goodReport)
+				return metadata.Pairs("endpoint-load-metrics-bin", string(b))
+			}(),
+			want: goodReport,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got, err := internal.ToLoadReport(test.md)
+			if (err != nil) != test.wantErr {
+				t.Fatalf("orca.ToLoadReport(%v) = %v, wantErr: %v", test.md, err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			if !cmp.Equal(got, test.want, cmp.Comparer(proto.Equal)) {
+				t.Fatalf("Extracted load report from metadata: %s, want: %s", pretty.ToJSON(got), pretty.ToJSON(test.want))
+			}
+		})
+	}
+}

orca/producer.go

@@ -0,0 +1,225 @@
+/*
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package orca
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal/backoff"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/grpc/status"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3"
+	v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+type producerBuilder struct{}
+
+// Build constructs and returns a producer and its cleanup function
+func (*producerBuilder) Build(cci any) (balancer.Producer, func()) {
+	p := &producer{
+		client:    v3orcaservicegrpc.NewOpenRcaServiceClient(cci.(grpc.ClientConnInterface)),
+		intervals: make(map[time.Duration]int),
+		listeners: make(map[OOBListener]struct{}),
+		backoff:   internal.DefaultBackoffFunc,
+	}
+	return p, func() {
+		p.mu.Lock()
+		if p.stop != nil {
+			p.stop()
+			p.stop = nil
+		}
+		p.mu.Unlock()
+		<-p.stopped
+	}
+}
+
+var producerBuilderSingleton = &producerBuilder{}
+
+// OOBListener is used to receive out-of-band load reports as they arrive.
+type OOBListener interface {
+	// OnLoadReport is called when a load report is received.
+	OnLoadReport(*v3orcapb.OrcaLoadReport)
+}
+
+// OOBListenerOptions contains options to control how an OOBListener is called.
+type OOBListenerOptions struct {
+	// ReportInterval specifies how often to request the server to provide a
+	// load report.  May be provided less frequently if the server requires a
+	// longer interval, or may be provided more frequently if another
+	// subscriber requests a shorter interval.
+	ReportInterval time.Duration
+}
+
+// RegisterOOBListener registers an out-of-band load report listener on a Ready
+// sc.  Any OOBListener may only be registered once per subchannel at a time.
+// The returned stop function must be called when no longer needed.  Do not
+// register a single OOBListener more than once per SubConn.
+func RegisterOOBListener(sc balancer.SubConn, l OOBListener, opts OOBListenerOptions) (stop func()) {
+	pr, closeFn := sc.GetOrBuildProducer(producerBuilderSingleton)
+	p := pr.(*producer)
+
+	p.registerListener(l, opts.ReportInterval)
+
+	// If stop is called multiple times, prevent it from having any effect on
+	// subsequent calls.
+	return sync.OnceFunc(func() {
+		p.unregisterListener(l, opts.ReportInterval)
+		closeFn()
+	})
+}
+
+type producer struct {
+	client v3orcaservicegrpc.OpenRcaServiceClient
+
+	// backoff is called between stream attempts to determine how long to delay
+	// to avoid overloading a server experiencing problems.  The attempt count
+	// is incremented when stream errors occur and is reset when the stream
+	// reports a result.
+	backoff func(int) time.Duration
+	stopped chan struct{} // closed when the run goroutine exits
+
+	mu          sync.Mutex
+	intervals   map[time.Duration]int    // map from interval time to count of listeners requesting that time
+	listeners   map[OOBListener]struct{} // set of registered listeners
+	minInterval time.Duration
+	stop        func() // stops the current run goroutine
+}
+
+// registerListener adds the listener and its requested report interval to the
+// producer.
+func (p *producer) registerListener(l OOBListener, interval time.Duration) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.listeners[l] = struct{}{}
+	p.intervals[interval]++
+	if len(p.listeners) == 1 || interval < p.minInterval {
+		p.minInterval = interval
+		p.updateRunLocked()
+	}
+}
+
+// registerListener removes the listener and its requested report interval to
+// the producer.
+func (p *producer) unregisterListener(l OOBListener, interval time.Duration) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	delete(p.listeners, l)
+	p.intervals[interval]--
+	if p.intervals[interval] == 0 {
+		delete(p.intervals, interval)
+
+		if p.minInterval == interval {
+			p.recomputeMinInterval()
+			p.updateRunLocked()
+		}
+	}
+}
+
+// recomputeMinInterval sets p.minInterval to the minimum key's value in
+// p.intervals.
+func (p *producer) recomputeMinInterval() {
+	first := true
+	for interval := range p.intervals {
+		if first || interval < p.minInterval {
+			p.minInterval = interval
+			first = false
+		}
+	}
+}
+
+// updateRunLocked is called whenever the run goroutine needs to be started /
+// stopped / restarted due to: 1. the initial listener being registered, 2. the
+// final listener being unregistered, or 3. the minimum registered interval
+// changing.
+func (p *producer) updateRunLocked() {
+	if p.stop != nil {
+		p.stop()
+		p.stop = nil
+	}
+	if len(p.listeners) > 0 {
+		var ctx context.Context
+		ctx, p.stop = context.WithCancel(context.Background())
+		p.stopped = make(chan struct{})
+		go p.run(ctx, p.stopped, p.minInterval)
+	}
+}
+
+// run manages the ORCA OOB stream on the subchannel.
+func (p *producer) run(ctx context.Context, done chan struct{}, interval time.Duration) {
+	defer close(done)
+
+	runStream := func() error {
+		resetBackoff, err := p.runStream(ctx, interval)
+		if status.Code(err) == codes.Unimplemented {
+			// Unimplemented; do not retry.
+			logger.Error("Server doesn't support ORCA OOB load reporting protocol; not listening for load reports.")
+			return err
+		}
+		// Retry for all other errors.
+		if code := status.Code(err); code != codes.Unavailable && code != codes.Canceled {
+			// TODO: Unavailable and Canceled should also ideally log an error,
+			// but for now we receive them when shutting down the ClientConn
+			// (Unavailable if the stream hasn't started yet, and Canceled if it
+			// happens mid-stream).  Once we can determine the state or ensure
+			// the producer is stopped before the stream ends, we can log an
+			// error when it's not a natural shutdown.
+			logger.Error("Received unexpected stream error:", err)
+		}
+		if resetBackoff {
+			return backoff.ErrResetBackoff
+		}
+		return nil
+	}
+	backoff.RunF(ctx, runStream, p.backoff)
+}
+
+// runStream runs a single stream on the subchannel and returns the resulting
+// error, if any, and whether or not the run loop should reset the backoff
+// timer to zero or advance it.
+func (p *producer) runStream(ctx context.Context, interval time.Duration) (resetBackoff bool, err error) {
+	streamCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	stream, err := p.client.StreamCoreMetrics(streamCtx, &v3orcaservicepb.OrcaLoadReportRequest{
+		ReportInterval: durationpb.New(interval),
+	})
+	if err != nil {
+		return false, err
+	}
+
+	for {
+		report, err := stream.Recv()
+		if err != nil {
+			return resetBackoff, err
+		}
+		resetBackoff = true
+		p.mu.Lock()
+		for l := range p.listeners {
+			l.OnLoadReport(report)
+		}
+		p.mu.Unlock()
+	}
+}

orca/producer_test.go

@@ -0,0 +1,572 @@
+/*
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package orca_test
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/balancer"
+	"google.golang.org/grpc/balancer/roundrobin"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/connectivity"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/orca"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3"
+	v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3"
+)
+
+// customLBB wraps a round robin LB policy but provides a ClientConn wrapper to
+// add an ORCA OOB report producer for all created SubConns.
+type customLBB struct{}
+
+func (customLBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
+	return balancer.Get(roundrobin.Name).Build(&ccWrapper{ClientConn: cc}, opts)
+}
+
+func (customLBB) Name() string { return "customLB" }
+
+func init() {
+	balancer.Register(customLBB{})
+}
+
+type ccWrapper struct {
+	balancer.ClientConn
+}
+
+func (w *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) {
+	if len(addrs) != 1 {
+		panic(fmt.Sprintf("got addrs=%v; want len(addrs) == 1", addrs))
+	}
+	var sc balancer.SubConn
+	opts.StateListener = func(scs balancer.SubConnState) {
+		if scs.ConnectivityState != connectivity.Ready {
+			return
+		}
+		l := getListenerInfo(addrs[0])
+		l.listener.cleanup = orca.RegisterOOBListener(sc, l.listener, l.opts)
+		l.scChan <- sc
+	}
+	sc, err := w.ClientConn.NewSubConn(addrs, opts)
+	if err != nil {
+		return sc, err
+	}
+	return sc, nil
+}
+
+// listenerInfo is stored in an address's attributes to allow ORCA
+// listeners to be registered on subconns created for that address.
+type listenerInfo struct {
+	listener *testOOBListener
+	opts     orca.OOBListenerOptions
+	scChan   chan balancer.SubConn // Pushed on by the LB policy
+}
+
+type listenerInfoKey struct{}
+
+func setListenerInfo(addr resolver.Address, l *listenerInfo) resolver.Address {
+	addr.Attributes = addr.Attributes.WithValue(listenerInfoKey{}, l)
+	return addr
+}
+
+func getListenerInfo(addr resolver.Address) *listenerInfo {
+	return addr.Attributes.Value(listenerInfoKey{}).(*listenerInfo)
+}
+
+// testOOBListener is a simple listener that pushes load reports to a channel.
+type testOOBListener struct {
+	cleanup      func()
+	loadReportCh chan *v3orcapb.OrcaLoadReport
+}
+
+func newTestOOBListener() *testOOBListener {
+	return &testOOBListener{cleanup: func() {}, loadReportCh: make(chan *v3orcapb.OrcaLoadReport)}
+}
+
+func (t *testOOBListener) Stop() { t.cleanup() }
+
+func (t *testOOBListener) OnLoadReport(r *v3orcapb.OrcaLoadReport) {
+	t.loadReportCh <- r
+}
+
+// TestProducer is a basic, end-to-end style test of an LB policy with an
+// OOBListener communicating with a server with an ORCA service.
+func (s) TestProducer(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Use a fixed backoff for stream recreation.
+	oldBackoff := internal.DefaultBackoffFunc
+	internal.DefaultBackoffFunc = func(int) time.Duration { return 10 * time.Millisecond }
+	defer func() { internal.DefaultBackoffFunc = oldBackoff }()
+
+	// Initialize listener for our ORCA server.
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Register the OpenRCAService with a very short metrics reporting interval.
+	const shortReportingInterval = 50 * time.Millisecond
+	smr := orca.NewServerMetricsRecorder()
+	opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr}
+	internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts)
+	s := grpc.NewServer()
+	if err := orca.Register(s, opts); err != nil {
+		t.Fatalf("orca.Register failed: %v", err)
+	}
+	go s.Serve(lis)
+	defer s.Stop()
+
+	// Create our client with an OOB listener in the LB policy it selects.
+	r := manual.NewBuilderWithScheme("whatever")
+	oobLis := newTestOOBListener()
+
+	lisOpts := orca.OOBListenerOptions{ReportInterval: 50 * time.Millisecond}
+	li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis, opts: lisOpts}
+	addr := setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)
+	r.InitialState(resolver.State{Addresses: []resolver.Address{addr}})
+	dopts := []grpc.DialOption{
+		grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`),
+		grpc.WithResolvers(r),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+	cc, err := grpc.NewClient("whatever:///whatever", dopts...)
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	defer cc.Close()
+	cc.Connect()
+	// Set a few metrics and wait for them on the client side.
+	smr.SetCPUUtilization(10)
+	smr.SetMemoryUtilization(0.1)
+	smr.SetNamedUtilization("bob", 0.555)
+	loadReportWant := &v3orcapb.OrcaLoadReport{
+		CpuUtilization: 10,
+		MemUtilization: 0.1,
+		Utilization:    map[string]float64{"bob": 0.555},
+	}
+
+testReport:
+	for {
+		select {
+		case r := <-oobLis.loadReportCh:
+			t.Log("Load report received: ", r)
+			if proto.Equal(r, loadReportWant) {
+				// Success!
+				break testReport
+			}
+		case <-ctx.Done():
+			t.Fatalf("timed out waiting for load report: %v", loadReportWant)
+		}
+	}
+
+	// Change and add metrics and wait for them on the client side.
+	smr.SetCPUUtilization(0.5)
+	smr.SetMemoryUtilization(0.2)
+	smr.SetNamedUtilization("mary", 0.321)
+	loadReportWant = &v3orcapb.OrcaLoadReport{
+		CpuUtilization: 0.5,
+		MemUtilization: 0.2,
+		Utilization:    map[string]float64{"bob": 0.555, "mary": 0.321},
+	}
+
+	for {
+		select {
+		case r := <-oobLis.loadReportCh:
+			t.Log("Load report received: ", r)
+			if proto.Equal(r, loadReportWant) {
+				// Success!
+				return
+			}
+		case <-ctx.Done():
+			t.Fatalf("timed out waiting for load report: %v", loadReportWant)
+		}
+	}
+
+}
+
+// fakeORCAService is a simple implementation of an ORCA service that pushes
+// requests it receives from clients to a channel and sends responses from a
+// channel back.  This allows tests to verify the client is sending requests
+// and processing responses properly.
+type fakeORCAService struct {
+	v3orcaservicegrpc.UnimplementedOpenRcaServiceServer
+
+	reqCh  chan *v3orcaservicepb.OrcaLoadReportRequest
+	respCh chan any // either *v3orcapb.OrcaLoadReport or error
+}
+
+func newFakeORCAService() *fakeORCAService {
+	return &fakeORCAService{
+		reqCh:  make(chan *v3orcaservicepb.OrcaLoadReportRequest),
+		respCh: make(chan any),
+	}
+}
+
+func (f *fakeORCAService) close() {
+	close(f.respCh)
+}
+
+func (f *fakeORCAService) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error {
+	select {
+	case f.reqCh <- req:
+	case <-stream.Context().Done():
+		return stream.Context().Err()
+	}
+	for {
+		var resp any
+		select {
+		case resp = <-f.respCh:
+		case <-stream.Context().Done():
+			return stream.Context().Err()
+		}
+
+		if err, ok := resp.(error); ok {
+			return err
+		}
+		if err := stream.Send(resp.(*v3orcapb.OrcaLoadReport)); err != nil {
+			// In the event that a stream error occurs, a new stream will have
+			// been created that was waiting for this response message.  Push
+			// it back onto the channel and return.
+			//
+			// This happens because we range over respCh.  If we changed to
+			// instead select on respCh + stream.Context(), the same situation
+			// could still occur due to a race between noticing the two events,
+			// so such a workaround would still be needed to prevent flakiness.
+			f.respCh <- resp
+			return err
+		}
+	}
+}
+
+// TestProducerBackoff verifies that the ORCA producer applies the proper
+// backoff after stream failures.
+func (s) TestProducerBackoff(t *testing.T) {
+	grpctest.ExpectErrorN("injected error", 4)
+
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Provide a convenient way to expect backoff calls and return a minimal
+	// value.
+	const backoffShouldNotBeCalled = 9999 // Use to assert backoff function is not called.
+	const backoffAllowAny = -1            // Use to ignore any backoff calls.
+	expectedBackoff := atomic.Int32{}
+	expectedBackoff.Store(backoffAllowAny)
+	oldBackoff := internal.DefaultBackoffFunc
+	internal.DefaultBackoffFunc = func(got int) time.Duration {
+		if expectedBackoff.Load() == backoffShouldNotBeCalled {
+			t.Errorf("Unexpected backoff call; parameter = %v", got)
+		} else if expectedBackoff.Load() != backoffAllowAny {
+			if got != int(expectedBackoff.Load()) {
+				t.Errorf("Unexpected backoff received; got %v want %v", got, expectedBackoff.Load())
+			}
+		}
+		return time.Millisecond
+	}
+	defer func() { internal.DefaultBackoffFunc = oldBackoff }()
+
+	// Initialize listener for our ORCA server.
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Register our fake ORCA service.
+	s := grpc.NewServer()
+	fake := newFakeORCAService()
+	defer fake.close()
+	v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake)
+	go s.Serve(lis)
+	defer s.Stop()
+
+	// Define the report interval and a function to wait for it to be sent to
+	// the server.
+	const reportInterval = 123 * time.Second
+	awaitRequest := func(interval time.Duration) {
+		select {
+		case req := <-fake.reqCh:
+			if got := req.GetReportInterval().AsDuration(); got != interval {
+				t.Errorf("Unexpected report interval; got %v want %v", got, interval)
+			}
+		case <-ctx.Done():
+			t.Fatalf("Did not receive client request")
+		}
+	}
+
+	// Create our client with an OOB listener in the LB policy it selects.
+	r := manual.NewBuilderWithScheme("whatever")
+	oobLis := newTestOOBListener()
+
+	lisOpts := orca.OOBListenerOptions{ReportInterval: reportInterval}
+	li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis, opts: lisOpts}
+	r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}})
+	dopts := []grpc.DialOption{
+		grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`),
+		grpc.WithResolvers(r),
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+	cc, err := grpc.NewClient("whatever:///whatever", dopts...)
+	if err != nil {
+		t.Fatalf("grpc.NewClient failed: %v", err)
+	}
+	cc.Connect()
+	defer cc.Close()
+
+	// Define a load report to send and expect the client to see.
+	loadReportWant := &v3orcapb.OrcaLoadReport{
+		CpuUtilization: 10,
+		MemUtilization: 0.1,
+		Utilization:    map[string]float64{"bob": 0.555},
+	}
+
+	// Unblock the fake.
+	awaitRequest(reportInterval)
+	fake.respCh <- loadReportWant
+	select {
+	case r := <-oobLis.loadReportCh:
+		t.Log("Load report received: ", r)
+		if proto.Equal(r, loadReportWant) {
+			// Success!
+			break
+		}
+	case <-ctx.Done():
+		t.Fatalf("timed out waiting for load report: %v", loadReportWant)
+	}
+
+	// The next request should be immediate, since there was a message
+	// received.
+	expectedBackoff.Store(backoffShouldNotBeCalled)
+	fake.respCh <- status.Errorf(codes.Internal, "injected error")
+	awaitRequest(reportInterval)
+
+	// The next requests will need to backoff.
+	expectedBackoff.Store(0)
+	fake.respCh <- status.Errorf(codes.Internal, "injected error")
+	awaitRequest(reportInterval)
+	expectedBackoff.Store(1)
+	fake.respCh <- status.Errorf(codes.Internal, "injected error")
+	awaitRequest(reportInterval)
+	expectedBackoff.Store(2)
+	fake.respCh <- status.Errorf(codes.Internal, "injected error")
+	awaitRequest(reportInterval)
+	// The next request should be immediate, since there was a message
+	// received.
+	expectedBackoff.Store(backoffShouldNotBeCalled)
+
+	// Send another valid response and wait for it on the client.
+	fake.respCh <- loadReportWant
+	select {
+	case r := <-oobLis.loadReportCh:
+		t.Log("Load report received: ", r)
+		if proto.Equal(r, loadReportWant) {
+			// Success!
+			break
+		}
+	case <-ctx.Done():
+		t.Fatalf("timed out waiting for load report: %v", loadReportWant)
+	}
+}
+
+// TestProducerMultipleListeners tests that multiple listeners works as
+// expected in a producer: requesting the proper interval and delivering the
+// update to all listeners.
+func (s) TestProducerMultipleListeners(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+
+	// Provide a convenient way to expect backoff calls and return a minimal
+	// value.
+	oldBackoff := internal.DefaultBackoffFunc
+	internal.DefaultBackoffFunc = func(int) time.Duration {
+		return time.Millisecond
+	}
+	defer func() { internal.DefaultBackoffFunc = oldBackoff }()
+
+	// Initialize listener for our ORCA server.
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Register our fake ORCA service.
+	s := grpc.NewServer()
+	fake := newFakeORCAService()
+	defer fake.close()
+	v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake)
+	go s.Serve(lis)
+	defer s.Stop()
+
+	// Define the report interval and a function to wait for it to be sent to
+	// the server.
+	const reportInterval1 = 123 * time.Second
+	const reportInterval2 = 234 * time.Second
+	const reportInterval3 = 56 * time.Second
+	awaitRequest := func(interval time.Duration) {
+		select {
+		case req := <-fake.reqCh:
+			if got := req.GetReportInterval().AsDuration(); got != interval {
+				t.Errorf("Unexpected report interval; got %v want %v", got, interval)
+			}
+		case <-ctx.Done():
+			t.Fatalf("Did not receive client request")
+		}
+	}
+
+	// Create our client with an OOB listener in the LB policy it selects.
+	r := manual.NewBuilderWithScheme("whatever")
+	oobLis1 := newTestOOBListener()
+	lisOpts1 := orca.OOBListenerOptions{ReportInterval: reportInterval1}
+	li := &listenerInfo{scChan: make(chan balancer.SubConn, 1), listener: oobLis1, opts: lisOpts1}
+	r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}})
+	cc, err := grpc.NewClient("whatever:///whatever", grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient() failed: %v", err)
+	}
+	cc.Connect()
+	defer cc.Close()
+
+	// Ensure the OOB listener is stopped before the client is closed to avoid
+	// a potential irrelevant error in the logs.
+	defer oobLis1.Stop()
+
+	oobLis2 := newTestOOBListener()
+	lisOpts2 := orca.OOBListenerOptions{ReportInterval: reportInterval2}
+
+	oobLis3 := newTestOOBListener()
+	lisOpts3 := orca.OOBListenerOptions{ReportInterval: reportInterval3}
+
+	// Define a load report to send and expect the client to see.
+	loadReportWant := &v3orcapb.OrcaLoadReport{
+		CpuUtilization: 10,
+		MemUtilization: 0.1,
+		Utilization:    map[string]float64{"bob": 0.555},
+	}
+
+	// Receive reports and update counts for the three listeners.
+	var reportsMu sync.Mutex
+	var reportsReceived1, reportsReceived2, reportsReceived3 int
+	go func() {
+		for {
+			select {
+			case r := <-oobLis1.loadReportCh:
+				t.Log("Load report 1 received: ", r)
+				if !proto.Equal(r, loadReportWant) {
+					t.Errorf("Unexpected report received: %+v", r)
+				}
+				reportsMu.Lock()
+				reportsReceived1++
+				reportsMu.Unlock()
+			case r := <-oobLis2.loadReportCh:
+				t.Log("Load report 2 received: ", r)
+				if !proto.Equal(r, loadReportWant) {
+					t.Errorf("Unexpected report received: %+v", r)
+				}
+				reportsMu.Lock()
+				reportsReceived2++
+				reportsMu.Unlock()
+			case r := <-oobLis3.loadReportCh:
+				t.Log("Load report 3 received: ", r)
+				if !proto.Equal(r, loadReportWant) {
+					t.Errorf("Unexpected report received: %+v", r)
+				}
+				reportsMu.Lock()
+				reportsReceived3++
+				reportsMu.Unlock()
+			case <-ctx.Done():
+				// Test has ended; exit
+				return
+			}
+		}
+	}()
+
+	// checkReports is a helper function to check the report counts for the three listeners.
+	checkReports := func(r1, r2, r3 int) {
+		t.Helper()
+		for ctx.Err() == nil {
+			reportsMu.Lock()
+			if r1 == reportsReceived1 && r2 == reportsReceived2 && r3 == reportsReceived3 {
+				// Success!
+				reportsMu.Unlock()
+				return
+			}
+			if reportsReceived1 > r1 || reportsReceived2 > r2 || reportsReceived3 > r3 {
+				reportsMu.Unlock()
+				t.Fatalf("received excess reports. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3)
+				return
+			}
+			reportsMu.Unlock()
+			time.Sleep(10 * time.Millisecond)
+		}
+		t.Fatalf("timed out waiting for reports received. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3)
+	}
+
+	// Only 1 listener; expect reportInterval1 to be used and expect the report
+	// to be sent to the listener.
+	awaitRequest(reportInterval1)
+	fake.respCh <- loadReportWant
+	checkReports(1, 0, 0)
+
+	sc := <-li.scChan
+	// Register listener 2 with a less frequent interval; no need to recreate
+	// stream.  Report should go to both listeners.
+	oobLis2.cleanup = orca.RegisterOOBListener(sc, oobLis2, lisOpts2)
+	fake.respCh <- loadReportWant
+	checkReports(2, 1, 0)
+
+	// Register listener 3 with a more frequent interval; stream is recreated
+	// with this interval.  The next report will go to all three listeners.
+	oobLis3.cleanup = orca.RegisterOOBListener(sc, oobLis3, lisOpts3)
+	awaitRequest(reportInterval3)
+	fake.respCh <- loadReportWant
+	checkReports(3, 2, 1)
+
+	// Another report without a change in listeners should go to all three listeners.
+	fake.respCh <- loadReportWant
+	checkReports(4, 3, 2)
+
+	// Stop listener 2.  This does not affect the interval as listener 3 is
+	// still the shortest.  The next update goes to listeners 1 and 3.
+	oobLis2.Stop()
+	fake.respCh <- loadReportWant
+	checkReports(5, 3, 3)
+
+	// Stop listener 3.  This makes the interval longer.  Reports should only
+	// go to listener 1 now.
+	oobLis3.Stop()
+	awaitRequest(reportInterval1)
+	fake.respCh <- loadReportWant
+	checkReports(6, 3, 3)
+	// Another report without a change in listeners should go to the first listener.
+	fake.respCh <- loadReportWant
+	checkReports(7, 3, 3)
+}

orca/server_metrics.go

@@ -0,0 +1,352 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca
+
+import (
+	"sync/atomic"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+)
+
+// ServerMetrics is the data returned from a server to a client to describe the
+// current state of the server and/or the cost of a request when used per-call.
+type ServerMetrics struct {
+	CPUUtilization float64 // CPU utilization: [0, inf); unset=-1
+	MemUtilization float64 // Memory utilization: [0, 1.0]; unset=-1
+	AppUtilization float64 // Application utilization: [0, inf); unset=-1
+	QPS            float64 // queries per second: [0, inf); unset=-1
+	EPS            float64 // errors per second: [0, inf); unset=-1
+
+	// The following maps must never be nil.
+
+	Utilization  map[string]float64 // Custom fields: [0, 1.0]
+	RequestCost  map[string]float64 // Custom fields: [0, inf); not sent OOB
+	NamedMetrics map[string]float64 // Custom fields: [0, inf); not sent OOB
+}
+
+// toLoadReportProto dumps sm as an OrcaLoadReport proto.
+func (sm *ServerMetrics) toLoadReportProto() *v3orcapb.OrcaLoadReport {
+	ret := &v3orcapb.OrcaLoadReport{
+		Utilization:  sm.Utilization,
+		RequestCost:  sm.RequestCost,
+		NamedMetrics: sm.NamedMetrics,
+	}
+	if sm.CPUUtilization != -1 {
+		ret.CpuUtilization = sm.CPUUtilization
+	}
+	if sm.MemUtilization != -1 {
+		ret.MemUtilization = sm.MemUtilization
+	}
+	if sm.AppUtilization != -1 {
+		ret.ApplicationUtilization = sm.AppUtilization
+	}
+	if sm.QPS != -1 {
+		ret.RpsFractional = sm.QPS
+	}
+	if sm.EPS != -1 {
+		ret.Eps = sm.EPS
+	}
+	return ret
+}
+
+// merge merges o into sm, overwriting any values present in both.
+func (sm *ServerMetrics) merge(o *ServerMetrics) {
+	mergeMap(sm.Utilization, o.Utilization)
+	mergeMap(sm.RequestCost, o.RequestCost)
+	mergeMap(sm.NamedMetrics, o.NamedMetrics)
+	if o.CPUUtilization != -1 {
+		sm.CPUUtilization = o.CPUUtilization
+	}
+	if o.MemUtilization != -1 {
+		sm.MemUtilization = o.MemUtilization
+	}
+	if o.AppUtilization != -1 {
+		sm.AppUtilization = o.AppUtilization
+	}
+	if o.QPS != -1 {
+		sm.QPS = o.QPS
+	}
+	if o.EPS != -1 {
+		sm.EPS = o.EPS
+	}
+}
+
+func mergeMap(a, b map[string]float64) {
+	for k, v := range b {
+		a[k] = v
+	}
+}
+
+// ServerMetricsRecorder allows for recording and providing out of band server
+// metrics.
+type ServerMetricsRecorder interface {
+	ServerMetricsProvider
+
+	// SetCPUUtilization sets the CPU utilization server metric.  Must be
+	// greater than zero.
+	SetCPUUtilization(float64)
+	// DeleteCPUUtilization deletes the CPU utilization server metric to
+	// prevent it from being sent.
+	DeleteCPUUtilization()
+
+	// SetMemoryUtilization sets the memory utilization server metric.  Must be
+	// in the range [0, 1].
+	SetMemoryUtilization(float64)
+	// DeleteMemoryUtilization deletes the memory utilization server metric to
+	// prevent it from being sent.
+	DeleteMemoryUtilization()
+
+	// SetApplicationUtilization sets the application utilization server
+	// metric.  Must be greater than zero.
+	SetApplicationUtilization(float64)
+	// DeleteApplicationUtilization deletes the application utilization server
+	// metric to prevent it from being sent.
+	DeleteApplicationUtilization()
+
+	// SetQPS sets the Queries Per Second server metric.  Must be greater than
+	// zero.
+	SetQPS(float64)
+	// DeleteQPS deletes the Queries Per Second server metric to prevent it
+	// from being sent.
+	DeleteQPS()
+
+	// SetEPS sets the Errors Per Second server metric.  Must be greater than
+	// zero.
+	SetEPS(float64)
+	// DeleteEPS deletes the Errors Per Second server metric to prevent it from
+	// being sent.
+	DeleteEPS()
+
+	// SetNamedUtilization sets the named utilization server metric for the
+	// name provided.  val must be in the range [0, 1].
+	SetNamedUtilization(name string, val float64)
+	// DeleteNamedUtilization deletes the named utilization server metric for
+	// the name provided to prevent it from being sent.
+	DeleteNamedUtilization(name string)
+}
+
+type serverMetricsRecorder struct {
+	state atomic.Pointer[ServerMetrics] // the current metrics
+}
+
+// NewServerMetricsRecorder returns an in-memory store for ServerMetrics and
+// allows for safe setting and retrieving of ServerMetrics.  Also implements
+// ServerMetricsProvider for use with NewService.
+func NewServerMetricsRecorder() ServerMetricsRecorder {
+	return newServerMetricsRecorder()
+}
+
+func newServerMetricsRecorder() *serverMetricsRecorder {
+	s := new(serverMetricsRecorder)
+	s.state.Store(&ServerMetrics{
+		CPUUtilization: -1,
+		MemUtilization: -1,
+		AppUtilization: -1,
+		QPS:            -1,
+		EPS:            -1,
+		Utilization:    make(map[string]float64),
+		RequestCost:    make(map[string]float64),
+		NamedMetrics:   make(map[string]float64),
+	})
+	return s
+}
+
+// ServerMetrics returns a copy of the current ServerMetrics.
+func (s *serverMetricsRecorder) ServerMetrics() *ServerMetrics {
+	return copyServerMetrics(s.state.Load())
+}
+
+func copyMap(m map[string]float64) map[string]float64 {
+	ret := make(map[string]float64, len(m))
+	for k, v := range m {
+		ret[k] = v
+	}
+	return ret
+}
+
+func copyServerMetrics(sm *ServerMetrics) *ServerMetrics {
+	return &ServerMetrics{
+		CPUUtilization: sm.CPUUtilization,
+		MemUtilization: sm.MemUtilization,
+		AppUtilization: sm.AppUtilization,
+		QPS:            sm.QPS,
+		EPS:            sm.EPS,
+		Utilization:    copyMap(sm.Utilization),
+		RequestCost:    copyMap(sm.RequestCost),
+		NamedMetrics:   copyMap(sm.NamedMetrics),
+	}
+}
+
+// SetCPUUtilization records a measurement for the CPU utilization metric.
+func (s *serverMetricsRecorder) SetCPUUtilization(val float64) {
+	if val < 0 {
+		if logger.V(2) {
+			logger.Infof("Ignoring CPU Utilization value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.CPUUtilization = val
+	s.state.Store(smCopy)
+}
+
+// DeleteCPUUtilization deletes the relevant server metric to prevent it from
+// being sent.
+func (s *serverMetricsRecorder) DeleteCPUUtilization() {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.CPUUtilization = -1
+	s.state.Store(smCopy)
+}
+
+// SetMemoryUtilization records a measurement for the memory utilization metric.
+func (s *serverMetricsRecorder) SetMemoryUtilization(val float64) {
+	if val < 0 || val > 1 {
+		if logger.V(2) {
+			logger.Infof("Ignoring Memory Utilization value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.MemUtilization = val
+	s.state.Store(smCopy)
+}
+
+// DeleteMemoryUtilization deletes the relevant server metric to prevent it
+// from being sent.
+func (s *serverMetricsRecorder) DeleteMemoryUtilization() {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.MemUtilization = -1
+	s.state.Store(smCopy)
+}
+
+// SetApplicationUtilization records a measurement for a generic utilization
+// metric.
+func (s *serverMetricsRecorder) SetApplicationUtilization(val float64) {
+	if val < 0 {
+		if logger.V(2) {
+			logger.Infof("Ignoring Application Utilization value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.AppUtilization = val
+	s.state.Store(smCopy)
+}
+
+// DeleteApplicationUtilization deletes the relevant server metric to prevent
+// it from being sent.
+func (s *serverMetricsRecorder) DeleteApplicationUtilization() {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.AppUtilization = -1
+	s.state.Store(smCopy)
+}
+
+// SetQPS records a measurement for the QPS metric.
+func (s *serverMetricsRecorder) SetQPS(val float64) {
+	if val < 0 {
+		if logger.V(2) {
+			logger.Infof("Ignoring QPS value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.QPS = val
+	s.state.Store(smCopy)
+}
+
+// DeleteQPS deletes the relevant server metric to prevent it from being sent.
+func (s *serverMetricsRecorder) DeleteQPS() {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.QPS = -1
+	s.state.Store(smCopy)
+}
+
+// SetEPS records a measurement for the EPS metric.
+func (s *serverMetricsRecorder) SetEPS(val float64) {
+	if val < 0 {
+		if logger.V(2) {
+			logger.Infof("Ignoring EPS value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.EPS = val
+	s.state.Store(smCopy)
+}
+
+// DeleteEPS deletes the relevant server metric to prevent it from being sent.
+func (s *serverMetricsRecorder) DeleteEPS() {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.EPS = -1
+	s.state.Store(smCopy)
+}
+
+// SetNamedUtilization records a measurement for a utilization metric uniquely
+// identifiable by name.
+func (s *serverMetricsRecorder) SetNamedUtilization(name string, val float64) {
+	if val < 0 || val > 1 {
+		if logger.V(2) {
+			logger.Infof("Ignoring Named Utilization value out of range: %v", val)
+		}
+		return
+	}
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.Utilization[name] = val
+	s.state.Store(smCopy)
+}
+
+// DeleteNamedUtilization deletes any previously recorded measurement for a
+// utilization metric uniquely identifiable by name.
+func (s *serverMetricsRecorder) DeleteNamedUtilization(name string) {
+	smCopy := copyServerMetrics(s.state.Load())
+	delete(smCopy.Utilization, name)
+	s.state.Store(smCopy)
+}
+
+// SetRequestCost records a measurement for a utilization metric uniquely
+// identifiable by name.
+func (s *serverMetricsRecorder) SetRequestCost(name string, val float64) {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.RequestCost[name] = val
+	s.state.Store(smCopy)
+}
+
+// DeleteRequestCost deletes any previously recorded measurement for a
+// utilization metric uniquely identifiable by name.
+func (s *serverMetricsRecorder) DeleteRequestCost(name string) {
+	smCopy := copyServerMetrics(s.state.Load())
+	delete(smCopy.RequestCost, name)
+	s.state.Store(smCopy)
+}
+
+// SetNamedMetric records a measurement for a utilization metric uniquely
+// identifiable by name.
+func (s *serverMetricsRecorder) SetNamedMetric(name string, val float64) {
+	smCopy := copyServerMetrics(s.state.Load())
+	smCopy.NamedMetrics[name] = val
+	s.state.Store(smCopy)
+}
+
+// DeleteNamedMetric deletes any previously recorded measurement for a
+// utilization metric uniquely identifiable by name.
+func (s *serverMetricsRecorder) DeleteNamedMetric(name string) {
+	smCopy := copyServerMetrics(s.state.Load())
+	delete(smCopy.NamedMetrics, name)
+	s.state.Store(smCopy)
+}

orca/server_metrics_test.go

@@ -0,0 +1,175 @@
+/*
+ *
+ * Copyright 2023 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc/internal/grpctest"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestServerMetrics_Setters(t *testing.T) {
+	smr := NewServerMetricsRecorder()
+
+	smr.SetCPUUtilization(0.1)
+	smr.SetMemoryUtilization(0.2)
+	smr.SetApplicationUtilization(0.3)
+	smr.SetQPS(0.4)
+	smr.SetEPS(0.5)
+	smr.SetNamedUtilization("x", 0.6)
+
+	want := &ServerMetrics{
+		CPUUtilization: 0.1,
+		MemUtilization: 0.2,
+		AppUtilization: 0.3,
+		QPS:            0.4,
+		EPS:            0.5,
+		Utilization:    map[string]float64{"x": 0.6},
+		NamedMetrics:   map[string]float64{},
+		RequestCost:    map[string]float64{},
+	}
+
+	got := smr.ServerMetrics()
+	if d := cmp.Diff(got, want); d != "" {
+		t.Fatalf("unexpected server metrics: -got +want: %v", d)
+	}
+}
+
+func (s) TestServerMetrics_Deleters(t *testing.T) {
+	smr := NewServerMetricsRecorder()
+
+	smr.SetCPUUtilization(0.1)
+	smr.SetMemoryUtilization(0.2)
+	smr.SetApplicationUtilization(0.3)
+	smr.SetQPS(0.4)
+	smr.SetEPS(0.5)
+	smr.SetNamedUtilization("x", 0.6)
+	smr.SetNamedUtilization("y", 0.7)
+
+	// Now delete everything except named_utilization "y".
+	smr.DeleteCPUUtilization()
+	smr.DeleteMemoryUtilization()
+	smr.DeleteApplicationUtilization()
+	smr.DeleteQPS()
+	smr.DeleteEPS()
+	smr.DeleteNamedUtilization("x")
+
+	want := &ServerMetrics{
+		CPUUtilization: -1,
+		MemUtilization: -1,
+		AppUtilization: -1,
+		QPS:            -1,
+		EPS:            -1,
+		Utilization:    map[string]float64{"y": 0.7},
+		NamedMetrics:   map[string]float64{},
+		RequestCost:    map[string]float64{},
+	}
+
+	got := smr.ServerMetrics()
+	if d := cmp.Diff(got, want); d != "" {
+		t.Fatalf("unexpected server metrics: -got +want: %v", d)
+	}
+}
+
+func (s) TestServerMetrics_Setters_Range(t *testing.T) {
+	smr := NewServerMetricsRecorder()
+
+	smr.SetCPUUtilization(0.1)
+	smr.SetMemoryUtilization(0.2)
+	smr.SetApplicationUtilization(0.3)
+	smr.SetQPS(0.4)
+	smr.SetEPS(0.5)
+	smr.SetNamedUtilization("x", 0.6)
+
+	// Negatives for all these fields should be ignored.
+	smr.SetCPUUtilization(-2)
+	smr.SetMemoryUtilization(-3)
+	smr.SetApplicationUtilization(-4)
+	smr.SetQPS(-0.1)
+	smr.SetEPS(-0.6)
+	smr.SetNamedUtilization("x", -2)
+
+	// Memory and named utilizations over 1 are ignored.
+	smr.SetMemoryUtilization(1.1)
+	smr.SetNamedUtilization("x", 1.1)
+
+	want := &ServerMetrics{
+		CPUUtilization: 0.1,
+		MemUtilization: 0.2,
+		AppUtilization: 0.3,
+		QPS:            0.4,
+		EPS:            0.5,
+		Utilization:    map[string]float64{"x": 0.6},
+		NamedMetrics:   map[string]float64{},
+		RequestCost:    map[string]float64{},
+	}
+
+	got := smr.ServerMetrics()
+	if d := cmp.Diff(got, want); d != "" {
+		t.Fatalf("unexpected server metrics: -got +want: %v", d)
+	}
+}
+
+func (s) TestServerMetrics_Merge(t *testing.T) {
+	sm1 := &ServerMetrics{
+		CPUUtilization: 0.1,
+		MemUtilization: 0.2,
+		AppUtilization: 0.3,
+		QPS:            -1,
+		EPS:            0,
+		Utilization:    map[string]float64{"x": 0.6},
+		NamedMetrics:   map[string]float64{"y": 0.2},
+		RequestCost:    map[string]float64{"a": 0.1},
+	}
+
+	sm2 := &ServerMetrics{
+		CPUUtilization: -1,
+		AppUtilization: 0,
+		QPS:            0.9,
+		EPS:            20,
+		Utilization:    map[string]float64{"x": 0.5, "y": 0.4},
+		NamedMetrics:   map[string]float64{"x": 0.1},
+		RequestCost:    map[string]float64{"a": 0.2},
+	}
+
+	want := &ServerMetrics{
+		CPUUtilization: 0.1,
+		MemUtilization: 0,
+		AppUtilization: 0,
+		QPS:            0.9,
+		EPS:            20,
+		Utilization:    map[string]float64{"x": 0.5, "y": 0.4},
+		NamedMetrics:   map[string]float64{"x": 0.1, "y": 0.2},
+		RequestCost:    map[string]float64{"a": 0.2},
+	}
+
+	sm1.merge(sm2)
+	if d := cmp.Diff(sm1, want); d != "" {
+		t.Fatalf("unexpected server metrics: -got +want: %v", d)
+	}
+}

orca/service.go

@@ -0,0 +1,161 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca
+
+import (
+	"fmt"
+	"time"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/internal"
+	ointernal "google.golang.org/grpc/orca/internal"
+	"google.golang.org/grpc/status"
+
+	v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3"
+	v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3"
+)
+
+func init() {
+	ointernal.AllowAnyMinReportingInterval = func(so *ServiceOptions) {
+		so.allowAnyMinReportingInterval = true
+	}
+	internal.ORCAAllowAnyMinReportingInterval = ointernal.AllowAnyMinReportingInterval
+}
+
+// minReportingInterval is the absolute minimum value supported for
+// out-of-band metrics reporting from the ORCA service implementation
+// provided by the orca package.
+const minReportingInterval = 30 * time.Second
+
+// Service provides an implementation of the OpenRcaService as defined in the
+// [ORCA] service protos. Instances of this type must be created via calls to
+// Register() or NewService().
+//
+// Server applications can use the SetXxx() and DeleteXxx() methods to record
+// measurements corresponding to backend metrics, which eventually get pushed to
+// clients who have initiated the SteamCoreMetrics streaming RPC.
+//
+// [ORCA]: https://github.com/cncf/xds/blob/main/xds/service/orca/v3/orca.proto
+type Service struct {
+	v3orcaservicegrpc.UnimplementedOpenRcaServiceServer
+
+	// Minimum reporting interval, as configured by the user, or the default.
+	minReportingInterval time.Duration
+
+	smProvider ServerMetricsProvider
+}
+
+// ServiceOptions contains options to configure the ORCA service implementation.
+type ServiceOptions struct {
+	// ServerMetricsProvider is the provider to be used by the service for
+	// reporting OOB server metrics to clients.  Typically obtained via
+	// NewServerMetricsRecorder.  This field is required.
+	ServerMetricsProvider ServerMetricsProvider
+
+	// MinReportingInterval sets the lower bound for how often out-of-band
+	// metrics are reported on the streaming RPC initiated by the client. If
+	// unspecified, negative or less than the default value of 30s, the default
+	// is used. Clients may request a higher value as part of the
+	// StreamCoreMetrics streaming RPC.
+	MinReportingInterval time.Duration
+
+	// Allow a minReportingInterval which is less than the default of 30s.
+	// Used for testing purposes only.
+	allowAnyMinReportingInterval bool
+}
+
+// A ServerMetricsProvider provides ServerMetrics upon request.
+type ServerMetricsProvider interface {
+	// ServerMetrics returns the current set of server metrics.  It should
+	// return a read-only, immutable copy of the data that is active at the
+	// time of the call.
+	ServerMetrics() *ServerMetrics
+}
+
+// NewService creates a new ORCA service implementation configured using the
+// provided options.
+func NewService(opts ServiceOptions) (*Service, error) {
+	// The default minimum supported reporting interval value can be overridden
+	// for testing purposes through the orca internal package.
+	if opts.ServerMetricsProvider == nil {
+		return nil, fmt.Errorf("ServerMetricsProvider not specified")
+	}
+	if !opts.allowAnyMinReportingInterval {
+		if opts.MinReportingInterval < 0 || opts.MinReportingInterval < minReportingInterval {
+			opts.MinReportingInterval = minReportingInterval
+		}
+	}
+	service := &Service{
+		minReportingInterval: opts.MinReportingInterval,
+		smProvider:           opts.ServerMetricsProvider,
+	}
+	return service, nil
+}
+
+// Register creates a new ORCA service implementation configured using the
+// provided options and registers the same on the provided grpc Server.
+func Register(s grpc.ServiceRegistrar, opts ServiceOptions) error {
+	service, err := NewService(opts)
+	if err != nil {
+		return err
+	}
+	v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, service)
+	return nil
+}
+
+// determineReportingInterval determines the reporting interval for out-of-band
+// metrics. If the reporting interval is not specified in the request, or is
+// negative or is less than the configured minimum (via
+// ServiceOptions.MinReportingInterval), the latter is used. Else the value from
+// the incoming request is used.
+func (s *Service) determineReportingInterval(req *v3orcaservicepb.OrcaLoadReportRequest) time.Duration {
+	if req.GetReportInterval() == nil {
+		return s.minReportingInterval
+	}
+	dur := req.GetReportInterval().AsDuration()
+	if dur < s.minReportingInterval {
+		logger.Warningf("Received reporting interval %q is less than configured minimum: %v. Using minimum", dur, s.minReportingInterval)
+		return s.minReportingInterval
+	}
+	return dur
+}
+
+func (s *Service) sendMetricsResponse(stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error {
+	return stream.Send(s.smProvider.ServerMetrics().toLoadReportProto())
+}
+
+// StreamCoreMetrics streams custom backend metrics injected by the server
+// application.
+func (s *Service) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error {
+	ticker := time.NewTicker(s.determineReportingInterval(req))
+	defer ticker.Stop()
+
+	for {
+		if err := s.sendMetricsResponse(stream); err != nil {
+			return err
+		}
+		// Send a response containing the currently recorded metrics
+		select {
+		case <-stream.Context().Done():
+			return status.Error(codes.Canceled, "Stream has ended.")
+		case <-ticker.C:
+		}
+	}
+}

orca/service_test.go

@@ -0,0 +1,187 @@
+/*
+ *
+ * Copyright 2022 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package orca_test
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/internal/pretty"
+	"google.golang.org/grpc/internal/stubserver"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/orca"
+	"google.golang.org/grpc/orca/internal"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/durationpb"
+
+	v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3"
+	v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3"
+	v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3"
+	testgrpc "google.golang.org/grpc/interop/grpc_testing"
+	testpb "google.golang.org/grpc/interop/grpc_testing"
+)
+
+const requestsMetricKey = "test-service-requests"
+
+// TestE2E_CustomBackendMetrics_OutOfBand tests the injection of out-of-band
+// custom backend metrics from the server application, and verifies that
+// expected load reports are received at the client.
+//
+// TODO: Change this test to use the client API, when ready, to read the
+// out-of-band metrics pushed by the server.
+func (s) TestE2E_CustomBackendMetrics_OutOfBand(t *testing.T) {
+	lis, err := testutils.LocalTCPListener()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Override the min reporting interval in the internal package.
+	const shortReportingInterval = 10 * time.Millisecond
+	smr := orca.NewServerMetricsRecorder()
+	opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr}
+	internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts)
+
+	var requests atomic.Int64
+
+	stub := &stubserver.StubServer{
+		Listener: lis,
+		UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
+			newRequests := requests.Add(1)
+
+			smr.SetNamedUtilization(requestsMetricKey, float64(newRequests)*0.01)
+			smr.SetCPUUtilization(50.0)
+			smr.SetMemoryUtilization(0.9)
+			smr.SetApplicationUtilization(1.2)
+			return &testpb.SimpleResponse{}, nil
+		},
+		EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) {
+			smr.DeleteNamedUtilization(requestsMetricKey)
+			smr.SetCPUUtilization(0)
+			smr.SetMemoryUtilization(0)
+			smr.DeleteApplicationUtilization()
+			return &testpb.Empty{}, nil
+		},
+	}
+
+	// Assign the gRPC server to the stub server and start serving.
+	stub.S = grpc.NewServer()
+	// Register the OpenRCAService with a very short metrics reporting interval.
+	if err := orca.Register(stub.S, opts); err != nil {
+		t.Fatalf("orca.EnableOutOfBandMetricsReportingForTesting() failed: %v", err)
+	}
+	stubserver.StartTestService(t, stub)
+	defer stub.S.Stop()
+	t.Logf("Started gRPC server at %s...", lis.Addr().String())
+
+	// Dial the test server.
+	cc, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		t.Fatalf("grpc.NewClient(%s) failed: %v", lis.Addr().String(), err)
+	}
+	defer cc.Close()
+
+	// Spawn a goroutine which sends 20 unary RPCs to the stub server. This
+	// will trigger the injection of custom backend metrics from the
+	// stubServer.
+	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
+	defer cancel()
+	testStub := testgrpc.NewTestServiceClient(cc)
+	const numRequests = 20
+	errCh := make(chan error, 1)
+	go func() {
+		for i := 0; i < numRequests; i++ {
+			if _, err := testStub.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
+				errCh <- fmt.Errorf("UnaryCall failed: %v", err)
+				return
+			}
+			time.Sleep(time.Millisecond)
+		}
+		errCh <- nil
+	}()
+
+	// Start the server streaming RPC to receive custom backend metrics.
+	oobStub := v3orcaservicegrpc.NewOpenRcaServiceClient(cc)
+	stream, err := oobStub.StreamCoreMetrics(ctx, &v3orcaservicepb.OrcaLoadReportRequest{ReportInterval: durationpb.New(shortReportingInterval)})
+	if err != nil {
+		t.Fatalf("Failed to create a stream for out-of-band metrics")
+	}
+
+	// Wait for the server to push metrics which indicate the completion of all
+	// the unary RPCs made from the above goroutine.
+	for {
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values")
+		case err := <-errCh:
+			if err != nil {
+				t.Fatal(err)
+			}
+		default:
+		}
+
+		wantProto := &v3orcapb.OrcaLoadReport{
+			CpuUtilization:         50.0,
+			MemUtilization:         0.9,
+			ApplicationUtilization: 1.2,
+			Utilization:            map[string]float64{requestsMetricKey: numRequests * 0.01},
+		}
+		gotProto, err := stream.Recv()
+		if err != nil {
+			t.Fatalf("Recv() failed: %v", err)
+		}
+		if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) {
+			t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto))
+			continue
+		}
+		// This means that we received the metrics which we expected.
+		break
+	}
+
+	// The EmptyCall RPC is expected to delete earlier injected metrics.
+	if _, err := testStub.EmptyCall(ctx, &testpb.Empty{}); err != nil {
+		t.Fatalf("EmptyCall failed: %v", err)
+	}
+	// Wait for the server to push empty metrics which indicate the processing
+	// of the above EmptyCall RPC.
+	for {
+		select {
+		case <-ctx.Done():
+			t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values")
+		default:
+		}
+
+		wantProto := &v3orcapb.OrcaLoadReport{}
+		gotProto, err := stream.Recv()
+		if err != nil {
+			t.Fatalf("Recv() failed: %v", err)
+		}
+		if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) {
+			t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto))
+			continue
+		}
+		// This means that we received the metrics which we expected.
+		break
+	}
+}