snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
Testing / vet-proto (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Stale bot / stale (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-15 20:40:15 +05:00
commit e241b33f26
1365 changed files with 339570 additions and 0 deletions

273
gcp/observability/config.go Normal file
View File

@@ -0,0 +1,273 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package observability
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
gcplogging "cloud.google.com/go/logging"
"golang.org/x/oauth2/google"
"google.golang.org/grpc/internal/envconfig"
)
const envProjectID = "GOOGLE_CLOUD_PROJECT"
// fetchDefaultProjectID fetches the default GCP project id from environment.
func fetchDefaultProjectID(ctx context.Context) string {
// Step 1: Check ENV var
if s := os.Getenv(envProjectID); s != "" {
logger.Infof("Found project ID from env %v: %v", envProjectID, s)
return s
}
// Step 2: Check default credential
credentials, err := google.FindDefaultCredentials(ctx, gcplogging.WriteScope)
if err != nil {
logger.Infof("Failed to locate Google Default Credential: %v", err)
return ""
}
if credentials.ProjectID == "" {
logger.Infof("Failed to find project ID in default credential: %v", err)
return ""
}
logger.Infof("Found project ID from Google Default Credential: %v", credentials.ProjectID)
return credentials.ProjectID
}
// validateMethodString validates whether the string passed in is a valid
// pattern.
func validateMethodString(method string) error {
if strings.HasPrefix(method, "/") {
return errors.New("cannot have a leading slash")
}
serviceMethod := strings.Split(method, "/")
if len(serviceMethod) != 2 {
return errors.New("/ must come in between service and method, only one /")
}
if serviceMethod[1] == "" {
return errors.New("method name must be non empty")
}
if serviceMethod[0] == "*" {
return errors.New("cannot have service wildcard * i.e. (*/m)")
}
return nil
}
func validateLogEventMethod(methods []string, exclude bool) error {
for _, method := range methods {
if method == "*" {
if exclude {
return errors.New("cannot have exclude and a '*' wildcard")
}
continue
}
if err := validateMethodString(method); err != nil {
return fmt.Errorf("invalid method string: %v, err: %v", method, err)
}
}
return nil
}
func validateLoggingEvents(config *config) error {
if config.CloudLogging == nil {
return nil
}
for _, clientRPCEvent := range config.CloudLogging.ClientRPCEvents {
if err := validateLogEventMethod(clientRPCEvent.Methods, clientRPCEvent.Exclude); err != nil {
return fmt.Errorf("error in clientRPCEvent method: %v", err)
}
}
for _, serverRPCEvent := range config.CloudLogging.ServerRPCEvents {
if err := validateLogEventMethod(serverRPCEvent.Methods, serverRPCEvent.Exclude); err != nil {
return fmt.Errorf("error in serverRPCEvent method: %v", err)
}
}
return nil
}
// unmarshalAndVerifyConfig unmarshals a json string representing an
// observability config into its internal go format, and also verifies the
// configuration's fields for validity.
func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*config, error) {
var config config
if err := json.Unmarshal(rawJSON, &config); err != nil {
return nil, fmt.Errorf("error parsing observability config: %v", err)
}
if err := validateLoggingEvents(&config); err != nil {
return nil, fmt.Errorf("error parsing observability config: %v", err)
}
if config.CloudTrace != nil && (config.CloudTrace.SamplingRate > 1 || config.CloudTrace.SamplingRate < 0) {
return nil, fmt.Errorf("error parsing observability config: invalid cloud trace sampling rate %v", config.CloudTrace.SamplingRate)
}
logger.Infof("Parsed ObservabilityConfig: %+v", &config)
return &config, nil
}
func parseObservabilityConfig() (*config, error) {
if f := envconfig.ObservabilityConfigFile; f != "" {
if envconfig.ObservabilityConfig != "" {
logger.Warning("Ignoring GRPC_GCP_OBSERVABILITY_CONFIG and using GRPC_GCP_OBSERVABILITY_CONFIG_FILE contents.")
}
content, err := os.ReadFile(f)
if err != nil {
return nil, fmt.Errorf("error reading observability configuration file %q: %v", f, err)
}
return unmarshalAndVerifyConfig(content)
} else if envconfig.ObservabilityConfig != "" {
return unmarshalAndVerifyConfig([]byte(envconfig.ObservabilityConfig))
}
// If the ENV var doesn't exist, do nothing
return nil, nil
}
func ensureProjectIDInObservabilityConfig(ctx context.Context, config *config) error {
if config.ProjectID == "" {
// Try to fetch the GCP project id
projectID := fetchDefaultProjectID(ctx)
if projectID == "" {
return fmt.Errorf("empty destination project ID")
}
config.ProjectID = projectID
}
return nil
}
type clientRPCEvents struct {
// Methods is a list of strings which can select a group of methods. By
// default, the list is empty, matching no methods.
//
// The value of the method is in the form of <service>/<method>.
//
// "*" is accepted as a wildcard for:
// 1. The method name. If the value is <service>/*, it matches all
// methods in the specified service.
// 2. The whole value of the field which matches any <service>/<method>.
// It’s not supported when Exclude is true.
// 3. The * wildcard cannot be used on the service name independently,
// */<method> is not supported.
//
// The service name, when specified, must be the fully qualified service
// name, including the package name.
//
// Examples:
// 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo",
// here “goo” is the package name.
// 2."goo.Foo/*" selects all methods from service "goo.Foo"
// 3. "*" selects all methods from all services.
Methods []string `json:"methods,omitempty"`
// Exclude represents whether the methods denoted by Methods should be
// excluded from logging. The default value is false, meaning the methods
// denoted by Methods are included in the logging. If Exclude is true, the
// wildcard `*` cannot be used as value of an entry in Methods.
Exclude bool `json:"exclude,omitempty"`
// MaxMetadataBytes is the maximum number of bytes of each header to log. If
// the size of the metadata is greater than the defined limit, content past
// the limit will be truncated. The default value is 0.
MaxMetadataBytes int `json:"max_metadata_bytes"`
// MaxMessageBytes is the maximum number of bytes of each message to log. If
// the size of the message is greater than the defined limit, content past
// the limit will be truncated. The default value is 0.
MaxMessageBytes int `json:"max_message_bytes"`
}
type serverRPCEvents struct {
// Methods is a list of strings which can select a group of methods. By
// default, the list is empty, matching no methods.
//
// The value of the method is in the form of <service>/<method>.
//
// "*" is accepted as a wildcard for:
// 1. The method name. If the value is <service>/*, it matches all
// methods in the specified service.
// 2. The whole value of the field which matches any <service>/<method>.
// It’s not supported when Exclude is true.
// 3. The * wildcard cannot be used on the service name independently,
// */<method> is not supported.
//
// The service name, when specified, must be the fully qualified service
// name, including the package name.
//
// Examples:
// 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo",
// here “goo” is the package name.
// 2."goo.Foo/*" selects all methods from service "goo.Foo"
// 3. "*" selects all methods from all services.
Methods []string `json:"methods,omitempty"`
// Exclude represents whether the methods denoted by Methods should be
// excluded from logging. The default value is false, meaning the methods
// denoted by Methods are included in the logging. If Exclude is true, the
// wildcard `*` cannot be used as value of an entry in Methods.
Exclude bool `json:"exclude,omitempty"`
// MaxMetadataBytes is the maximum number of bytes of each header to log. If
// the size of the metadata is greater than the defined limit, content past
// the limit will be truncated. The default value is 0.
MaxMetadataBytes int `json:"max_metadata_bytes"`
// MaxMessageBytes is the maximum number of bytes of each message to log. If
// the size of the message is greater than the defined limit, content past
// the limit will be truncated. The default value is 0.
MaxMessageBytes int `json:"max_message_bytes"`
}
type cloudLogging struct {
// ClientRPCEvents represents the configuration for outgoing RPC's from the
// binary. The client_rpc_events configs are evaluated in text order, the
// first one matched is used. If an RPC doesn't match an entry, it will
// continue on to the next entry in the list.
ClientRPCEvents []clientRPCEvents `json:"client_rpc_events,omitempty"`
// ServerRPCEvents represents the configuration for incoming RPC's to the
// binary. The server_rpc_events configs are evaluated in text order, the
// first one matched is used. If an RPC doesn't match an entry, it will
// continue on to the next entry in the list.
ServerRPCEvents []serverRPCEvents `json:"server_rpc_events,omitempty"`
}
type cloudMonitoring struct{}
type cloudTrace struct {
// SamplingRate is the global setting that controls the probability of an RPC
// being traced. For example, 0.05 means there is a 5% chance for an RPC to
// be traced, 1.0 means trace every call, 0 means don’t start new traces. By
// default, the sampling_rate is 0.
SamplingRate float64 `json:"sampling_rate,omitempty"`
}
type config struct {
// ProjectID is the destination GCP project identifier for uploading log
// entries. If empty, the gRPC Observability plugin will attempt to fetch
// the project_id from the GCP environment variables, or from the default
// credentials. If not found, the observability init functions will return
// an error.
ProjectID string `json:"project_id,omitempty"`
// CloudLogging defines the logging options. If not present, logging is disabled.
CloudLogging *cloudLogging `json:"cloud_logging,omitempty"`
// CloudMonitoring determines whether or not metrics are enabled based on
// whether it is present or not. If present, monitoring will be enabled, if
// not present, monitoring is disabled.
CloudMonitoring *cloudMonitoring `json:"cloud_monitoring,omitempty"`
// CloudTrace defines the tracing options. When present, tracing is enabled
// with default configurations. When absent, the tracing is disabled.
CloudTrace *cloudTrace `json:"cloud_trace,omitempty"`
// Labels are applied to cloud logging, monitoring, and trace.
Labels map[string]string `json:"labels,omitempty"`
}

View File

@@ -0,0 +1,108 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package observability
import (
"context"
"fmt"
"time"
"google.golang.org/api/option"
"google.golang.org/grpc"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/stats/opencensus"
gcplogging "cloud.google.com/go/logging"
)
// cOptsDisableLogTrace are client options for the go client libraries which are
// used to configure connections to GCP exporting backends. These disable global
// dial and server options set by this module, which configure logging, metrics,
// and tracing on all created grpc.ClientConn's and grpc.Server's. These options
// turn on only metrics, and also disable the client libraries behavior of
// plumbing in the older opencensus instrumentation code.
var cOptsDisableLogTrace = []option.ClientOption{
option.WithTelemetryDisabled(),
option.WithGRPCDialOption(internal.DisableGlobalDialOptions.(func() grpc.DialOption)()),
option.WithGRPCDialOption(opencensus.DialOption(opencensus.TraceOptions{
DisableTrace: true,
})),
}
// loggingExporter is the interface of logging exporter for gRPC Observability.
// In future, we might expose this to allow users provide custom exporters. But
// now, it exists for testing purposes.
type loggingExporter interface {
// EmitGrpcLogRecord writes a gRPC LogRecord to cache without blocking.
EmitGcpLoggingEntry(entry gcplogging.Entry)
// Close flushes all pending data and closes the exporter.
Close() error
}
type cloudLoggingExporter struct {
projectID string
client *gcplogging.Client
logger *gcplogging.Logger
}
func newCloudLoggingExporter(ctx context.Context, config *config) (loggingExporter, error) {
c, err := gcplogging.NewClient(ctx, fmt.Sprintf("projects/%v", config.ProjectID), cOptsDisableLogTrace...)
if err != nil {
return nil, fmt.Errorf("failed to create cloudLoggingExporter: %v", err)
}
defer logger.Infof("Successfully created cloudLoggingExporter")
if len(config.Labels) != 0 {
logger.Infof("Adding labels: %+v", config.Labels)
}
return &cloudLoggingExporter{
projectID: config.ProjectID,
client: c,
logger: c.Logger("microservices.googleapis.com/observability/grpc", gcplogging.CommonLabels(config.Labels), gcplogging.BufferedByteLimit(1024*1024*50), gcplogging.DelayThreshold(time.Second*10)),
}, nil
}
func (cle *cloudLoggingExporter) EmitGcpLoggingEntry(entry gcplogging.Entry) {
cle.logger.Log(entry)
if logger.V(2) {
logger.Infof("Uploading event to CloudLogging: %+v", entry)
}
}
func (cle *cloudLoggingExporter) Close() error {
var errFlush, errClose error
if cle.logger != nil {
errFlush = cle.logger.Flush()
}
if cle.client != nil {
errClose = cle.client.Close()
}
if errFlush != nil && errClose != nil {
return fmt.Errorf("failed to close exporter. Flush failed: %v; Close failed: %v", errFlush, errClose)
}
if errFlush != nil {
return errFlush
}
if errClose != nil {
return errClose
}
cle.logger = nil
cle.client = nil
logger.Infof("Closed CloudLogging exporter")
return nil
}

67
gcp/observability/go.mod Normal file
View File

@@ -0,0 +1,67 @@
module google.golang.org/grpc/gcp/observability
go 1.25.0
require (
cloud.google.com/go/logging v1.19.1
contrib.go.opencensus.io/exporter/stackdriver v0.13.15-0.20230702191903-2de6d2748484
github.com/google/go-cmp v0.7.0
github.com/google/uuid v1.6.0
go.opencensus.io v0.24.0
golang.org/x/oauth2 v0.36.0
google.golang.org/api v0.293.0
google.golang.org/grpc v1.83.0
google.golang.org/grpc/stats/opencensus v1.0.0
)
require (
cloud.google.com/go v0.123.0 // indirect
cloud.google.com/go/auth v0.23.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/longrunning v1.2.0 // indirect
cloud.google.com/go/monitoring v1.30.0 // indirect
cloud.google.com/go/trace v1.16.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.43.6 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.37 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.36 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.37 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.38 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.37 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.5.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect
github.com/aws/smithy-go v1.27.8 // indirect
github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.21 // indirect
github.com/googleapis/gax-go/v2 v2.23.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect
go.opentelemetry.io/otel v1.45.0 // indirect
go.opentelemetry.io/otel/metric v1.45.0 // indirect
go.opentelemetry.io/otel/trace v1.45.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/net v0.58.0 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.41.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260817212433-ac3dfec99bb1 // indirect
google.golang.org/protobuf v1.36.12 // indirect
)
replace google.golang.org/grpc => ../..

View File

@@ -0,0 +1,505 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package observability
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"strings"
"time"
gcplogging "cloud.google.com/go/logging"
"github.com/google/uuid"
"go.opencensus.io/trace"
"google.golang.org/grpc"
binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/internal"
iblog "google.golang.org/grpc/internal/binarylog"
"google.golang.org/grpc/internal/grpcutil"
"google.golang.org/grpc/stats/opencensus"
)
var lExporter loggingExporter
var newLoggingExporter = newCloudLoggingExporter
var canonicalString = internal.CanonicalString.(func(codes.Code) string)
// translateMetadata translates the metadata from Binary Logging format to
// its GrpcLogEntry equivalent.
func translateMetadata(m *binlogpb.Metadata) map[string]string {
metadata := make(map[string]string)
for _, entry := range m.GetEntry() {
entryKey := entry.GetKey()
var newVal string
if strings.HasSuffix(entryKey, "-bin") { // bin header
newVal = base64.StdEncoding.EncodeToString(entry.GetValue())
} else { // normal header
newVal = string(entry.GetValue())
}
var oldVal string
var ok bool
if oldVal, ok = metadata[entryKey]; !ok {
metadata[entryKey] = newVal
continue
}
metadata[entryKey] = oldVal + "," + newVal
}
return metadata
}
func setPeerIfPresent(binlogEntry *binlogpb.GrpcLogEntry, grpcLogEntry *grpcLogEntry) {
if binlogEntry.GetPeer() != nil {
grpcLogEntry.Peer.Type = addrType(binlogEntry.GetPeer().GetType())
grpcLogEntry.Peer.Address = binlogEntry.GetPeer().GetAddress()
grpcLogEntry.Peer.IPPort = binlogEntry.GetPeer().GetIpPort()
}
}
var loggerTypeToEventLogger = map[binlogpb.GrpcLogEntry_Logger]loggerType{
binlogpb.GrpcLogEntry_LOGGER_UNKNOWN: loggerUnknown,
binlogpb.GrpcLogEntry_LOGGER_CLIENT: loggerClient,
binlogpb.GrpcLogEntry_LOGGER_SERVER: loggerServer,
}
type eventType int
const (
// eventTypeUnknown is an unknown event type.
eventTypeUnknown eventType = iota
// eventTypeClientHeader is a header sent from client to server.
eventTypeClientHeader
// eventTypeServerHeader is a header sent from server to client.
eventTypeServerHeader
// eventTypeClientMessage is a message sent from client to server.
eventTypeClientMessage
// eventTypeServerMessage is a message sent from server to client.
eventTypeServerMessage
// eventTypeClientHalfClose is a signal that the loggerClient is done sending.
eventTypeClientHalfClose
// eventTypeServerTrailer indicated the end of a gRPC call.
eventTypeServerTrailer
// eventTypeCancel is a signal that the rpc is canceled.
eventTypeCancel
)
func (t eventType) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString(`"`)
switch t {
case eventTypeUnknown:
buffer.WriteString("EVENT_TYPE_UNKNOWN")
case eventTypeClientHeader:
buffer.WriteString("CLIENT_HEADER")
case eventTypeServerHeader:
buffer.WriteString("SERVER_HEADER")
case eventTypeClientMessage:
buffer.WriteString("CLIENT_MESSAGE")
case eventTypeServerMessage:
buffer.WriteString("SERVER_MESSAGE")
case eventTypeClientHalfClose:
buffer.WriteString("CLIENT_HALF_CLOSE")
case eventTypeServerTrailer:
buffer.WriteString("SERVER_TRAILER")
case eventTypeCancel:
buffer.WriteString("CANCEL")
}
buffer.WriteString(`"`)
return buffer.Bytes(), nil
}
type loggerType int
const (
loggerUnknown loggerType = iota
loggerClient
loggerServer
)
func (t loggerType) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString(`"`)
switch t {
case loggerUnknown:
buffer.WriteString("LOGGER_UNKNOWN")
case loggerClient:
buffer.WriteString("CLIENT")
case loggerServer:
buffer.WriteString("SERVER")
}
buffer.WriteString(`"`)
return buffer.Bytes(), nil
}
type payload struct {
Metadata map[string]string `json:"metadata,omitempty"`
// Timeout is the RPC timeout value.
Timeout time.Duration `json:"timeout,omitempty"`
// StatusCode is the gRPC status code.
StatusCode string `json:"statusCode,omitempty"`
// StatusMessage is the gRPC status message.
StatusMessage string `json:"statusMessage,omitempty"`
// StatusDetails is the value of the grpc-status-details-bin metadata key,
// if any. This is always an encoded google.rpc.Status message.
StatusDetails []byte `json:"statusDetails,omitempty"`
// MessageLength is the length of the message.
MessageLength uint32 `json:"messageLength,omitempty"`
// Message is the message of this entry. This is populated in the case of a
// message event.
Message []byte `json:"message,omitempty"`
}
type addrType int
const (
typeUnknown addrType = iota // `json:"TYPE_UNKNOWN"`
ipv4 // `json:"IPV4"`
ipv6 // `json:"IPV6"`
unix // `json:"UNIX"`
)
func (at addrType) MarshalJSON() ([]byte, error) {
buffer := bytes.NewBufferString(`"`)
switch at {
case typeUnknown:
buffer.WriteString("TYPE_UNKNOWN")
case ipv4:
buffer.WriteString("IPV4")
case ipv6:
buffer.WriteString("IPV6")
case unix:
buffer.WriteString("UNIX")
}
buffer.WriteString(`"`)
return buffer.Bytes(), nil
}
type address struct {
// Type is the address type of the address of the peer of the RPC.
Type addrType `json:"type,omitempty"`
// Address is the address of the peer of the RPC.
Address string `json:"address,omitempty"`
// IPPort is the ip and port in string form. It is used only for addrType
// typeIPv4 and typeIPv6.
IPPort uint32 `json:"ipPort,omitempty"`
}
type grpcLogEntry struct {
// CallID is a uuid which uniquely identifies a call. Each call may have
// several log entries. They will all have the same CallID. Nothing is
// guaranteed about their value other than they are unique across different
// RPCs in the same gRPC process.
CallID string `json:"callId,omitempty"`
// SequenceID is the entry sequence ID for this call. The first message has
// a value of 1, to disambiguate from an unset value. The purpose of this
// field is to detect missing entries in environments where durability or
// ordering is not guaranteed.
SequenceID uint64 `json:"sequenceId,omitempty"`
// Type is the type of binary logging event being logged.
Type eventType `json:"type,omitempty"`
// Logger is the entity that generates the log entry.
Logger loggerType `json:"logger,omitempty"`
// Payload is the payload of this log entry.
Payload payload `json:"payload,omitempty"`
// PayloadTruncated is whether the message or metadata field is either
// truncated or emitted due to options specified in the configuration.
PayloadTruncated bool `json:"payloadTruncated,omitempty"`
// Peer is information about the Peer of the RPC.
Peer address `json:"peer,omitempty"`
// A single process may be used to run multiple virtual servers with
// different identities.
// Authority is the name of such a server identify. It is typically a
// portion of the URI in the form of <host> or <host>:<port>.
Authority string `json:"authority,omitempty"`
// ServiceName is the name of the service.
ServiceName string `json:"serviceName,omitempty"`
// MethodName is the name of the RPC method.
MethodName string `json:"methodName,omitempty"`
}
type methodLoggerBuilder interface {
Build(iblog.LogEntryConfig) *binlogpb.GrpcLogEntry
}
type binaryMethodLogger struct {
callID, serviceName, methodName, authority, projectID string
mlb methodLoggerBuilder
exporter loggingExporter
clientSide bool
}
// buildGCPLoggingEntry converts the binary log entry into a gcp logging
// entry.
func (bml *binaryMethodLogger) buildGCPLoggingEntry(ctx context.Context, c iblog.LogEntryConfig) gcplogging.Entry {
binLogEntry := bml.mlb.Build(c)
grpcLogEntry := &grpcLogEntry{
CallID: bml.callID,
SequenceID: binLogEntry.GetSequenceIdWithinCall(),
Logger: loggerTypeToEventLogger[binLogEntry.Logger],
}
switch binLogEntry.GetType() {
case binlogpb.GrpcLogEntry_EVENT_TYPE_UNKNOWN:
grpcLogEntry.Type = eventTypeUnknown
case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER:
grpcLogEntry.Type = eventTypeClientHeader
if binLogEntry.GetClientHeader() != nil {
methodName := binLogEntry.GetClientHeader().MethodName
// Example method name: /grpc.testing.TestService/UnaryCall
if strings.Contains(methodName, "/") {
tokens := strings.Split(methodName, "/")
if len(tokens) == 3 {
// Record service name and method name for all events.
bml.serviceName = tokens[1]
bml.methodName = tokens[2]
} else {
logger.Infof("Malformed method name: %v", methodName)
}
}
bml.authority = binLogEntry.GetClientHeader().GetAuthority()
grpcLogEntry.Payload.Timeout = binLogEntry.GetClientHeader().GetTimeout().AsDuration()
grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetClientHeader().GetMetadata())
}
grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated()
setPeerIfPresent(binLogEntry, grpcLogEntry)
case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER:
grpcLogEntry.Type = eventTypeServerHeader
if binLogEntry.GetServerHeader() != nil {
grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetServerHeader().GetMetadata())
}
grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated()
setPeerIfPresent(binLogEntry, grpcLogEntry)
case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE:
grpcLogEntry.Type = eventTypeClientMessage
grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData()
grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength()
grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated()
case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE:
grpcLogEntry.Type = eventTypeServerMessage
grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData()
grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength()
grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated()
case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE:
grpcLogEntry.Type = eventTypeClientHalfClose
case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER:
grpcLogEntry.Type = eventTypeServerTrailer
grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetTrailer().Metadata)
grpcLogEntry.Payload.StatusCode = canonicalString(codes.Code(binLogEntry.GetTrailer().GetStatusCode()))
grpcLogEntry.Payload.StatusMessage = binLogEntry.GetTrailer().GetStatusMessage()
grpcLogEntry.Payload.StatusDetails = binLogEntry.GetTrailer().GetStatusDetails()
grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated()
setPeerIfPresent(binLogEntry, grpcLogEntry)
case binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL:
grpcLogEntry.Type = eventTypeCancel
}
grpcLogEntry.ServiceName = bml.serviceName
grpcLogEntry.MethodName = bml.methodName
grpcLogEntry.Authority = bml.authority
var sc trace.SpanContext
var ok bool
if bml.clientSide {
// client side span, populated through opencensus trace package.
if span := trace.FromContext(ctx); span != nil {
sc = span.SpanContext()
ok = true
}
} else {
// server side span, populated through stats/opencensus package.
sc, ok = opencensus.SpanContextFromContext(ctx)
}
gcploggingEntry := gcplogging.Entry{
Timestamp: binLogEntry.GetTimestamp().AsTime(),
Severity: 100,
Payload: grpcLogEntry,
}
if ok {
gcploggingEntry.Trace = "projects/" + bml.projectID + "/traces/" + sc.TraceID.String()
gcploggingEntry.SpanID = sc.SpanID.String()
gcploggingEntry.TraceSampled = sc.IsSampled()
}
return gcploggingEntry
}
func (bml *binaryMethodLogger) Log(ctx context.Context, c iblog.LogEntryConfig) {
bml.exporter.EmitGcpLoggingEntry(bml.buildGCPLoggingEntry(ctx, c))
}
type eventConfig struct {
// ServiceMethod has /s/m syntax for fast matching.
ServiceMethod map[string]bool
Services map[string]bool
MatchAll bool
// If true, won't log anything.
Exclude bool
HeaderBytes uint64
MessageBytes uint64
}
type binaryLogger struct {
EventConfigs []eventConfig
projectID string
exporter loggingExporter
clientSide bool
}
func (bl *binaryLogger) GetMethodLogger(methodName string) iblog.MethodLogger {
s, _, err := grpcutil.ParseMethod(methodName)
if err != nil {
logger.Infof("binarylogging: failed to parse %q: %v", methodName, err)
return nil
}
for _, eventConfig := range bl.EventConfigs {
if eventConfig.MatchAll || eventConfig.ServiceMethod[methodName] || eventConfig.Services[s] {
if eventConfig.Exclude {
return nil
}
return &binaryMethodLogger{
exporter: bl.exporter,
mlb: iblog.NewTruncatingMethodLogger(eventConfig.HeaderBytes, eventConfig.MessageBytes),
callID: uuid.NewString(),
projectID: bl.projectID,
clientSide: bl.clientSide,
}
}
}
return nil
}
// parseMethod splits service and method from the input. It expects format
// "service/method".
func parseMethod(method string) (string, string, error) {
pos := strings.Index(method, "/")
if pos < 0 {
// Shouldn't happen, config already validated.
return "", "", errors.New("invalid method name: no / found")
}
return method[:pos], method[pos+1:], nil
}
func registerClientRPCEvents(config *config, exporter loggingExporter) {
clientRPCEvents := config.CloudLogging.ClientRPCEvents
if len(clientRPCEvents) == 0 {
return
}
var eventConfigs []eventConfig
for _, clientRPCEvent := range clientRPCEvents {
eventConfig := eventConfig{
Exclude: clientRPCEvent.Exclude,
HeaderBytes: uint64(clientRPCEvent.MaxMetadataBytes),
MessageBytes: uint64(clientRPCEvent.MaxMessageBytes),
}
for _, method := range clientRPCEvent.Methods {
eventConfig.ServiceMethod = make(map[string]bool)
eventConfig.Services = make(map[string]bool)
if method == "*" {
eventConfig.MatchAll = true
continue
}
s, m, err := parseMethod(method)
if err != nil {
continue
}
if m == "*" {
eventConfig.Services[s] = true
continue
}
eventConfig.ServiceMethod["/"+method] = true
}
eventConfigs = append(eventConfigs, eventConfig)
}
clientSideLogger := &binaryLogger{
EventConfigs: eventConfigs,
exporter: exporter,
projectID: config.ProjectID,
clientSide: true,
}
internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(internal.WithBinaryLogger.(func(bl iblog.Logger) grpc.DialOption)(clientSideLogger))
}
func registerServerRPCEvents(config *config, exporter loggingExporter) {
serverRPCEvents := config.CloudLogging.ServerRPCEvents
if len(serverRPCEvents) == 0 {
return
}
var eventConfigs []eventConfig
for _, serverRPCEvent := range serverRPCEvents {
eventConfig := eventConfig{
Exclude: serverRPCEvent.Exclude,
HeaderBytes: uint64(serverRPCEvent.MaxMetadataBytes),
MessageBytes: uint64(serverRPCEvent.MaxMessageBytes),
}
for _, method := range serverRPCEvent.Methods {
eventConfig.ServiceMethod = make(map[string]bool)
eventConfig.Services = make(map[string]bool)
if method == "*" {
eventConfig.MatchAll = true
continue
}
s, m, err := parseMethod(method)
if err != nil {
continue
}
if m == "*" {
eventConfig.Services[s] = true
continue
}
eventConfig.ServiceMethod["/"+method] = true
}
eventConfigs = append(eventConfigs, eventConfig)
}
serverSideLogger := &binaryLogger{
EventConfigs: eventConfigs,
exporter: exporter,
projectID: config.ProjectID,
clientSide: false,
}
internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(internal.BinaryLogger.(func(bl iblog.Logger) grpc.ServerOption)(serverSideLogger))
}
func startLogging(ctx context.Context, config *config) error {
if config == nil || config.CloudLogging == nil {
return nil
}
var err error
lExporter, err = newLoggingExporter(ctx, config)
if err != nil {
return fmt.Errorf("unable to create CloudLogging exporter: %v", err)
}
registerClientRPCEvents(config, lExporter)
registerServerRPCEvents(config, lExporter)
return nil
}
func stopLogging() {
internal.ClearGlobalDialOptions()
internal.ClearGlobalServerOptions()
if lExporter != nil {
// This Close() call handles the flushing of the logging buffer.
lExporter.Close()
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,92 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
// Package observability implements the tracing, metrics, and logging data
// collection, and provides controlling knobs via a config file.
//
// # Experimental
//
// Notice: This package is EXPERIMENTAL and may be changed or removed in a
// later release.
package observability
import (
"context"
"fmt"
"google.golang.org/grpc/grpclog"
)
var logger = grpclog.Component("observability")
// Start is the opt-in API for gRPC Observability plugin. This function should
// be invoked in the main function, and before creating any gRPC clients or
// servers, otherwise, they might not be instrumented. At high-level, this
// module does the following:
//
// - it loads observability config from environment;
// - it registers default exporters if not disabled by the config;
// - it sets up telemetry collectors (binary logging sink or StatsHandlers).
//
// Note: this method should only be invoked once.
// Note: handle the error
func Start(ctx context.Context) error {
config, err := parseObservabilityConfig()
if err != nil {
return err
}
if config == nil {
return fmt.Errorf("no ObservabilityConfig found")
}
// Set the project ID if it isn't configured manually.
if err = ensureProjectIDInObservabilityConfig(ctx, config); err != nil {
return err
}
// Cleanup any created resources this function created in case this function
// errors.
defer func() {
if err != nil {
End()
}
}()
// Enabling tracing and metrics via OpenCensus
if err = startOpenCensus(config); err != nil {
return fmt.Errorf("failed to instrument OpenCensus: %v", err)
}
if err = startLogging(ctx, config); err != nil {
return fmt.Errorf("failed to start logging: %v", err)
}
// Logging is controlled by the config at methods level.
return nil
}
// End is the clean-up API for gRPC Observability plugin. It is expected to be
// invoked in the main function of the application. The suggested usage is
// "defer observability.End()". This function also flushes data to upstream, and
// cleanup resources.
//
// Note: this method should only be invoked once.
func End() {
stopLogging()
stopOpenCensus()
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,179 @@
/*
*
* Copyright 2022 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package observability
import (
"fmt"
"os"
"strconv"
"time"
"contrib.go.opencensus.io/exporter/stackdriver"
"contrib.go.opencensus.io/exporter/stackdriver/monitoredresource"
"go.opencensus.io/stats/view"
"go.opencensus.io/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/stats/opencensus"
)
var (
// It's a variable instead of const to speed up testing
defaultMetricsReportingInterval = time.Second * 30
defaultViews = []*view.View{
opencensus.ClientStartedRPCsView,
opencensus.ClientCompletedRPCsView,
opencensus.ClientRoundtripLatencyView,
opencensus.ClientSentCompressedMessageBytesPerRPCView,
opencensus.ClientReceivedCompressedMessageBytesPerRPCView,
opencensus.ClientAPILatencyView,
opencensus.ServerStartedRPCsView,
opencensus.ServerCompletedRPCsView,
opencensus.ServerSentCompressedMessageBytesPerRPCView,
opencensus.ServerReceivedCompressedMessageBytesPerRPCView,
opencensus.ServerLatencyView,
}
)
func labelsToMonitoringLabels(labels map[string]string) *stackdriver.Labels {
sdLabels := &stackdriver.Labels{}
for k, v := range labels {
sdLabels.Set(k, v, "")
}
return sdLabels
}
func labelsToTraceAttributes(labels map[string]string) map[string]any {
ta := make(map[string]any, len(labels))
for k, v := range labels {
ta[k] = v
}
return ta
}
type tracingMetricsExporter interface {
trace.Exporter
view.Exporter
Flush()
Close() error
}
var exporter tracingMetricsExporter
var newExporter = newStackdriverExporter
func newStackdriverExporter(config *config) (tracingMetricsExporter, error) {
// Create the Stackdriver exporter, which is shared between tracing and stats
mr := monitoredresource.Autodetect()
logger.Infof("Detected MonitoredResource:: %+v", mr)
var err error
// Custom labels completely overwrite any labels generated in the OpenCensus
// library, including their label that uniquely identifies the process.
// Thus, generate a unique process identifier here to uniquely identify
// process for metrics exporting to function correctly.
metricsLabels := make(map[string]string, len(config.Labels)+1)
for k, v := range config.Labels {
metricsLabels[k] = v
}
metricsLabels["opencensus_task"] = generateUniqueProcessIdentifier()
exporter, err := stackdriver.NewExporter(stackdriver.Options{
ProjectID: config.ProjectID,
MonitoredResource: mr,
DefaultMonitoringLabels: labelsToMonitoringLabels(metricsLabels),
DefaultTraceAttributes: labelsToTraceAttributes(config.Labels),
MonitoringClientOptions: cOptsDisableLogTrace,
TraceClientOptions: cOptsDisableLogTrace,
})
if err != nil {
return nil, fmt.Errorf("failed to create Stackdriver exporter: %v", err)
}
return exporter, nil
}
// generateUniqueProcessIdentifier returns a unique process identifier for the
// process this code is running in. This is the same way the OpenCensus library
// generates the unique process identifier, in the format of
// "go-<pid>@<hostname>".
func generateUniqueProcessIdentifier() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "localhost"
}
return "go-" + strconv.Itoa(os.Getpid()) + "@" + hostname
}
// This method accepts config and exporter; the exporter argument is exposed to
// assist unit testing of the OpenCensus behavior.
func startOpenCensus(config *config) error {
// If both tracing and metrics are disabled, there's no point inject default
// StatsHandler.
if config == nil || (config.CloudTrace == nil && config.CloudMonitoring == nil) {
return nil
}
var err error
exporter, err = newExporter(config)
if err != nil {
return err
}
var to opencensus.TraceOptions
if config.CloudTrace != nil {
to.TS = trace.ProbabilitySampler(config.CloudTrace.SamplingRate)
trace.RegisterExporter(exporter.(trace.Exporter))
logger.Infof("Start collecting and exporting trace spans with global_trace_sampling_rate=%.2f", config.CloudTrace.SamplingRate)
}
if config.CloudMonitoring != nil {
if err := view.Register(defaultViews...); err != nil {
return fmt.Errorf("failed to register observability views: %v", err)
}
view.SetReportingPeriod(defaultMetricsReportingInterval)
view.RegisterExporter(exporter.(view.Exporter))
logger.Infof("Start collecting and exporting metrics")
}
internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(opencensus.ServerOption(to))
internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(opencensus.DialOption(to))
logger.Infof("Enabled OpenCensus StatsHandlers for clients and servers")
return nil
}
// stopOpenCensus flushes the exporter's and cleans up globals across all
// packages if exporter was created.
func stopOpenCensus() {
if exporter != nil {
internal.ClearGlobalDialOptions()
internal.ClearGlobalServerOptions()
// This Unregister call guarantees the data recorded gets sent to
// exporter, synchronising the view package and exporter. Doesn't matter
// if views not registered, will be a noop if not registered.
view.Unregister(defaultViews...)
// Call these unconditionally, doesn't matter if not registered, will be
// a noop if not registered.
trace.UnregisterExporter(exporter)
view.UnregisterExporter(exporter)
// This Flush call makes sure recorded telemetry get sent to backend.
exporter.Flush()
exporter.Close()
}
}