snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Testing / vet-proto (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s
Stale bot / stale (push) Failing after 1s
Some checks failed
codecov / upload (push) Failing after 13m59s
CodeQL / Analyze (push) Failing after 1s
govulncheck / govulncheck (.) (push) Failing after 1s
govulncheck / govulncheck (./cmd/protoc-gen-go-grpc) (push) Failing after 1s
govulncheck / govulncheck (./gcp/observability) (push) Failing after 1s
govulncheck / govulncheck (./security/advancedtls) (push) Failing after 1s
govulncheck / govulncheck (./stats/opencensus) (push) Failing after 1s
Testing / vet-proto (push) Failing after 1s
Testing / tests (arm, latest) (push) Waiting to run
Testing / extras (latest) (push) Failing after 1s
Testing / static checks (latest-1) (push) Failing after 1s
Testing / tests (-race, latest) (push) Failing after 1s
Testing / tests (i386, latest) (push) Failing after 1s
Testing / tests (latest) (push) Failing after 1s
Testing / tests (latest-1) (push) Failing after 1s
Lock Threads / lock (push) Failing after 1s
Stale bot / stale (push) Failing after 1s
This commit is contained in:
281
encoding/compressor_test.go
Normal file
281
encoding/compressor_test.go
Normal file
@@ -0,0 +1,281 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package encoding_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/encoding"
|
||||
"google.golang.org/grpc/encoding/internal"
|
||||
"google.golang.org/grpc/internal/stubserver"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
testgrpc "google.golang.org/grpc/interop/grpc_testing"
|
||||
testpb "google.golang.org/grpc/interop/grpc_testing"
|
||||
|
||||
_ "google.golang.org/grpc/encoding/gzip"
|
||||
)
|
||||
|
||||
// wrapCompressor is a wrapper of encoding.Compressor which maintains count of
|
||||
// Compressor method invokes.
|
||||
type wrapCompressor struct {
|
||||
encoding.Compressor
|
||||
compressInvokes int32
|
||||
}
|
||||
|
||||
func (wc *wrapCompressor) Compress(w io.Writer) (io.WriteCloser, error) {
|
||||
atomic.AddInt32(&wc.compressInvokes, 1)
|
||||
return wc.Compressor.Compress(w)
|
||||
}
|
||||
|
||||
func setupGzipWrapCompressor(t *testing.T) *wrapCompressor {
|
||||
regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func())
|
||||
c := &wrapCompressor{Compressor: encoding.GetCompressor("gzip")}
|
||||
unreg := regFn(c)
|
||||
t.Cleanup(unreg)
|
||||
return c
|
||||
}
|
||||
|
||||
func (s) TestSetSendCompressorSuccess(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
desc string
|
||||
payload *testpb.Payload
|
||||
dialOpts []grpc.DialOption
|
||||
resCompressor string
|
||||
wantCompressInvokes int32
|
||||
}{
|
||||
{
|
||||
name: "identity_request_and_gzip_response",
|
||||
desc: "request is uncompressed and response is gzip compressed",
|
||||
payload: &testpb.Payload{Body: []byte("payload")},
|
||||
resCompressor: "gzip",
|
||||
wantCompressInvokes: 1,
|
||||
},
|
||||
{
|
||||
name: "identity_request_and_empty_response",
|
||||
desc: "request is uncompressed and response is gzip compressed",
|
||||
payload: nil,
|
||||
resCompressor: "gzip",
|
||||
wantCompressInvokes: 0,
|
||||
},
|
||||
{
|
||||
name: "gzip_request_and_identity_response",
|
||||
desc: "request is gzip compressed and response is uncompressed with identity",
|
||||
payload: &testpb.Payload{Body: []byte("payload")},
|
||||
resCompressor: "identity",
|
||||
dialOpts: []grpc.DialOption{
|
||||
// Use WithCompressor instead of UseCompressor to avoid counting
|
||||
// the client's compressor usage.
|
||||
grpc.WithCompressor(grpc.NewGZIPCompressor()),
|
||||
},
|
||||
wantCompressInvokes: 0,
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Run("unary", func(t *testing.T) {
|
||||
testUnarySetSendCompressorSuccess(t, tt.payload, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts)
|
||||
})
|
||||
|
||||
t.Run("stream", func(t *testing.T) {
|
||||
testStreamSetSendCompressorSuccess(t, tt.payload, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testUnarySetSendCompressorSuccess(t *testing.T, payload *testpb.Payload, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) {
|
||||
wc := setupGzipWrapCompressor(t)
|
||||
ss := &stubserver.StubServer{
|
||||
UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
|
||||
if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &testpb.SimpleResponse{
|
||||
Payload: payload,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
if err := ss.Start(nil, dialOpts...); err != nil {
|
||||
t.Fatalf("Error starting endpoint server: %v", err)
|
||||
}
|
||||
defer ss.Stop()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil {
|
||||
t.Fatalf("Unexpected unary call error, got: %v, want: nil", err)
|
||||
}
|
||||
|
||||
compressInvokes := atomic.LoadInt32(&wc.compressInvokes)
|
||||
if compressInvokes != wantCompressInvokes {
|
||||
t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes)
|
||||
}
|
||||
}
|
||||
|
||||
func testStreamSetSendCompressorSuccess(t *testing.T, payload *testpb.Payload, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) {
|
||||
wc := setupGzipWrapCompressor(t)
|
||||
ss := &stubserver.StubServer{
|
||||
FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
|
||||
if _, err := stream.Recv(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return stream.Send(&testpb.StreamingOutputCallResponse{
|
||||
Payload: payload,
|
||||
})
|
||||
},
|
||||
}
|
||||
if err := ss.Start(nil, dialOpts...); err != nil {
|
||||
t.Fatalf("Error starting endpoint server: %v", err)
|
||||
}
|
||||
defer ss.Stop()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
s, err := ss.Client.FullDuplexCall(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err)
|
||||
}
|
||||
|
||||
if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil {
|
||||
t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err)
|
||||
}
|
||||
|
||||
if _, err := s.Recv(); err != nil {
|
||||
t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err)
|
||||
}
|
||||
|
||||
compressInvokes := atomic.LoadInt32(&wc.compressInvokes)
|
||||
if compressInvokes != wantCompressInvokes {
|
||||
t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeCompressor returns a messages of a configured size, irrespective of the
|
||||
// input.
|
||||
type fakeCompressor struct {
|
||||
decompressedMessageSize int
|
||||
}
|
||||
|
||||
func (f *fakeCompressor) Compress(w io.Writer) (io.WriteCloser, error) {
|
||||
return nopWriteCloser{w}, nil
|
||||
}
|
||||
|
||||
func (f *fakeCompressor) Decompress(io.Reader) (io.Reader, error) {
|
||||
return bytes.NewReader(make([]byte, f.decompressedMessageSize)), nil
|
||||
}
|
||||
|
||||
func (f *fakeCompressor) Name() string {
|
||||
// Use the name of an existing compressor to avoid interactions with other
|
||||
// tests since compressors can't be un-registered.
|
||||
return "fake"
|
||||
}
|
||||
|
||||
type nopWriteCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
func (nopWriteCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestDecompressionExceedsMaxMessageSize uses a fake compressor that produces
|
||||
// messages of size 100 bytes on decompression. A server is started with the
|
||||
// max receive message size restricted to 99 bytes. The test verifies that the
|
||||
// client receives a ResourceExhausted response from the server.
|
||||
func (s) TestDecompressionExceedsMaxMessageSize(t *testing.T) {
|
||||
const messageLen = 100
|
||||
regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func())
|
||||
compressor := &fakeCompressor{decompressedMessageSize: messageLen}
|
||||
unreg := regFn(compressor)
|
||||
defer unreg()
|
||||
ss := &stubserver.StubServer{
|
||||
UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
|
||||
return &testpb.SimpleResponse{}, nil
|
||||
},
|
||||
}
|
||||
if err := ss.Start([]grpc.ServerOption{grpc.MaxRecvMsgSize(messageLen - 1)}); err != nil {
|
||||
t.Fatalf("Error starting endpoint server: %v", err)
|
||||
}
|
||||
defer ss.Stop()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
req := &testpb.SimpleRequest{Payload: &testpb.Payload{}}
|
||||
_, err := ss.Client.UnaryCall(ctx, req, grpc.UseCompressor(compressor.Name()))
|
||||
if got, want := status.Code(err), codes.ResourceExhausted; got != want {
|
||||
t.Errorf("Client.UnaryCall(%+v) returned status %v, want %v", req, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Test verifies that decompression failures (after stream establishment) are
|
||||
// handled gracefully without duplicate status writes.
|
||||
func (s) TestStreamingDecompressionExceedsMaxMessageSize(t *testing.T) {
|
||||
const messageLen = 100
|
||||
regFn := internal.RegisterCompressorForTesting.(func(encoding.Compressor) func())
|
||||
compressor := &fakeCompressor{decompressedMessageSize: messageLen}
|
||||
unreg := regFn(compressor)
|
||||
defer unreg()
|
||||
|
||||
ss := &stubserver.StubServer{
|
||||
FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error {
|
||||
for {
|
||||
if _, err := stream.Recv(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
if err := ss.Start([]grpc.ServerOption{grpc.MaxRecvMsgSize(messageLen - 1)}); err != nil {
|
||||
t.Fatalf("Error starting endpoint server: %v", err)
|
||||
}
|
||||
defer ss.Stop()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
stream, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(compressor.Name()))
|
||||
if err != nil {
|
||||
t.Fatalf("FullDuplexCall failed: %v", err)
|
||||
}
|
||||
|
||||
// Send message that will result in decompression failure on the server
|
||||
// because it exceeds the max message size.
|
||||
if err := stream.Send(&testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: []byte("payload")}}); err != nil {
|
||||
t.Fatalf("Send() failed: %v", err)
|
||||
}
|
||||
|
||||
// We expect the stream to fail with ResourceExhausted on Recv.
|
||||
if _, err = stream.Recv(); status.Code(err) != codes.ResourceExhausted {
|
||||
t.Fatalf("stream.Recv() returned status %v, want %v", status.Code(err), codes.ResourceExhausted)
|
||||
}
|
||||
}
|
||||
150
encoding/encoding.go
Normal file
150
encoding/encoding.go
Normal file
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2017 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package encoding defines the interface for the compressor and codec, and
|
||||
// functions to register and retrieve compressors and codecs.
|
||||
//
|
||||
// # Experimental
|
||||
//
|
||||
// Notice: This package is EXPERIMENTAL and may be changed or removed in a
|
||||
// later release.
|
||||
package encoding
|
||||
|
||||
import (
|
||||
"io"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/grpc/encoding/internal"
|
||||
"google.golang.org/grpc/internal/grpcutil"
|
||||
)
|
||||
|
||||
// Identity specifies the optional encoding for uncompressed streams.
|
||||
// It is intended for grpc internal use only.
|
||||
const Identity = "identity"
|
||||
|
||||
func init() {
|
||||
internal.RegisterCompressorForTesting = func(c Compressor) func() {
|
||||
name := c.Name()
|
||||
curCompressor, found := registeredCompressor[name]
|
||||
RegisterCompressor(c)
|
||||
return func() {
|
||||
if found {
|
||||
registeredCompressor[name] = curCompressor
|
||||
return
|
||||
}
|
||||
delete(registeredCompressor, name)
|
||||
grpcutil.RegisteredCompressorNames = slices.DeleteFunc(grpcutil.RegisteredCompressorNames, func(s string) bool {
|
||||
return s == name
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compressor is used for compressing and decompressing when sending or
|
||||
// receiving messages.
|
||||
type Compressor interface {
|
||||
// Compress writes the data written to wc to w after compressing it. If an
|
||||
// error occurs while initializing the compressor, that error is returned
|
||||
// instead.
|
||||
Compress(w io.Writer) (io.WriteCloser, error)
|
||||
// Decompress reads data from r, decompresses it, and provides the
|
||||
// uncompressed data via the returned io.Reader. If an error occurs while
|
||||
// initializing the decompressor, that error is returned instead.
|
||||
//
|
||||
// The returned io.Reader may optionally implement io.ReadCloser, and if it
|
||||
// does, gRPC will call Close() exactly once.
|
||||
Decompress(r io.Reader) (io.Reader, error)
|
||||
// Name is the name of the compression codec and is used to set the content
|
||||
// coding header. The result must be static; the result cannot change
|
||||
// between calls.
|
||||
Name() string
|
||||
}
|
||||
|
||||
var registeredCompressor = make(map[string]Compressor)
|
||||
|
||||
// RegisterCompressor registers the compressor with gRPC by its name. It can
|
||||
// be activated when sending an RPC via grpc.UseCompressor(). It will be
|
||||
// automatically accessed when receiving a message based on the content coding
|
||||
// header. Servers also use it to send a response with the same encoding as
|
||||
// the request.
|
||||
//
|
||||
// NOTE: this function must only be called during initialization time (i.e. in
|
||||
// an init() function), and is not thread-safe. If multiple Compressors are
|
||||
// registered with the same name, the one registered last will take effect.
|
||||
func RegisterCompressor(c Compressor) {
|
||||
registeredCompressor[c.Name()] = c
|
||||
if !grpcutil.IsCompressorNameRegistered(c.Name()) {
|
||||
grpcutil.RegisteredCompressorNames = append(grpcutil.RegisteredCompressorNames, c.Name())
|
||||
}
|
||||
}
|
||||
|
||||
// GetCompressor returns Compressor for the given compressor name.
|
||||
func GetCompressor(name string) Compressor {
|
||||
return registeredCompressor[name]
|
||||
}
|
||||
|
||||
// Codec defines the interface gRPC uses to encode and decode messages. Note
|
||||
// that implementations of this interface must be thread safe; a Codec's
|
||||
// methods can be called from concurrent goroutines.
|
||||
type Codec interface {
|
||||
// Marshal returns the wire format of v.
|
||||
Marshal(v any) ([]byte, error)
|
||||
// Unmarshal parses the wire format into v.
|
||||
Unmarshal(data []byte, v any) error
|
||||
// Name returns the name of the Codec implementation. The returned string
|
||||
// will be used as part of content type in transmission. The result must be
|
||||
// static; the result cannot change between calls.
|
||||
Name() string
|
||||
}
|
||||
|
||||
var registeredCodecs = make(map[string]any)
|
||||
|
||||
// RegisterCodec registers the provided Codec for use with all gRPC clients and
|
||||
// servers.
|
||||
//
|
||||
// The Codec will be stored and looked up by result of its Name() method, which
|
||||
// should match the content-subtype of the encoding handled by the Codec. This
|
||||
// is case-insensitive, and is stored and looked up as lowercase. If the
|
||||
// result of calling Name() is an empty string, RegisterCodec will panic. See
|
||||
// Content-Type on
|
||||
// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
|
||||
// more details.
|
||||
//
|
||||
// NOTE: this function must only be called during initialization time (i.e. in
|
||||
// an init() function), and is not thread-safe. If multiple Codecs are
|
||||
// registered with the same name, the one registered last will take effect.
|
||||
func RegisterCodec(codec Codec) {
|
||||
if codec == nil {
|
||||
panic("cannot register a nil Codec")
|
||||
}
|
||||
if codec.Name() == "" {
|
||||
panic("cannot register Codec with empty string result for Name()")
|
||||
}
|
||||
contentSubtype := strings.ToLower(codec.Name())
|
||||
registeredCodecs[contentSubtype] = codec
|
||||
}
|
||||
|
||||
// GetCodec gets a registered Codec by content-subtype, or nil if no Codec is
|
||||
// registered for the content-subtype.
|
||||
//
|
||||
// The content-subtype is expected to be lowercase.
|
||||
func GetCodec(contentSubtype string) Codec {
|
||||
c, _ := registeredCodecs[contentSubtype].(Codec)
|
||||
return c
|
||||
}
|
||||
400
encoding/encoding_test.go
Normal file
400
encoding/encoding_test.go
Normal file
@@ -0,0 +1,400 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2023 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package encoding_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/encoding"
|
||||
"google.golang.org/grpc/encoding/proto"
|
||||
"google.golang.org/grpc/internal/grpctest"
|
||||
"google.golang.org/grpc/internal/grpcutil"
|
||||
"google.golang.org/grpc/internal/stubserver"
|
||||
"google.golang.org/grpc/mem"
|
||||
"google.golang.org/grpc/metadata"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
testgrpc "google.golang.org/grpc/interop/grpc_testing"
|
||||
testpb "google.golang.org/grpc/interop/grpc_testing"
|
||||
)
|
||||
|
||||
const defaultTestTimeout = 10 * time.Second
|
||||
|
||||
type s struct {
|
||||
grpctest.Tester
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
grpctest.RunSubTests(t, s{})
|
||||
}
|
||||
|
||||
type mockNamedCompressor struct {
|
||||
encoding.Compressor
|
||||
}
|
||||
|
||||
func (mockNamedCompressor) Name() string {
|
||||
return "mock-compressor"
|
||||
}
|
||||
|
||||
// Tests the case where a compressor with the same name is registered multiple
|
||||
// times. Test verifies the following:
|
||||
// - the most recent registration is the one which is active
|
||||
// - grpcutil.RegisteredCompressorNames contains a single instance of the
|
||||
// previously registered compressor's name
|
||||
func (s) TestDuplicateCompressorRegister(t *testing.T) {
|
||||
encoding.RegisterCompressor(&mockNamedCompressor{})
|
||||
|
||||
// Register another instance of the same compressor.
|
||||
mc := &mockNamedCompressor{}
|
||||
encoding.RegisterCompressor(mc)
|
||||
if got := encoding.GetCompressor("mock-compressor"); got != mc {
|
||||
t.Fatalf("Unexpected compressor, got: %+v, want:%+v", got, mc)
|
||||
}
|
||||
|
||||
wantNames := []string{"gzip", "mock-compressor"}
|
||||
if !cmp.Equal(wantNames, grpcutil.RegisteredCompressorNames) {
|
||||
t.Fatalf("Unexpected compressor names, got: %+v, want:%+v", grpcutil.RegisteredCompressorNames, wantNames)
|
||||
}
|
||||
}
|
||||
|
||||
// errProtoCodec wraps the proto codec and delegates to it if it is configured
|
||||
// to return a nil error. Else, it returns the configured error.
|
||||
type errProtoCodec struct {
|
||||
name string
|
||||
encodingErr error
|
||||
decodingErr error
|
||||
}
|
||||
|
||||
func (c *errProtoCodec) Marshal(v any) (mem.BufferSlice, error) {
|
||||
if c.encodingErr != nil {
|
||||
return nil, c.encodingErr
|
||||
}
|
||||
return encoding.GetCodecV2(proto.Name).Marshal(v)
|
||||
}
|
||||
|
||||
func (c *errProtoCodec) Unmarshal(data mem.BufferSlice, v any) error {
|
||||
if c.decodingErr != nil {
|
||||
return c.decodingErr
|
||||
}
|
||||
return encoding.GetCodecV2(proto.Name).Unmarshal(data, v)
|
||||
}
|
||||
|
||||
func (c *errProtoCodec) Name() string {
|
||||
return c.name
|
||||
}
|
||||
|
||||
// Tests the case where encoding fails on the server. Verifies that there is
|
||||
// no panic and that the encoding error is propagated to the client.
|
||||
func (s) TestEncodeDoesntPanicOnServer(t *testing.T) {
|
||||
// Create a codec that errors when encoding messages.
|
||||
encodingErr := errors.New("encoding failed")
|
||||
ec := &errProtoCodec{name: t.Name(), encodingErr: encodingErr}
|
||||
|
||||
// Start a server with the above codec.
|
||||
backend1 := stubserver.StubServer{
|
||||
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
|
||||
}
|
||||
if err := backend1.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer backend1.Stop()
|
||||
|
||||
// Make an RPC and expect it to fail. Since we do not specify any codec
|
||||
// here, the proto codec will get automatically used.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
_, err := backend1.Client.EmptyCall(ctx, &testpb.Empty{})
|
||||
if err == nil || !strings.Contains(err.Error(), encodingErr.Error()) {
|
||||
t.Fatalf("RPC failed with error: %v, want: %v", err, encodingErr)
|
||||
}
|
||||
|
||||
// Configure the codec on the server to not return errors anymore and expect
|
||||
// the RPC to succeed.
|
||||
ec = &errProtoCodec{name: t.Name()}
|
||||
backend2 := stubserver.StubServer{
|
||||
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
|
||||
}
|
||||
if err := backend2.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer backend2.Stop()
|
||||
|
||||
if _, err := backend2.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
|
||||
t.Fatalf("RPC failed with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where decoding fails on the server. Verifies that there is
|
||||
// no panic and that the decoding error is propagated to the client.
|
||||
func (s) TestDecodeDoesntPanicOnServer(t *testing.T) {
|
||||
// Create a codec that errors when decoding messages.
|
||||
decodingErr := errors.New("decoding failed")
|
||||
ec := &errProtoCodec{name: t.Name(), decodingErr: decodingErr}
|
||||
|
||||
// Start a server with the above codec and a channel to the server.
|
||||
backend1 := stubserver.StubServer{
|
||||
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
|
||||
}
|
||||
if err := backend1.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer backend1.Stop()
|
||||
|
||||
// Make an RPC and expect it to fail. Since we do not specify any codec
|
||||
// here, the proto codec will get automatically used.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
_, err := backend1.Client.EmptyCall(ctx, &testpb.Empty{})
|
||||
if err == nil || !strings.Contains(err.Error(), decodingErr.Error()) || !strings.Contains(err.Error(), "grpc: failed to unmarshal the received message") {
|
||||
t.Fatalf("RPC failed with error: %v, want: %v", err, decodingErr)
|
||||
}
|
||||
|
||||
// Configure the codec on the server to not return errors anymore and expect
|
||||
// the RPC to succeed.
|
||||
ec = &errProtoCodec{name: t.Name()}
|
||||
backend2 := stubserver.StubServer{
|
||||
EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil },
|
||||
}
|
||||
if err := backend2.Start([]grpc.ServerOption{grpc.ForceServerCodecV2(ec)}, grpc.WithTransportCredentials(insecure.NewCredentials())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer backend2.Stop()
|
||||
|
||||
if _, err := backend2.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
|
||||
t.Fatalf("RPC failed with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where encoding fails on the client . Verifies that there is
|
||||
// no panic and that the encoding error is propagated to the RPC caller.
|
||||
func (s) TestEncodeDoesntPanicOnClient(t *testing.T) {
|
||||
// Start a server and since we do not specify any codec here, the proto
|
||||
// codec will get automatically used.
|
||||
backend := stubserver.StartTestService(t, nil)
|
||||
defer backend.Stop()
|
||||
|
||||
// Create a codec that errors when encoding messages.
|
||||
encodingErr := errors.New("encoding failed")
|
||||
ec := &errProtoCodec{name: t.Name(), encodingErr: encodingErr}
|
||||
|
||||
// Create a channel to the above server.
|
||||
cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err)
|
||||
}
|
||||
defer cc.Close()
|
||||
|
||||
// Make an RPC with the erroring codec and expect it to fail.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
client := testgrpc.NewTestServiceClient(cc)
|
||||
_, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec))
|
||||
if err == nil || !strings.Contains(err.Error(), encodingErr.Error()) {
|
||||
t.Fatalf("RPC failed with error: %v, want: %v", err, encodingErr)
|
||||
}
|
||||
|
||||
// Configure the codec on the client to not return errors anymore and expect
|
||||
// the RPC to succeed.
|
||||
ec.encodingErr = nil
|
||||
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)); err != nil {
|
||||
t.Fatalf("RPC failed with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where decoding fails on the server. Verifies that there is
|
||||
// no panic and that the decoding error is propagated to the RPC caller.
|
||||
func (s) TestDecodeDoesntPanicOnClient(t *testing.T) {
|
||||
// Start a server and since we do not specify any codec here, the proto
|
||||
// codec will get automatically used.
|
||||
backend := stubserver.StartTestService(t, nil)
|
||||
defer backend.Stop()
|
||||
|
||||
// Create a codec that errors when decoding messages.
|
||||
decodingErr := errors.New("decoding failed")
|
||||
ec := &errProtoCodec{name: t.Name(), decodingErr: decodingErr}
|
||||
|
||||
// Create a channel to the above server.
|
||||
cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err)
|
||||
}
|
||||
defer cc.Close()
|
||||
|
||||
// Make an RPC with the erroring codec and expect it to fail.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
client := testgrpc.NewTestServiceClient(cc)
|
||||
_, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec))
|
||||
if err == nil || !strings.Contains(err.Error(), decodingErr.Error()) {
|
||||
t.Fatalf("RPC failed with error: %v, want: %v", err, decodingErr)
|
||||
}
|
||||
|
||||
// Configure the codec on the client to not return errors anymore and expect
|
||||
// the RPC to succeed.
|
||||
ec.decodingErr = nil
|
||||
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(ec)); err != nil {
|
||||
t.Fatalf("RPC failed with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// countingProtoCodec wraps the proto codec and counts the number of times
|
||||
// Marshal and Unmarshal are called.
|
||||
type countingProtoCodec struct {
|
||||
name string
|
||||
|
||||
// The following fields are accessed atomically.
|
||||
marshalCount int32
|
||||
unmarshalCount int32
|
||||
}
|
||||
|
||||
func (p *countingProtoCodec) Marshal(v any) (mem.BufferSlice, error) {
|
||||
atomic.AddInt32(&p.marshalCount, 1)
|
||||
return encoding.GetCodecV2(proto.Name).Marshal(v)
|
||||
}
|
||||
|
||||
func (p *countingProtoCodec) Unmarshal(data mem.BufferSlice, v any) error {
|
||||
atomic.AddInt32(&p.unmarshalCount, 1)
|
||||
return encoding.GetCodecV2(proto.Name).Unmarshal(data, v)
|
||||
}
|
||||
|
||||
func (p *countingProtoCodec) Name() string {
|
||||
return p.name
|
||||
}
|
||||
|
||||
// Tests the case where ForceServerCodec option is used on the server. Verifies
|
||||
// that encoding and decoding happen once per RPC.
|
||||
func (s) TestForceServerCodec(t *testing.T) {
|
||||
// Create a server with the counting proto codec.
|
||||
codec := &countingProtoCodec{name: t.Name()}
|
||||
backend := stubserver.StartTestService(t, nil, grpc.ForceServerCodecV2(codec))
|
||||
defer backend.Stop()
|
||||
|
||||
// Create a channel to the above server.
|
||||
cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err)
|
||||
}
|
||||
defer cc.Close()
|
||||
|
||||
// Make an RPC and expect it to fail. Since we do not specify any codec
|
||||
// here, the proto codec will get automatically used.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
client := testgrpc.NewTestServiceClient(cc)
|
||||
if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil {
|
||||
t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
|
||||
}
|
||||
|
||||
unmarshalCount := atomic.LoadInt32(&codec.unmarshalCount)
|
||||
const wantUnmarshalCount = 1
|
||||
if unmarshalCount != wantUnmarshalCount {
|
||||
t.Fatalf("Unmarshal Count = %d; want %d", unmarshalCount, wantUnmarshalCount)
|
||||
}
|
||||
marshalCount := atomic.LoadInt32(&codec.marshalCount)
|
||||
const wantMarshalCount = 1
|
||||
if marshalCount != wantMarshalCount {
|
||||
t.Fatalf("MarshalCount = %d; want %d", marshalCount, wantMarshalCount)
|
||||
}
|
||||
}
|
||||
|
||||
// renameProtoCodec wraps the proto codec and allows customizing the Name().
|
||||
type renameProtoCodec struct {
|
||||
encoding.CodecV2
|
||||
name string
|
||||
}
|
||||
|
||||
func (r *renameProtoCodec) Name() string { return r.name }
|
||||
|
||||
// TestForceCodecName confirms that the ForceCodec call option sets the subtype
|
||||
// in the content-type header according to the Name() of the codec provided.
|
||||
// Verifies that the name is converted to lowercase before transmitting.
|
||||
func (s) TestForceCodecName(t *testing.T) {
|
||||
wantContentTypeCh := make(chan []string, 1)
|
||||
defer close(wantContentTypeCh)
|
||||
|
||||
// Create a test service backend that pushes the received content-type on a
|
||||
// channel for the test to inspect.
|
||||
ss := &stubserver.StubServer{
|
||||
EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) {
|
||||
md, ok := metadata.FromIncomingContext(ctx)
|
||||
if !ok {
|
||||
return nil, status.Errorf(codes.Internal, "no metadata in context")
|
||||
}
|
||||
if got, want := md["content-type"], <-wantContentTypeCh; !cmp.Equal(got, want) {
|
||||
return nil, status.Errorf(codes.Internal, "got content-type=%q; want [%q]", got, want)
|
||||
}
|
||||
return &testpb.Empty{}, nil
|
||||
},
|
||||
}
|
||||
// Since we don't specify a codec as a server option, it will end up
|
||||
// automatically using the proto codec.
|
||||
if err := ss.Start(nil); err != nil {
|
||||
t.Fatalf("Error starting endpoint server: %v", err)
|
||||
}
|
||||
defer ss.Stop()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Force the use of the custom codec on the client with the ForceCodec call
|
||||
// option. Confirm the name is converted to lowercase before transmitting.
|
||||
codec := &renameProtoCodec{CodecV2: encoding.GetCodecV2(proto.Name), name: t.Name()}
|
||||
wantContentTypeCh <- []string{fmt.Sprintf("application/grpc+%s", strings.ToLower(t.Name()))}
|
||||
if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(codec)); err != nil {
|
||||
t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests the case where the client uses a codec (one that uses proto encoding
|
||||
// but uses a different content-subtype name) that the server does not support.
|
||||
// Verifies that the server falls back to the proto codec and that the client
|
||||
// can successfully make RPCs.
|
||||
//
|
||||
// TODO(https://github.com/grpc/grpc-go/issues/1824): Once we add an environment
|
||||
// variable to change the behavior on the server to reject unsupported codecs,
|
||||
// we should modify this test to verify that the RPC fails in that case.
|
||||
func (s) TestUnsupportedCodecOnServer(t *testing.T) {
|
||||
backend := stubserver.StartTestService(t, nil)
|
||||
defer backend.Stop()
|
||||
|
||||
cc, err := grpc.NewClient(backend.Address, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to dial test backend at %q: %v", backend.Address, err)
|
||||
}
|
||||
defer cc.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
||||
defer cancel()
|
||||
|
||||
codec := &renameProtoCodec{CodecV2: encoding.GetCodecV2(proto.Name), name: t.Name()}
|
||||
client := testgrpc.NewTestServiceClient(cc)
|
||||
if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodecV2(codec)); err != nil {
|
||||
t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err)
|
||||
}
|
||||
}
|
||||
81
encoding/encoding_v2.go
Normal file
81
encoding/encoding_v2.go
Normal file
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package encoding
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"google.golang.org/grpc/mem"
|
||||
)
|
||||
|
||||
// CodecV2 defines the interface gRPC uses to encode and decode messages. Note
|
||||
// that implementations of this interface must be thread safe; a CodecV2's
|
||||
// methods can be called from concurrent goroutines.
|
||||
type CodecV2 interface {
|
||||
// Marshal returns the wire format of v. The buffers in the returned
|
||||
// [mem.BufferSlice] must have at least one reference each, which will be freed
|
||||
// by gRPC when they are no longer needed.
|
||||
Marshal(v any) (out mem.BufferSlice, err error)
|
||||
// Unmarshal parses the wire format into v. Note that data will be freed as soon
|
||||
// as this function returns. If the codec wishes to guarantee access to the data
|
||||
// after this function, it must take its own reference that it frees when it is
|
||||
// no longer needed.
|
||||
Unmarshal(data mem.BufferSlice, v any) error
|
||||
// Name returns the name of the Codec implementation. The returned string
|
||||
// will be used as part of content type in transmission. The result must be
|
||||
// static; the result cannot change between calls.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// RegisterCodecV2 registers the provided CodecV2 for use with all gRPC clients and
|
||||
// servers.
|
||||
//
|
||||
// The CodecV2 will be stored and looked up by result of its Name() method, which
|
||||
// should match the content-subtype of the encoding handled by the CodecV2. This
|
||||
// is case-insensitive, and is stored and looked up as lowercase. If the
|
||||
// result of calling Name() is an empty string, RegisterCodecV2 will panic. See
|
||||
// Content-Type on
|
||||
// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for
|
||||
// more details.
|
||||
//
|
||||
// If both a Codec and CodecV2 are registered with the same name, the CodecV2
|
||||
// will be used.
|
||||
//
|
||||
// NOTE: this function must only be called during initialization time (i.e. in
|
||||
// an init() function), and is not thread-safe. If multiple Codecs are
|
||||
// registered with the same name, the one registered last will take effect.
|
||||
func RegisterCodecV2(codec CodecV2) {
|
||||
if codec == nil {
|
||||
panic("cannot register a nil CodecV2")
|
||||
}
|
||||
if codec.Name() == "" {
|
||||
panic("cannot register CodecV2 with empty string result for Name()")
|
||||
}
|
||||
contentSubtype := strings.ToLower(codec.Name())
|
||||
registeredCodecs[contentSubtype] = codec
|
||||
}
|
||||
|
||||
// GetCodecV2 gets a registered CodecV2 by content-subtype, or nil if no CodecV2 is
|
||||
// registered for the content-subtype.
|
||||
//
|
||||
// The content-subtype is expected to be lowercase.
|
||||
func GetCodecV2(contentSubtype string) CodecV2 {
|
||||
c, _ := registeredCodecs[contentSubtype].(CodecV2)
|
||||
return c
|
||||
}
|
||||
124
encoding/gzip/gzip.go
Normal file
124
encoding/gzip/gzip.go
Normal file
@@ -0,0 +1,124 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2017 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package gzip implements and registers the gzip compressor
|
||||
// during the initialization.
|
||||
//
|
||||
// # Experimental
|
||||
//
|
||||
// Notice: This package is EXPERIMENTAL and may be changed or removed in a
|
||||
// later release.
|
||||
package gzip
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"google.golang.org/grpc/encoding"
|
||||
)
|
||||
|
||||
// Name is the name registered for the gzip compressor.
|
||||
const Name = "gzip"
|
||||
|
||||
func init() {
|
||||
c := &compressor{}
|
||||
c.poolCompressor.New = func() any {
|
||||
return &writer{Writer: gzip.NewWriter(io.Discard), pool: &c.poolCompressor}
|
||||
}
|
||||
encoding.RegisterCompressor(c)
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
*gzip.Writer
|
||||
pool *sync.Pool
|
||||
}
|
||||
|
||||
// SetLevel updates the registered gzip compressor to use the compression level specified (gzip.HuffmanOnly is not supported).
|
||||
// NOTE: this function must only be called during initialization time (i.e. in an init() function),
|
||||
// and is not thread-safe.
|
||||
//
|
||||
// The error returned will be nil if the specified level is valid.
|
||||
func SetLevel(level int) error {
|
||||
if level < gzip.DefaultCompression || level > gzip.BestCompression {
|
||||
return fmt.Errorf("grpc: invalid gzip compression level: %d", level)
|
||||
}
|
||||
c := encoding.GetCompressor(Name).(*compressor)
|
||||
c.poolCompressor.New = func() any {
|
||||
w, err := gzip.NewWriterLevel(io.Discard, level)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &writer{Writer: w, pool: &c.poolCompressor}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *compressor) Compress(w io.Writer) (io.WriteCloser, error) {
|
||||
z := c.poolCompressor.Get().(*writer)
|
||||
z.Writer.Reset(w)
|
||||
return z, nil
|
||||
}
|
||||
|
||||
func (z *writer) Close() error {
|
||||
defer z.pool.Put(z)
|
||||
return z.Writer.Close()
|
||||
}
|
||||
|
||||
var _ io.Closer = &reader{}
|
||||
|
||||
type reader struct {
|
||||
*gzip.Reader
|
||||
pool *sync.Pool
|
||||
}
|
||||
|
||||
func (c *compressor) Decompress(r io.Reader) (io.Reader, error) {
|
||||
z, inPool := c.poolDecompressor.Get().(*reader)
|
||||
if !inPool {
|
||||
newZ, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &reader{Reader: newZ, pool: &c.poolDecompressor}, nil
|
||||
}
|
||||
if err := z.Reset(r); err != nil {
|
||||
c.poolDecompressor.Put(z)
|
||||
return nil, err
|
||||
}
|
||||
return z, nil
|
||||
}
|
||||
|
||||
func (r *reader) Read(p []byte) (n int, err error) {
|
||||
n, err = r.Reader.Read(p)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (r *reader) Close() error {
|
||||
defer r.pool.Put(r)
|
||||
return r.Reader.Close()
|
||||
}
|
||||
|
||||
func (c *compressor) Name() string {
|
||||
return Name
|
||||
}
|
||||
|
||||
type compressor struct {
|
||||
poolCompressor sync.Pool
|
||||
poolDecompressor sync.Pool
|
||||
}
|
||||
28
encoding/internal/internal.go
Normal file
28
encoding/internal/internal.go
Normal file
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2025 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package internal contains code internal to the encoding package.
|
||||
package internal
|
||||
|
||||
// RegisterCompressorForTesting registers a compressor in the global compressor
|
||||
// registry. It returns a cleanup function that should be called at the end
|
||||
// of the test to unregister the compressor.
|
||||
//
|
||||
// This prevents compressors registered in one test from appearing in the
|
||||
// encoding headers of subsequent tests.
|
||||
var RegisterCompressorForTesting any // func RegisterCompressor(c Compressor) func()
|
||||
112
encoding/proto/proto.go
Normal file
112
encoding/proto/proto.go
Normal file
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2024 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
// Package proto defines the protobuf codec. Importing this package will
|
||||
// register the codec.
|
||||
package proto
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"google.golang.org/grpc/encoding"
|
||||
"google.golang.org/grpc/mem"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/protoadapt"
|
||||
)
|
||||
|
||||
// Name is the name registered for the proto compressor.
|
||||
const Name = "proto"
|
||||
|
||||
func init() {
|
||||
encoding.RegisterCodecV2(&codecV2{})
|
||||
}
|
||||
|
||||
// codec is a CodecV2 implementation with protobuf. It is the default codec for
|
||||
// gRPC.
|
||||
type codecV2 struct{}
|
||||
|
||||
func (c *codecV2) Marshal(v any) (data mem.BufferSlice, err error) {
|
||||
vv := messageV2Of(v)
|
||||
if vv == nil {
|
||||
return nil, fmt.Errorf("proto: failed to marshal, message is %T, want proto.Message", v)
|
||||
}
|
||||
|
||||
// Important: if we remove this Size call then we cannot use
|
||||
// UseCachedSize in MarshalOptions below.
|
||||
size := proto.Size(vv)
|
||||
|
||||
// MarshalOptions with UseCachedSize allows reusing the result from the
|
||||
// previous Size call. This is safe here because:
|
||||
//
|
||||
// 1. We just computed the size.
|
||||
// 2. We assume the message is not being mutated concurrently.
|
||||
//
|
||||
// Important: If the proto.Size call above is removed, using UseCachedSize
|
||||
// becomes unsafe and may lead to incorrect marshaling.
|
||||
//
|
||||
// For more details, see the doc of UseCachedSize:
|
||||
// https://pkg.go.dev/google.golang.org/protobuf/proto#MarshalOptions
|
||||
marshalOptions := proto.MarshalOptions{UseCachedSize: true}
|
||||
|
||||
if mem.IsBelowBufferPoolingThreshold(size) {
|
||||
buf, err := marshalOptions.Marshal(vv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data = append(data, mem.SliceBuffer(buf))
|
||||
} else {
|
||||
pool := mem.DefaultBufferPool()
|
||||
buf := pool.Get(size)
|
||||
if _, err := marshalOptions.MarshalAppend((*buf)[:0], vv); err != nil {
|
||||
pool.Put(buf)
|
||||
return nil, err
|
||||
}
|
||||
data = append(data, mem.NewBuffer(buf, pool))
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *codecV2) Unmarshal(data mem.BufferSlice, v any) (err error) {
|
||||
vv := messageV2Of(v)
|
||||
if vv == nil {
|
||||
return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v)
|
||||
}
|
||||
|
||||
buf := data.MaterializeToBuffer(mem.DefaultBufferPool())
|
||||
defer buf.Free()
|
||||
// TODO: Upgrade proto.Unmarshal to support mem.BufferSlice. Right now, it's not
|
||||
// really possible without a major overhaul of the proto package, but the
|
||||
// vtprotobuf library may be able to support this.
|
||||
return proto.Unmarshal(buf.ReadOnlyData(), vv)
|
||||
}
|
||||
|
||||
func messageV2Of(v any) proto.Message {
|
||||
switch v := v.(type) {
|
||||
case protoadapt.MessageV1:
|
||||
return protoadapt.MessageV2Of(v)
|
||||
case protoadapt.MessageV2:
|
||||
return v
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *codecV2) Name() string {
|
||||
return Name
|
||||
}
|
||||
99
encoding/proto/proto_benchmark_test.go
Normal file
99
encoding/proto/proto_benchmark_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2014 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package proto
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc/encoding"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
pb "google.golang.org/grpc/test/codec_perf"
|
||||
)
|
||||
|
||||
func setupBenchmarkProtoCodecInputs(payloadBaseSize uint32) []proto.Message {
|
||||
payloadBase := make([]byte, payloadBaseSize)
|
||||
// arbitrary byte slices
|
||||
payloadSuffixes := [][]byte{
|
||||
[]byte("one"),
|
||||
[]byte("two"),
|
||||
[]byte("three"),
|
||||
[]byte("four"),
|
||||
[]byte("five"),
|
||||
}
|
||||
protoStructs := make([]proto.Message, 0)
|
||||
|
||||
for _, p := range payloadSuffixes {
|
||||
ps := &pb.Buffer{}
|
||||
ps.Body = append(payloadBase, p...)
|
||||
protoStructs = append(protoStructs, ps)
|
||||
}
|
||||
|
||||
return protoStructs
|
||||
}
|
||||
|
||||
// The possible use of certain protobuf APIs like the proto.Buffer API potentially involves caching
|
||||
// on our side. This can add checks around memory allocations and possible contention.
|
||||
// Example run: go test -v -run=^$ -bench=BenchmarkProtoCodec -benchmem
|
||||
func BenchmarkProtoCodec(b *testing.B) {
|
||||
// range of message sizes
|
||||
payloadBaseSizes := make([]uint32, 0)
|
||||
for i := uint32(0); i <= 12; i += 4 {
|
||||
payloadBaseSizes = append(payloadBaseSizes, 1<<i)
|
||||
}
|
||||
// range of SetParallelism
|
||||
parallelisms := make([]int, 0)
|
||||
for i := uint32(0); i <= 16; i += 4 {
|
||||
parallelisms = append(parallelisms, int(1<<i))
|
||||
}
|
||||
for _, s := range payloadBaseSizes {
|
||||
for _, p := range parallelisms {
|
||||
protoStructs := setupBenchmarkProtoCodecInputs(s)
|
||||
name := fmt.Sprintf("MinPayloadSize:%v/SetParallelism(%v)", s, p)
|
||||
b.Run(name, func(b *testing.B) {
|
||||
codec := &codecV2{}
|
||||
b.SetParallelism(p)
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
benchmarkProtoCodec(codec, protoStructs, pb, b)
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkProtoCodec(codec *codecV2, protoStructs []proto.Message, pb *testing.PB, b *testing.B) {
|
||||
counter := 0
|
||||
for pb.Next() {
|
||||
counter++
|
||||
ps := protoStructs[counter%len(protoStructs)]
|
||||
fastMarshalAndUnmarshal(codec, ps, b)
|
||||
}
|
||||
}
|
||||
|
||||
func fastMarshalAndUnmarshal(codec encoding.CodecV2, protoStruct proto.Message, b *testing.B) {
|
||||
marshaledBytes, err := codec.Marshal(protoStruct)
|
||||
if err != nil {
|
||||
b.Errorf("codec.Marshal(_) returned an error")
|
||||
}
|
||||
res := pb.Buffer{}
|
||||
if err := codec.Unmarshal(marshaledBytes, &res); err != nil {
|
||||
b.Errorf("codec.Unmarshal(_) returned an error")
|
||||
}
|
||||
}
|
||||
139
encoding/proto/proto_test.go
Normal file
139
encoding/proto/proto_test.go
Normal file
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
*
|
||||
* Copyright 2018 gRPC authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
package proto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc/encoding"
|
||||
"google.golang.org/grpc/internal/grpctest"
|
||||
"google.golang.org/grpc/mem"
|
||||
pb "google.golang.org/grpc/test/codec_perf"
|
||||
)
|
||||
|
||||
func marshalAndUnmarshal(t *testing.T, codec encoding.CodecV2, expectedBody []byte) {
|
||||
p := &pb.Buffer{}
|
||||
p.Body = expectedBody
|
||||
|
||||
marshalledBytes, err := codec.Marshal(p)
|
||||
if err != nil {
|
||||
t.Errorf("codec.Marshal(_) returned an error")
|
||||
}
|
||||
|
||||
if err := codec.Unmarshal(marshalledBytes, p); err != nil {
|
||||
t.Errorf("codec.Unmarshal(_) returned an error")
|
||||
}
|
||||
|
||||
if !bytes.Equal(p.GetBody(), expectedBody) {
|
||||
t.Errorf("Unexpected body; got %v; want %v", p.GetBody(), expectedBody)
|
||||
}
|
||||
}
|
||||
|
||||
type s struct {
|
||||
grpctest.Tester
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
grpctest.RunSubTests(t, s{})
|
||||
}
|
||||
|
||||
func (s) TestBasicProtoCodecMarshalAndUnmarshal(t *testing.T) {
|
||||
marshalAndUnmarshal(t, &codecV2{}, []byte{1, 2, 3})
|
||||
}
|
||||
|
||||
// Try to catch possible race conditions around use of pools
|
||||
func (s) TestConcurrentUsage(t *testing.T) {
|
||||
const (
|
||||
numGoRoutines = 100
|
||||
numMarshUnmarsh = 1000
|
||||
)
|
||||
|
||||
// small, arbitrary byte slices
|
||||
protoBodies := [][]byte{
|
||||
[]byte("one"),
|
||||
[]byte("two"),
|
||||
[]byte("three"),
|
||||
[]byte("four"),
|
||||
[]byte("five"),
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
codec := &codecV2{}
|
||||
|
||||
for i := 0; i < numGoRoutines; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for k := 0; k < numMarshUnmarsh; k++ {
|
||||
marshalAndUnmarshal(t, codec, protoBodies[k%len(protoBodies)])
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// TestStaggeredMarshalAndUnmarshalUsingSamePool tries to catch potential errors in which slices get
|
||||
// stomped on during reuse of a proto.Buffer.
|
||||
func (s) TestStaggeredMarshalAndUnmarshalUsingSamePool(t *testing.T) {
|
||||
codec1 := &codecV2{}
|
||||
codec2 := &codecV2{}
|
||||
|
||||
expectedBody1 := []byte{1, 2, 3}
|
||||
expectedBody2 := []byte{4, 5, 6}
|
||||
|
||||
proto1 := pb.Buffer{Body: expectedBody1}
|
||||
proto2 := pb.Buffer{Body: expectedBody2}
|
||||
|
||||
var m1, m2 mem.BufferSlice
|
||||
var err error
|
||||
|
||||
if m1, err = codec1.Marshal(&proto1); err != nil {
|
||||
t.Errorf("codec.Marshal(%s) failed", &proto1)
|
||||
}
|
||||
|
||||
if m2, err = codec2.Marshal(&proto2); err != nil {
|
||||
t.Errorf("codec.Marshal(%s) failed", &proto2)
|
||||
}
|
||||
|
||||
if err = codec1.Unmarshal(m1, &proto1); err != nil {
|
||||
t.Errorf("codec.Unmarshal(%v) failed", m1)
|
||||
}
|
||||
|
||||
if err = codec2.Unmarshal(m2, &proto2); err != nil {
|
||||
t.Errorf("codec.Unmarshal(%v) failed", m2)
|
||||
}
|
||||
|
||||
b1 := proto1.GetBody()
|
||||
b2 := proto2.GetBody()
|
||||
|
||||
for i, v := range b1 {
|
||||
if expectedBody1[i] != v {
|
||||
t.Errorf("expected %v at index %v but got %v", i, expectedBody1[i], v)
|
||||
}
|
||||
}
|
||||
|
||||
for i, v := range b2 {
|
||||
if expectedBody2[i] != v {
|
||||
t.Errorf("expected %v at index %v but got %v", i, expectedBody2[i], v)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user