snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

encoding/proto/proto.go

@@ -0,0 +1,112 @@
+/*
+ *
+ * Copyright 2024 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package proto defines the protobuf codec. Importing this package will
+// register the codec.
+package proto
+
+import (
+	"fmt"
+
+	"google.golang.org/grpc/encoding"
+	"google.golang.org/grpc/mem"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/protoadapt"
+)
+
+// Name is the name registered for the proto compressor.
+const Name = "proto"
+
+func init() {
+	encoding.RegisterCodecV2(&codecV2{})
+}
+
+// codec is a CodecV2 implementation with protobuf. It is the default codec for
+// gRPC.
+type codecV2 struct{}
+
+func (c *codecV2) Marshal(v any) (data mem.BufferSlice, err error) {
+	vv := messageV2Of(v)
+	if vv == nil {
+		return nil, fmt.Errorf("proto: failed to marshal, message is %T, want proto.Message", v)
+	}
+
+	// Important: if we remove this Size call then we cannot use
+	// UseCachedSize in MarshalOptions below.
+	size := proto.Size(vv)
+
+	// MarshalOptions with UseCachedSize allows reusing the result from the
+	// previous Size call. This is safe here because:
+	//
+	// 1. We just computed the size.
+	// 2. We assume the message is not being mutated concurrently.
+	//
+	// Important: If the proto.Size call above is removed, using UseCachedSize
+	// becomes unsafe and may lead to incorrect marshaling.
+	//
+	// For more details, see the doc of UseCachedSize:
+	// https://pkg.go.dev/google.golang.org/protobuf/proto#MarshalOptions
+	marshalOptions := proto.MarshalOptions{UseCachedSize: true}
+
+	if mem.IsBelowBufferPoolingThreshold(size) {
+		buf, err := marshalOptions.Marshal(vv)
+		if err != nil {
+			return nil, err
+		}
+		data = append(data, mem.SliceBuffer(buf))
+	} else {
+		pool := mem.DefaultBufferPool()
+		buf := pool.Get(size)
+		if _, err := marshalOptions.MarshalAppend((*buf)[:0], vv); err != nil {
+			pool.Put(buf)
+			return nil, err
+		}
+		data = append(data, mem.NewBuffer(buf, pool))
+	}
+
+	return data, nil
+}
+
+func (c *codecV2) Unmarshal(data mem.BufferSlice, v any) (err error) {
+	vv := messageV2Of(v)
+	if vv == nil {
+		return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v)
+	}
+
+	buf := data.MaterializeToBuffer(mem.DefaultBufferPool())
+	defer buf.Free()
+	// TODO: Upgrade proto.Unmarshal to support mem.BufferSlice. Right now, it's not
+	//  really possible without a major overhaul of the proto package, but the
+	//  vtprotobuf library may be able to support this.
+	return proto.Unmarshal(buf.ReadOnlyData(), vv)
+}
+
+func messageV2Of(v any) proto.Message {
+	switch v := v.(type) {
+	case protoadapt.MessageV1:
+		return protoadapt.MessageV2Of(v)
+	case protoadapt.MessageV2:
+		return v
+	}
+
+	return nil
+}
+
+func (c *codecV2) Name() string {
+	return Name
+}

encoding/proto/proto_benchmark_test.go

@@ -0,0 +1,99 @@
+/*
+ *
+ * Copyright 2014 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package proto
+
+import (
+	"fmt"
+	"testing"
+
+	"google.golang.org/grpc/encoding"
+	"google.golang.org/protobuf/proto"
+
+	pb "google.golang.org/grpc/test/codec_perf"
+)
+
+func setupBenchmarkProtoCodecInputs(payloadBaseSize uint32) []proto.Message {
+	payloadBase := make([]byte, payloadBaseSize)
+	// arbitrary byte slices
+	payloadSuffixes := [][]byte{
+		[]byte("one"),
+		[]byte("two"),
+		[]byte("three"),
+		[]byte("four"),
+		[]byte("five"),
+	}
+	protoStructs := make([]proto.Message, 0)
+
+	for _, p := range payloadSuffixes {
+		ps := &pb.Buffer{}
+		ps.Body = append(payloadBase, p...)
+		protoStructs = append(protoStructs, ps)
+	}
+
+	return protoStructs
+}
+
+// The possible use of certain protobuf APIs like the proto.Buffer API potentially involves caching
+// on our side. This can add checks around memory allocations and possible contention.
+// Example run: go test -v -run=^$ -bench=BenchmarkProtoCodec -benchmem
+func BenchmarkProtoCodec(b *testing.B) {
+	// range of message sizes
+	payloadBaseSizes := make([]uint32, 0)
+	for i := uint32(0); i <= 12; i += 4 {
+		payloadBaseSizes = append(payloadBaseSizes, 1<<i)
+	}
+	// range of SetParallelism
+	parallelisms := make([]int, 0)
+	for i := uint32(0); i <= 16; i += 4 {
+		parallelisms = append(parallelisms, int(1<<i))
+	}
+	for _, s := range payloadBaseSizes {
+		for _, p := range parallelisms {
+			protoStructs := setupBenchmarkProtoCodecInputs(s)
+			name := fmt.Sprintf("MinPayloadSize:%v/SetParallelism(%v)", s, p)
+			b.Run(name, func(b *testing.B) {
+				codec := &codecV2{}
+				b.SetParallelism(p)
+				b.RunParallel(func(pb *testing.PB) {
+					benchmarkProtoCodec(codec, protoStructs, pb, b)
+				})
+			})
+		}
+	}
+}
+
+func benchmarkProtoCodec(codec *codecV2, protoStructs []proto.Message, pb *testing.PB, b *testing.B) {
+	counter := 0
+	for pb.Next() {
+		counter++
+		ps := protoStructs[counter%len(protoStructs)]
+		fastMarshalAndUnmarshal(codec, ps, b)
+	}
+}
+
+func fastMarshalAndUnmarshal(codec encoding.CodecV2, protoStruct proto.Message, b *testing.B) {
+	marshaledBytes, err := codec.Marshal(protoStruct)
+	if err != nil {
+		b.Errorf("codec.Marshal(_) returned an error")
+	}
+	res := pb.Buffer{}
+	if err := codec.Unmarshal(marshaledBytes, &res); err != nil {
+		b.Errorf("codec.Unmarshal(_) returned an error")
+	}
+}

encoding/proto/proto_test.go

@@ -0,0 +1,139 @@
+/*
+ *
+ * Copyright 2018 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package proto
+
+import (
+	"bytes"
+	"sync"
+	"testing"
+
+	"google.golang.org/grpc/encoding"
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/mem"
+	pb "google.golang.org/grpc/test/codec_perf"
+)
+
+func marshalAndUnmarshal(t *testing.T, codec encoding.CodecV2, expectedBody []byte) {
+	p := &pb.Buffer{}
+	p.Body = expectedBody
+
+	marshalledBytes, err := codec.Marshal(p)
+	if err != nil {
+		t.Errorf("codec.Marshal(_) returned an error")
+	}
+
+	if err := codec.Unmarshal(marshalledBytes, p); err != nil {
+		t.Errorf("codec.Unmarshal(_) returned an error")
+	}
+
+	if !bytes.Equal(p.GetBody(), expectedBody) {
+		t.Errorf("Unexpected body; got %v; want %v", p.GetBody(), expectedBody)
+	}
+}
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestBasicProtoCodecMarshalAndUnmarshal(t *testing.T) {
+	marshalAndUnmarshal(t, &codecV2{}, []byte{1, 2, 3})
+}
+
+// Try to catch possible race conditions around use of pools
+func (s) TestConcurrentUsage(t *testing.T) {
+	const (
+		numGoRoutines   = 100
+		numMarshUnmarsh = 1000
+	)
+
+	// small, arbitrary byte slices
+	protoBodies := [][]byte{
+		[]byte("one"),
+		[]byte("two"),
+		[]byte("three"),
+		[]byte("four"),
+		[]byte("five"),
+	}
+
+	var wg sync.WaitGroup
+	codec := &codecV2{}
+
+	for i := 0; i < numGoRoutines; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for k := 0; k < numMarshUnmarsh; k++ {
+				marshalAndUnmarshal(t, codec, protoBodies[k%len(protoBodies)])
+			}
+		}()
+	}
+
+	wg.Wait()
+}
+
+// TestStaggeredMarshalAndUnmarshalUsingSamePool tries to catch potential errors in which slices get
+// stomped on during reuse of a proto.Buffer.
+func (s) TestStaggeredMarshalAndUnmarshalUsingSamePool(t *testing.T) {
+	codec1 := &codecV2{}
+	codec2 := &codecV2{}
+
+	expectedBody1 := []byte{1, 2, 3}
+	expectedBody2 := []byte{4, 5, 6}
+
+	proto1 := pb.Buffer{Body: expectedBody1}
+	proto2 := pb.Buffer{Body: expectedBody2}
+
+	var m1, m2 mem.BufferSlice
+	var err error
+
+	if m1, err = codec1.Marshal(&proto1); err != nil {
+		t.Errorf("codec.Marshal(%s) failed", &proto1)
+	}
+
+	if m2, err = codec2.Marshal(&proto2); err != nil {
+		t.Errorf("codec.Marshal(%s) failed", &proto2)
+	}
+
+	if err = codec1.Unmarshal(m1, &proto1); err != nil {
+		t.Errorf("codec.Unmarshal(%v) failed", m1)
+	}
+
+	if err = codec2.Unmarshal(m2, &proto2); err != nil {
+		t.Errorf("codec.Unmarshal(%v) failed", m2)
+	}
+
+	b1 := proto1.GetBody()
+	b2 := proto2.GetBody()
+
+	for i, v := range b1 {
+		if expectedBody1[i] != v {
+			t.Errorf("expected %v at index %v but got %v", i, expectedBody1[i], v)
+		}
+	}
+
+	for i, v := range b2 {
+		if expectedBody2[i] != v {
+			t.Errorf("expected %v at index %v but got %v", i, expectedBody2[i], v)
+		}
+	}
+}