snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/mem/buffer_pool.go

@@ -0,0 +1,363 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package mem provides utilities that facilitate memory reuse in byte slices
+// that are used as buffers.
+package mem
+
+import (
+	"fmt"
+	"math/bits"
+	"slices"
+	"sort"
+	"sync"
+
+	"google.golang.org/grpc/internal"
+)
+
+const (
+	goPageSize = 4 * 1024 // 4KiB. N.B. this must be a power of 2.
+)
+
+var (
+	// BufferPoolingThreshold is the minimum size of a buffer that can be pooled.
+	// This is used to determine whether to pool buffers or allocate them directly.
+	BufferPoolingThreshold = 1 << 10
+)
+
+func init() {
+	internal.SetBufferPoolingThresholdForTesting = func(threshold int) {
+		BufferPoolingThreshold = threshold
+	}
+}
+
+var uintSize = bits.UintSize // use a variable for mocking during tests.
+
+// bufferPool is a copy of the public bufferPool interface used to avoid
+// circular dependencies.
+type bufferPool interface {
+	// Get returns a buffer with specified length from the pool.
+	Get(length int) *[]byte
+
+	// Put returns a buffer to the pool.
+	//
+	// The provided pointer must hold a prefix of the buffer obtained via
+	// BufferPool.Get to ensure the buffer's entire capacity can be re-used.
+	Put(*[]byte)
+}
+
+// BinaryTieredBufferPool is a buffer pool that uses multiple sub-pools with
+// power-of-two sizes.
+type BinaryTieredBufferPool struct {
+	// exponentToNextLargestPoolMap maps a power-of-two exponent (e.g., 12 for
+	// 4KB) to the index of the next largest sizedBufferPool. This is used by
+	// Get() to find the smallest pool that can satisfy a request for a given
+	// size.
+	exponentToNextLargestPoolMap []int
+	// exponentToPreviousLargestPoolMap maps a power-of-two exponent to the
+	// index of the previous largest sizedBufferPool. This is used by Put()
+	// to return a buffer to the most appropriate pool based on its capacity.
+	exponentToPreviousLargestPoolMap []int
+	sizedPools                       []bufferPool
+	fallbackPool                     bufferPool
+	maxPoolCap                       int // Optimization: Cache max capacity
+}
+
+// NewBinaryTieredBufferPool returns a BufferPool backed by multiple sub-pools.
+// This structure enables O(1) lookup time for Get and Put operations.
+//
+// The arguments provided are the exponents for the buffer capacities (powers
+// of 2), not the raw byte sizes. For example, to create a pool of 16KB buffers
+// (2^14 bytes), pass 14 as the argument.
+func NewBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
+	return newBinaryTiered(func(size int) bufferPool {
+		return newSizedBufferPool(size, true)
+	}, &SimpleBufferPool{shouldZero: true}, powerOfTwoExponents...)
+}
+
+// NewDirtyBinaryTieredBufferPool returns a BufferPool backed by multiple
+// sub-pools. It is similar to NewBinaryTieredBufferPool but it does not
+// initialize the buffers before returning them.
+func NewDirtyBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
+	return newBinaryTiered(func(size int) bufferPool {
+		return newSizedBufferPool(size, false)
+	}, NewDirtySimplePool(), powerOfTwoExponents...)
+}
+
+func newBinaryTiered(sizedPoolFactory func(int) bufferPool, fallbackPool bufferPool, powerOfTwoExponents ...uint8) (*BinaryTieredBufferPool, error) {
+	slices.Sort(powerOfTwoExponents)
+	powerOfTwoExponents = slices.Compact(powerOfTwoExponents)
+
+	// Determine the maximum exponent we need to support. This depends on the
+	// word size (32-bit vs 64-bit).
+	maxExponent := uintSize - 2
+	indexOfNextLargestBit := slices.Repeat([]int{-1}, maxExponent+1)
+	indexOfPreviousLargestBit := slices.Repeat([]int{-1}, maxExponent+1)
+
+	maxTier := 0
+	pools := make([]bufferPool, 0, len(powerOfTwoExponents))
+
+	for i, exp := range powerOfTwoExponents {
+		// Allocating slices of size > 2^maxExponent isn't possible on
+		// maxExponent-bit machines.
+		if int(exp) > maxExponent {
+			return nil, fmt.Errorf("mem: allocating slice of size 2^%d is not possible", exp)
+		}
+		tierSize := 1 << exp
+		pools = append(pools, sizedPoolFactory(tierSize))
+		maxTier = max(maxTier, tierSize)
+
+		// Map the exact power of 2 to this pool index.
+		indexOfNextLargestBit[exp] = i
+		indexOfPreviousLargestBit[exp] = i
+	}
+
+	// Fill gaps for Get() (Next Largest)
+	// We iterate backwards. If current is empty, take the value from the right (larger).
+	for i := maxExponent - 1; i >= 0; i-- {
+		if indexOfNextLargestBit[i] == -1 {
+			indexOfNextLargestBit[i] = indexOfNextLargestBit[i+1]
+		}
+	}
+
+	// Fill gaps for Put() (Previous Largest)
+	// We iterate forwards. If current is empty, take the value from the left (smaller).
+	for i := 1; i <= maxExponent; i++ {
+		if indexOfPreviousLargestBit[i] == -1 {
+			indexOfPreviousLargestBit[i] = indexOfPreviousLargestBit[i-1]
+		}
+	}
+
+	return &BinaryTieredBufferPool{
+		exponentToNextLargestPoolMap:     indexOfNextLargestBit,
+		exponentToPreviousLargestPoolMap: indexOfPreviousLargestBit,
+		sizedPools:                       pools,
+		maxPoolCap:                       maxTier,
+		fallbackPool:                     fallbackPool,
+	}, nil
+}
+
+// Get returns a buffer with specified length from the pool.
+func (b *BinaryTieredBufferPool) Get(size int) *[]byte {
+	return b.poolForGet(size).Get(size)
+}
+
+func (b *BinaryTieredBufferPool) poolForGet(size int) bufferPool {
+	if size == 0 || size > b.maxPoolCap {
+		return b.fallbackPool
+	}
+
+	// Calculate the exponent of the smallest power of 2 >= size.
+	// We subtract 1 from size to handle exact powers of 2 correctly.
+	//
+	// Examples:
+	// size=16 (0b10000) -> size-1=15 (0b01111) -> bits.Len=4 -> Pool for 2^4
+	// size=17 (0b10001) -> size-1=16 (0b10000) -> bits.Len=5 -> Pool for 2^5
+	querySize := uint(size - 1)
+	poolIdx := b.exponentToNextLargestPoolMap[bits.Len(querySize)]
+
+	return b.sizedPools[poolIdx]
+}
+
+// Put returns a buffer to the pool.
+func (b *BinaryTieredBufferPool) Put(buf *[]byte) {
+	// We pass the capacity of the buffer, and not the size of the buffer here.
+	// If we did the latter, all buffers would eventually move to the smallest
+	// pool.
+	b.poolForPut(cap(*buf)).Put(buf)
+}
+
+func (b *BinaryTieredBufferPool) poolForPut(bCap int) bufferPool {
+	if bCap == 0 {
+		return NopBufferPool{}
+	}
+	if bCap > b.maxPoolCap {
+		return b.fallbackPool
+	}
+	// Find the pool with the largest capacity <= bCap.
+	//
+	// We calculate the exponent of the largest power of 2 <= bCap.
+	// bits.Len(x) returns the minimum number of bits required to represent x;
+	// i.e. the number of bits up to and including the most significant bit.
+	// Subtracting 1 gives the 0-based index of the most significant bit,
+	// which is the exponent of the largest power of 2 <= bCap.
+	//
+	// Examples:
+	// cap=16 (0b10000) -> Len=5 -> 5-1=4 -> 2^4
+	// cap=15 (0b01111) -> Len=4 -> 4-1=3 -> 2^3
+	largestPowerOfTwo := bits.Len(uint(bCap)) - 1
+	poolIdx := b.exponentToPreviousLargestPoolMap[largestPowerOfTwo]
+	// The buffer is smaller than the smallest power of 2, discard it.
+	if poolIdx == -1 {
+		// Buffer is smaller than our smallest pool bucket.
+		return NopBufferPool{}
+	}
+	return b.sizedPools[poolIdx]
+}
+
+// NopBufferPool is a buffer pool that returns new buffers without pooling.
+type NopBufferPool struct{}
+
+// Get returns a buffer with specified length from the pool.
+func (NopBufferPool) Get(length int) *[]byte {
+	b := make([]byte, length)
+	return &b
+}
+
+// Put returns a buffer to the pool.
+func (NopBufferPool) Put(*[]byte) {
+}
+
+// sizedBufferPool is a BufferPool implementation that is optimized for specific
+// buffer sizes. For example, HTTP/2 frames within gRPC have a default max size
+// of 16kb and a sizedBufferPool can be configured to only return buffers with a
+// capacity of 16kb. Note that however it does not support returning larger
+// buffers and in fact panics if such a buffer is requested. Because of this,
+// this BufferPool implementation is not meant to be used on its own and rather
+// is intended to be embedded in a TieredBufferPool such that Get is only
+// invoked when the required size is smaller than or equal to defaultSize.
+type sizedBufferPool struct {
+	pool        sync.Pool
+	defaultSize int
+	shouldZero  bool
+}
+
+func (p *sizedBufferPool) Get(size int) *[]byte {
+	buf, ok := p.pool.Get().(*[]byte)
+	if !ok {
+		buf := make([]byte, size, p.defaultSize)
+		return &buf
+	}
+	b := *buf
+	if p.shouldZero {
+		clear(b[:cap(b)])
+	}
+	*buf = b[:size]
+	return buf
+}
+
+func (p *sizedBufferPool) Put(buf *[]byte) {
+	if cap(*buf) < p.defaultSize {
+		// Ignore buffers that are too small to fit in the pool. Otherwise, when
+		// Get is called it will panic as it tries to index outside the bounds
+		// of the buffer.
+		return
+	}
+	p.pool.Put(buf)
+}
+
+func newSizedBufferPool(size int, zero bool) *sizedBufferPool {
+	return &sizedBufferPool{
+		defaultSize: size,
+		shouldZero:  zero,
+	}
+}
+
+// TieredBufferPool implements the BufferPool interface with multiple tiers of
+// buffer pools for different sizes of buffers.
+type TieredBufferPool struct {
+	sizedPools   []*sizedBufferPool
+	fallbackPool SimpleBufferPool
+}
+
+// NewTieredBufferPool returns a BufferPool implementation that uses multiple
+// underlying pools of the given pool sizes.
+func NewTieredBufferPool(poolSizes ...int) *TieredBufferPool {
+	sort.Ints(poolSizes)
+	pools := make([]*sizedBufferPool, len(poolSizes))
+	for i, s := range poolSizes {
+		pools[i] = newSizedBufferPool(s, true)
+	}
+	return &TieredBufferPool{
+		sizedPools:   pools,
+		fallbackPool: SimpleBufferPool{shouldZero: true},
+	}
+}
+
+// Get returns a buffer with specified length from the pool.
+func (p *TieredBufferPool) Get(size int) *[]byte {
+	return p.getPool(size).Get(size)
+}
+
+// Put returns a buffer to the pool.
+func (p *TieredBufferPool) Put(buf *[]byte) {
+	p.getPool(cap(*buf)).Put(buf)
+}
+
+func (p *TieredBufferPool) getPool(size int) bufferPool {
+	poolIdx := sort.Search(len(p.sizedPools), func(i int) bool {
+		return p.sizedPools[i].defaultSize >= size
+	})
+
+	if poolIdx == len(p.sizedPools) {
+		return &p.fallbackPool
+	}
+
+	return p.sizedPools[poolIdx]
+}
+
+// SimpleBufferPool is an implementation of the mem.BufferPool interface that
+// attempts to pool buffers with a sync.Pool. When Get is invoked, it tries to
+// acquire a buffer from the pool but if that buffer is too small, it returns it
+// to the pool and creates a new one.
+type SimpleBufferPool struct {
+	pool       sync.Pool
+	shouldZero bool
+}
+
+// NewDirtySimplePool constructs a [SimpleBufferPool]. It does not initialize
+// the buffers before returning them. Callers must ensure they don't read the
+// buffers before writing data to them.
+func NewDirtySimplePool() *SimpleBufferPool {
+	return &SimpleBufferPool{
+		shouldZero: false,
+	}
+}
+
+// Get returns a buffer with specified length from the pool.
+func (p *SimpleBufferPool) Get(size int) *[]byte {
+	bs, ok := p.pool.Get().(*[]byte)
+	if ok && cap(*bs) >= size {
+		if p.shouldZero {
+			clear((*bs)[:cap(*bs)])
+		}
+		*bs = (*bs)[:size]
+		return bs
+	}
+
+	// A buffer was pulled from the pool, but it is too small. Put it back in
+	// the pool and create one large enough.
+	if ok {
+		p.pool.Put(bs)
+	}
+
+	// If we're going to allocate, round up to the nearest page. This way if
+	// requests frequently arrive with small variation we don't allocate
+	// repeatedly if we get unlucky and they increase over time. By default we
+	// only allocate here if size > 1MiB. Because goPageSize is a power of 2, we
+	// can round up efficiently.
+	allocSize := (size + goPageSize - 1) & ^(goPageSize - 1)
+
+	b := make([]byte, size, allocSize)
+	return &b
+}
+
+// Put returns a buffer to the pool.
+func (p *SimpleBufferPool) Put(buf *[]byte) {
+	p.pool.Put(buf)
+}

internal/mem/buffer_pool_ext_test.go

@@ -0,0 +1,118 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package mem_test
+
+import (
+	"testing"
+	"unsafe"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/mem"
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+func (s) TestBufferPool_Clears(t *testing.T) {
+	poolConfigs := []struct {
+		name        string
+		factory     func() (*mem.BinaryTieredBufferPool, error)
+		wantCleared bool
+		bufferSize  int
+	}{
+		{
+			name: "regular_sized",
+			factory: func() (*mem.BinaryTieredBufferPool, error) {
+				return mem.NewBinaryTieredBufferPool(3) // 8 bytes
+			},
+			bufferSize:  8,
+			wantCleared: true,
+		},
+		{
+			name: "regular_fallback",
+			factory: func() (*mem.BinaryTieredBufferPool, error) {
+				return mem.NewBinaryTieredBufferPool(3)
+			},
+			bufferSize:  10,
+			wantCleared: true,
+		},
+		{
+			name: "dirty_sized",
+			factory: func() (*mem.BinaryTieredBufferPool, error) {
+				return mem.NewDirtyBinaryTieredBufferPool(3)
+			},
+			bufferSize:  8,
+			wantCleared: false,
+		},
+		{
+			name: "dirty_fallback",
+			factory: func() (*mem.BinaryTieredBufferPool, error) {
+				return mem.NewDirtyBinaryTieredBufferPool(3)
+			},
+			bufferSize:  10,
+			wantCleared: false,
+		},
+	}
+
+	for _, tc := range poolConfigs {
+		t.Run(tc.name, func(t *testing.T) {
+			pool, err := tc.factory()
+			if err != nil {
+				t.Fatalf("Failed to create pool: %v", err)
+			}
+
+			for {
+				buf1 := pool.Get(tc.bufferSize)
+				// Mark the buffer with data.
+				for i := range *buf1 {
+					(*buf1)[i] = 0xAA
+				}
+				pool.Put(buf1)
+
+				buf2 := pool.Get(tc.bufferSize)
+				// Check if we got the same underlying array.
+				if unsafe.SliceData(*buf1) != unsafe.SliceData(*buf2) {
+					pool.Put(buf2)
+					continue
+				}
+
+				// We have a reused buffer. Check if it's cleared.
+				gotCleared := true
+				for _, b := range *buf2 {
+					if b != 0 {
+						gotCleared = false
+						break
+					}
+				}
+
+				if tc.wantCleared != gotCleared {
+					t.Fatalf("buffer cleared state mismatch: want %t, got %v", tc.wantCleared, gotCleared)
+				}
+
+				pool.Put(buf2)
+				break
+			}
+		})
+	}
+}

internal/mem/buffer_pool_test.go

@@ -0,0 +1,130 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package mem
+
+import "testing"
+
+var defaultBufferPoolSizeExponents = []uint8{
+	8,
+	12,
+	14, // 16KB (max HTTP/2 frame size used by gRPC)
+	15, // 32KB (default buffer size for io.Copy)
+	20, // 1MB
+}
+
+func TestNewBinaryTieredBufferPool_WordSize(t *testing.T) {
+	origUintSize := uintSize
+	defer func() { uintSize = origUintSize }()
+
+	tests := []struct {
+		name      string
+		wordSize  int
+		exponents []uint8
+		wantErr   bool
+	}{
+		{
+			name:      "32-bit_valid_exponent",
+			wordSize:  32,
+			exponents: []uint8{30},
+			wantErr:   false,
+		},
+		{
+			name:      "32-bit_invalid_exponent",
+			wordSize:  32,
+			exponents: []uint8{31},
+			wantErr:   true,
+		},
+		{
+			name:      "64-bit_valid_exponent",
+			wordSize:  64,
+			exponents: []uint8{62},
+			wantErr:   false,
+		},
+		{
+			name:      "64-bit_invalid_exponent",
+			wordSize:  64,
+			exponents: []uint8{63},
+			wantErr:   true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			uintSize = tt.wordSize
+			pool, err := NewBinaryTieredBufferPool(tt.exponents...)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("NewBinaryTieredBufferPool() error = %t, wantErr %t", err, tt.wantErr)
+			}
+			if err != nil {
+				return
+			}
+			if len(pool.exponentToNextLargestPoolMap) != tt.wordSize-1 {
+				t.Errorf("exponentToNextLargestPoolMap length = %d, want %d", len(pool.exponentToNextLargestPoolMap), tt.wordSize)
+			}
+			if len(pool.exponentToPreviousLargestPoolMap) != tt.wordSize-1 {
+				t.Errorf("exponentToPreviousLargestPoolMap length = %d, want %d", len(pool.exponentToPreviousLargestPoolMap), tt.wordSize)
+			}
+		})
+	}
+}
+
+// BenchmarkTieredPool benchmarks the performance of the tiered buffer pool
+// implementations, specifically focusing on the overhead of selecting the
+// correct bucket for a given size.
+func BenchmarkTieredPool(b *testing.B) {
+	defaultBufferPoolSizes := make([]int, len(defaultBufferPoolSizeExponents))
+	for i, exp := range defaultBufferPoolSizeExponents {
+		defaultBufferPoolSizes[i] = 1 << exp
+	}
+
+	b.Run("pool=Tiered", func(b *testing.B) {
+		p := NewTieredBufferPool(defaultBufferPoolSizes...)
+		for b.Loop() {
+			for size := range 1 << 19 {
+				// One for get, one for put.
+				_ = p.getPool(size)
+				_ = p.getPool(size)
+			}
+		}
+	})
+
+	b.Run("pool=BinaryTiered", func(b *testing.B) {
+		pool, err := NewBinaryTieredBufferPool(defaultBufferPoolSizeExponents...)
+		if err != nil {
+			b.Fatalf("Failed to create buffer pool: %v", err)
+		}
+		for b.Loop() {
+			for size := range 1 << 19 {
+				_ = pool.poolForGet(size)
+				_ = pool.poolForPut(size)
+			}
+		}
+	})
+}
+
+func TestNewBinaryTieredBufferPool_Duplicates(t *testing.T) {
+	exponents := []uint8{1, 2, 3, 4, 5, 6, 6, 5, 4, 3, 2, 1}
+	pool, err := NewBinaryTieredBufferPool(exponents...)
+	if err != nil {
+		t.Fatalf("NewBinaryTieredBufferPool() error = %v", err)
+	}
+	if len(pool.sizedPools) != 6 {
+		t.Errorf("sized buffer pool count = %d, want %d", len(pool.sizedPools), 6)
+	}
+}