snapshot of https://github.com/grpc/grpc-go.git (master, 2026-09-15; blobs > 131072B omitted)

internal/transport/readyreader/raw_conn_linux.go

@@ -0,0 +1,39 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package readyreader
+
+import "syscall"
+
+func isRawConnSupported() bool {
+	return true
+}
+
+// sysRead uses the standard syscall package rather than the modern unix package
+// to avoid triggering the race detector. Because both packages perform sync
+// operations on a local variable to satisfy the race detector, mixing them
+// for read and write syscalls causes data races. We use syscall here to remain
+// consistent with net.Conn implementations in standard library.
+func sysRead(fd uintptr, p []byte) (int, error) {
+	return syscall.Read(int(fd), p)
+}
+
+// wouldBlock checks standard Unix non-blocking errors.
+func wouldBlock(err error) bool {
+	return err == syscall.EAGAIN || err == syscall.EWOULDBLOCK
+}

internal/transport/readyreader/raw_conn_nonlinux.go

@@ -0,0 +1,35 @@
+//go:build !linux
+
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package readyreader
+
+func isRawConnSupported() bool {
+	return false
+}
+
+// sysRead is not implemented. Support can be added in the future if necessary.
+func sysRead(uintptr, []byte) (int, error) {
+	panic("RawConn functionality is not implemented for non-unix platforms.")
+}
+
+// wouldBlock is not implemented. Support can be added in the future if necessary.
+func wouldBlock(error) bool {
+	panic("RawConn functionality is not implemented for non-unix platforms.")
+}

internal/transport/readyreader/ready_reader.go

@@ -0,0 +1,253 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+// Package readyreader provides utilities to perform non-memory-pinning reads.
+package readyreader
+
+import (
+	"io"
+	"net"
+	"syscall"
+
+	"google.golang.org/grpc/mem"
+)
+
+// Reader is an optional interface that can be implemented by [net.Conn]
+// implementations to enable gRPC to perform non-memory-pinning reads.
+type Reader interface {
+	// ReadOnReady waits for data to arrive, fetches a buffer, and performs a
+	// read. When the underlying IO is readable, it allocates a buffer of size
+	// bufSize from the pool and reads up to bufSize bytes into the buffer.
+	//
+	// It returns a pointer to the buffer so it can be returned to the pool
+	// later, the number of bytes read, and an error.
+	//
+	// Callers should always process the n > 0 bytes returned before considering
+	// the error. Doing so correctly handles I/O errors that happen after
+	// reading some bytes, as well as both of the allowed EOF behaviors.
+	ReadOnReady(bufSize int, pool mem.BufferPool) (b *[]byte, n int, err error)
+}
+
+// nonBlockingReader is optimized for non-memory-pinning reads using the RawConn
+// interface.
+type nonBlockingReader struct {
+	raw syscall.RawConn
+	// The following fields are stored as field to avoid heap allocations.
+	state  readState
+	doRead func(fd uintptr) bool
+}
+
+type readState struct {
+	// Request params.
+	bufSize int
+	pool    mem.BufferPool
+
+	// Response params.
+	readError error
+	bytesRead int
+	buf       *[]byte
+}
+
+// NewNonBlocking returns a ReadyReader if the passed reader supports
+// non-memory-pinning reads, else nil.
+func NewNonBlocking(r io.Reader) Reader {
+	if rr, ok := r.(Reader); ok {
+		return rr
+	}
+	if !isRawConnSupported() {
+		return nil
+	}
+	// We restrict the types before asserting syscall.Conn. The credentials
+	// package may return a wrapper that implements syscall.Conn by embedding
+	// both the raw connection and the encrypted connection. If the code
+	// attempts to read directly from the raw syscall.RawConn, it would read
+	// encrypted data.
+	switch r.(type) {
+	case *net.TCPConn, *net.UDPConn, *net.UnixConn, *net.IPConn:
+	default:
+		return nil
+	}
+	sysConn, ok := r.(syscall.Conn)
+	if !ok {
+		return nil
+	}
+	raw, err := sysConn.SyscallConn()
+	if err != nil {
+		return nil
+	}
+	rr := &nonBlockingReader{raw: raw}
+	rr.doRead = func(fd uintptr) bool {
+		s := &rr.state
+
+		s.buf = s.pool.Get(s.bufSize)
+		s.bytesRead, s.readError = sysRead(fd, *s.buf)
+
+		if s.readError != nil {
+			s.pool.Put(s.buf)
+			s.buf = nil
+		}
+		return !wouldBlock(s.readError)
+	}
+	return rr
+}
+
+func (c *nonBlockingReader) ReadOnReady(bufSize int, pool mem.BufferPool) (*[]byte, int, error) {
+	c.state = readState{
+		pool:    pool,
+		bufSize: bufSize,
+	}
+	err := c.raw.Read(c.doRead)
+
+	buf := c.state.buf
+	n := c.state.bytesRead
+	readErr := c.state.readError
+	c.state = readState{}
+
+	if err != nil {
+		if buf != nil {
+			pool.Put(buf)
+		}
+		return nil, 0, err
+	}
+	if readErr != nil {
+		// buffer is already released in the callback.
+		return nil, 0, readErr
+	}
+	if n == 0 {
+		// syscall.Read doesn't consider a graceful socket closure to be an
+		// error condition, but Go's io.Reader expects an EOF error.
+		pool.Put(buf)
+		return nil, 0, io.EOF
+	}
+	return buf, n, nil
+}
+
+type blockingReader struct {
+	reader io.Reader
+}
+
+func (c *blockingReader) ReadOnReady(bufSize int, pool mem.BufferPool) (*[]byte, int, error) {
+	buf := pool.Get(bufSize)
+	n, err := c.reader.Read(*buf)
+	if err != nil {
+		pool.Put(buf)
+		return nil, 0, err
+	}
+	return buf, n, nil
+}
+
+// New detects if [syscall.RawConn] is available for non-memory-pinning reads.
+// If [syscall.RawConn] is unavailable, it falls back to using the simpler
+// [io.Reader] interface for reads.
+func New(r io.Reader) Reader {
+	if r := NewNonBlocking(r); r != nil {
+		return r
+	}
+	return &blockingReader{reader: r}
+}
+
+// bufReadyReader implements buffering for a ReadyReader object.
+// A new bufReadyReader is created by calling [NewBuffered].
+type bufReadyReader struct {
+	buf       *[]byte
+	pool      mem.BufferPool
+	bufSize   int
+	rd        Reader // reader provided by the caller
+	r, w      int    // buf read and write positions
+	err       error
+	constPool constBufferPool // stored as a field to avoid heap allocations.
+}
+
+// NewBuffered returns a new [io.Reader] with a buffer of the specified size
+// which is allocated from the provided pool.
+func NewBuffered(rd Reader, size int, pool mem.BufferPool) io.Reader {
+	return &bufReadyReader{
+		rd:      rd,
+		pool:    pool,
+		bufSize: size,
+	}
+}
+
+func (b *bufReadyReader) readErr() error {
+	err := b.err
+	b.err = nil
+	return err
+}
+
+func (b *bufReadyReader) buffered() int { return b.w - b.r }
+
+// Read reads data into p. It returns the number of bytes read into p. The
+// bytes are taken from at most one Read on the underlying [ReadyReader],
+// hence n may be less than len(p). If the underlying [ReadyReader] can return
+// a non-zero count with io.EOF, then this Read method can do so as well; see
+// the [io.Reader] docs.
+func (b *bufReadyReader) Read(p []byte) (n int, err error) {
+	n = len(p)
+	if n == 0 {
+		if b.buffered() > 0 {
+			return 0, nil
+		}
+		return 0, b.readErr()
+	}
+	if b.r == b.w {
+		if b.err != nil {
+			return 0, b.readErr()
+		}
+		if len(p) >= b.bufSize {
+			// Large read, empty buffer.
+			// Read directly into p to avoid copy.
+			b.constPool.buffer = p
+			_, n, b.err = b.rd.ReadOnReady(len(p), &b.constPool)
+			return n, b.readErr()
+		}
+		// One read.
+		b.r = 0
+		b.w = 0
+		b.buf, n, b.err = b.rd.ReadOnReady(b.bufSize, b.pool)
+		if n == 0 {
+			if b.buf != nil {
+				b.pool.Put(b.buf)
+				b.buf = nil
+			}
+			return 0, b.readErr()
+		}
+		b.w += n
+	}
+
+	// copy as much as we can
+	// b.buf must be non-nil since b.r != b.w.
+	buf := *b.buf
+	n = copy(p, buf[b.r:b.w])
+	b.r += n
+	if b.r == b.w {
+		// Consumed entire buffer, release it.
+		b.pool.Put(b.buf)
+		b.buf = nil
+	}
+	return n, nil
+}
+
+type constBufferPool struct {
+	buffer []byte
+}
+
+func (p *constBufferPool) Get(int) *[]byte {
+	return &p.buffer
+}
+
+func (p *constBufferPool) Put(*[]byte) {}

internal/transport/readyreader/ready_reader_ext_test.go

@@ -0,0 +1,414 @@
+/*
+ *
+ * Copyright 2026 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package readyreader_test
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"google.golang.org/grpc/internal/grpctest"
+	"google.golang.org/grpc/internal/testutils"
+	"google.golang.org/grpc/internal/transport/readyreader"
+	"google.golang.org/grpc/mem"
+)
+
+var (
+	defaultTestTimeout      = 10 * time.Second
+	defaultTestShortTimeout = 10 * time.Millisecond
+)
+
+type s struct {
+	grpctest.Tester
+}
+
+func Test(t *testing.T) {
+	grpctest.RunSubTests(t, s{})
+}
+
+// TestReadyReader_NonRawConn verifies that ReadOnReady correctly reads data
+// from a net.Conn that doesn't support non-memory-pinning reads.
+func (s) TestReadyReader_NonRawConn(t *testing.T) {
+	data := []byte("hello world")
+	reader, writer := net.Pipe()
+	go writer.Write(data)
+
+	pool := mem.DefaultBufferPool()
+	readyReader := readyreader.New(reader)
+
+	bufHandle, n, err := readyReader.ReadOnReady(1024, pool)
+	if err != nil {
+		t.Fatalf("ReadOnReady() failed: %v", err)
+	}
+	defer pool.Put(bufHandle)
+
+	if n != len(data) {
+		t.Errorf("n = %d; want %d", n, len(data))
+	}
+	if !bytes.Equal((*bufHandle)[:n], data) {
+		t.Errorf("Read data = %s; want %s", string((*bufHandle)[:n]), string(data))
+	}
+}
+
+func (s) TestReadyReader_EOF(t *testing.T) {
+	ln, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		t.Fatalf("net.Listen failed: %v", err)
+	}
+	defer ln.Close()
+
+	data := []byte("hello")
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			return
+		}
+		if _, err := conn.Write(data); err != nil {
+			t.Errorf("Failed to write data: %v", err)
+			return
+		}
+		conn.Close()
+	}()
+
+	conn, err := net.Dial("tcp", ln.Addr().String())
+	if err != nil {
+		t.Fatalf("net.Dial failed: %v", err)
+	}
+	defer conn.Close()
+
+	pool := mem.DefaultBufferPool()
+	rr := readyreader.New(conn)
+	res, _, err := rr.ReadOnReady(len(data), pool)
+	if err != nil {
+		t.Errorf("Failed to read: %v", err)
+		return
+	}
+
+	if !bytes.Equal(*res, data) {
+		t.Errorf("Read data = %s; want %s", string(*res), string(data))
+	}
+	pool.Put(res)
+
+	// Since the server closes the TCP connection, the next read should return
+	// an io.EOF.
+	res, _, err = rr.ReadOnReady(len(data), pool)
+	if err != io.EOF {
+		t.Errorf("Read after server connection close returned err = %v; want io.EOF", err)
+	}
+	if res != nil {
+		t.Error("ReadOnReady() returned non-nil buffer.")
+	}
+}
+
+// Tests the behavior of readers wrapping TCP connections. It ensures that read
+// operations do not hold onto an allocated buffer while the connection is idle.
+// Buffers should only be permanently allocated from the pool when data is
+// actually ready on the socket, optimizing memory usage for idle connections.
+func (s) TestReadyReader_TCP(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("This test is only applicable for Linux, as RawConn functionality is not implemented for non-linux platforms.")
+	}
+	tests := []struct {
+		name string
+		read func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error)
+	}{
+		{
+			name: "ReadyReader",
+			read: func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error) {
+				rr := readyreader.New(conn)
+				bufHandle, n, err := rr.ReadOnReady(readBufSize, pool)
+				if err != nil {
+					return nil, err
+				}
+				defer pool.Put(bufHandle)
+				res := make([]byte, n)
+				copy(res, (*bufHandle)[:n])
+				return res, nil
+			},
+		},
+		{
+			name: "BufReadyReader",
+			read: func(conn net.Conn, pool *trackingBufferPool, readBufSize int) ([]byte, error) {
+				rr := readyreader.New(conn)
+				bufRR := readyreader.NewBuffered(rr, readBufSize, pool)
+				buf := make([]byte, 100)
+				n, err := bufRR.Read(buf)
+				if err != nil {
+					return nil, err
+				}
+				return buf[:n], nil
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(t.Context(), defaultTestTimeout)
+			defer cancel()
+			ln, err := net.Listen("tcp", "localhost:0")
+			if err != nil {
+				t.Fatalf("net.Listen failed: %v", err)
+			}
+			defer ln.Close()
+
+			data := []byte("hello tcp delayed")
+			connCh := make(chan net.Conn)
+			go func() {
+				conn, err := ln.Accept()
+				if err != nil {
+					return
+				}
+				connCh <- conn
+			}()
+
+			conn, err := net.Dial("tcp", ln.Addr().String())
+			if err != nil {
+				t.Fatalf("net.Dial failed: %v", err)
+			}
+			defer conn.Close()
+
+			serverConn := <-connCh
+			defer serverConn.Close()
+
+			pool := newTrackingPool(mem.DefaultBufferPool())
+			const readBufSize = 1024
+
+			resCh := make(chan []byte)
+
+			go func() {
+				res, err := tc.read(conn, pool, readBufSize)
+				if err != nil {
+					t.Errorf("Failed to read: %v", err)
+					return
+				}
+				resCh <- res
+			}()
+
+			// Verify that no read buffer is allocated for a short while. If it
+			// is allocated (e.g. for a probe), it must be returned immediately.
+			sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
+			defer sCancel()
+			if n, err := pool.requestChan.Receive(sCtx); err == nil {
+				if n != readBufSize {
+					t.Fatalf("Unexpected request buffer size: got %d, want %d", n, readBufSize)
+				}
+				if n, err := pool.putChan.Receive(ctx); err != nil {
+					t.Fatal("Read buffer allocated and NOT returned while idle.")
+				} else if n != readBufSize {
+					t.Fatalf("Unexpected returned buffer size: got %d, want %d", n, readBufSize)
+				}
+			}
+
+			// Write data to unblock.
+			serverConn.Write(data)
+
+			if n, err := pool.requestChan.Receive(ctx); err != nil {
+				t.Fatal("Context timed out while waiting for a read buffer to be allocated.")
+			} else if n != readBufSize {
+				t.Fatalf("Unexpected request buffer size: got %d, want %d", n, readBufSize)
+			}
+
+			var res []byte
+			select {
+			case res = <-resCh:
+			case <-ctx.Done():
+				t.Fatal("Context timed out waiting for read to complete.")
+			}
+			if !bytes.Equal(res, data) {
+				t.Errorf("Read data = %s; want %s", string(res), string(data))
+			}
+		})
+	}
+}
+
+// trackingBufferPool wraps a mem.BufferPool and provides channels to track
+// when buffers are requested and returned, useful for verifying allocation
+// behavior in tests.
+type trackingBufferPool struct {
+	mem.BufferPool
+	requestChan testutils.Channel
+	putChan     testutils.Channel
+}
+
+func newTrackingPool(pool mem.BufferPool) *trackingBufferPool {
+	return &trackingBufferPool{
+		BufferPool:  pool,
+		requestChan: *testutils.NewChannelWithSize(1),
+		putChan:     *testutils.NewChannelWithSize(1),
+	}
+}
+
+func (p *trackingBufferPool) Get(size int) *[]byte {
+	p.requestChan.Replace(size)
+	return p.BufferPool.Get(size)
+}
+
+func (p *trackingBufferPool) Put(b *[]byte) {
+	p.putChan.Replace(len(*b))
+	p.BufferPool.Put(b)
+}
+
+// Call Read to accumulate the text of a file
+func reads(buf io.Reader, m int) string {
+	var b [1000]byte
+	nb := 0
+	for {
+		n, err := buf.Read(b[nb : nb+m])
+		nb += n
+		if err == io.EOF {
+			break
+		}
+	}
+	return string(b[0:nb])
+}
+
+type bufReader struct {
+	name string
+	fn   func(io.Reader) string
+}
+
+var bufreaders = []bufReader{
+	{"1", func(b io.Reader) string { return reads(b, 1) }},
+	{"2", func(b io.Reader) string { return reads(b, 2) }},
+	{"3", func(b io.Reader) string { return reads(b, 3) }},
+	{"4", func(b io.Reader) string { return reads(b, 4) }},
+	{"5", func(b io.Reader) string { return reads(b, 5) }},
+	{"7", func(b io.Reader) string { return reads(b, 7) }},
+}
+
+const minReadBufferSize = 16
+
+var bufsizes = []int{
+	0, minReadBufferSize, 23, 32, 46, 64, 93, 128, 1024, 4096,
+}
+
+func (s) TestBufReader(t *testing.T) {
+	var texts [31]string
+	str := ""
+	all := ""
+	for i := range len(texts) - 1 {
+		texts[i] = str + "\n"
+		all += texts[i]
+		str += string(rune(i%26 + 'a'))
+	}
+	texts[len(texts)-1] = all
+
+	for _, text := range texts {
+		for _, bufreader := range bufreaders {
+			for _, bufsize := range bufsizes {
+				// We don't use t.Run() here to avoid excessive logging due to
+				// the large number of subtests.
+				read := readyreader.New(strings.NewReader(text))
+				buf := readyreader.NewBuffered(read, bufsize, mem.DefaultBufferPool())
+				s := bufreader.fn(buf)
+				if s != text {
+					t.Errorf("fn=%s bufsize=%d want=%q got=%q", bufreader.name, bufsize, text, s)
+				}
+			}
+		}
+	}
+}
+
+func (s) TestBufReader_ReadEmptyBuffer(t *testing.T) {
+	rr := readyreader.New(new(bytes.Buffer))
+	l := readyreader.NewBuffered(rr, minReadBufferSize, mem.DefaultBufferPool())
+	b := make([]byte, 100)
+	n, err := l.Read(b)
+	if err != io.EOF {
+		t.Errorf("expected EOF from Read, got %q %v", b[:n], err)
+	}
+}
+
+type errorThenGoodReader struct {
+	didErr bool
+	nread  int
+}
+
+var errFake = errors.New("fake error")
+
+func (r *errorThenGoodReader) Read(p []byte) (int, error) {
+	r.nread++
+	if !r.didErr {
+		r.didErr = true
+		return 0, errFake
+	}
+	return len(p), nil
+}
+
+func (s) TestBufReader_ClearError(t *testing.T) {
+	r := &errorThenGoodReader{}
+	b := readyreader.NewBuffered(readyreader.New(r), minReadBufferSize, mem.DefaultBufferPool())
+	buf := make([]byte, 1)
+	if _, err := b.Read(nil); err != nil {
+		t.Fatalf("1st nil Read = %v; want nil", err)
+	}
+	if _, err := b.Read(buf); err != errFake {
+		t.Fatalf("1st Read = %v; want errFake", err)
+	}
+	if _, err := b.Read(nil); err != nil {
+		t.Fatalf("2nd nil Read = %v; want nil", err)
+	}
+	if _, err := b.Read(buf); err != nil {
+		t.Fatalf("3rd Read with buffer = %v; want nil", err)
+	}
+	if r.nread != 2 {
+		t.Errorf("num reads = %d; want 2", r.nread)
+	}
+}
+
+type emptyThenNonEmptyReader struct {
+	r io.Reader
+	n int
+}
+
+func (r *emptyThenNonEmptyReader) Read(p []byte) (int, error) {
+	if r.n <= 0 {
+		return r.r.Read(p)
+	}
+	r.n--
+	return 0, nil
+}
+
+func (s) TestBufReader_ReadZero(t *testing.T) {
+	for _, size := range []int{100, 2} {
+		t.Run(fmt.Sprintf("bufsize=%d", size), func(t *testing.T) {
+			r := io.MultiReader(strings.NewReader("abc"), &emptyThenNonEmptyReader{r: strings.NewReader("def"), n: 1})
+			br := readyreader.NewBuffered(readyreader.New(r), size, mem.DefaultBufferPool())
+			want := func(s string, wantErr error) {
+				p := make([]byte, 50)
+				n, err := br.Read(p)
+				if err != wantErr || n != len(s) || string(p[:n]) != s {
+					t.Fatalf("read(%d) = %q, %v, want %q, %v", len(p), string(p[:n]), err, s, wantErr)
+				}
+				t.Logf("read(%d) = %q, %v", len(p), string(p[:n]), err)
+			}
+			want("abc", nil)
+			want("", nil)
+			want("def", nil)
+			want("", io.EOF)
+		})
+	}
+}