snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/s3api/s3api_object_handlers_copy_bench_test.go

@@ -0,0 +1,186 @@
+package s3api
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// benchEnv stands up a fake source volume (returns N bytes on GET) and a
+// fake destination volume (parses an incoming multipart POST and discards
+// the body, asserting the data arrived intact via SHA-256). It returns the
+// URLs and an AssignVolumeResponse pointing at the destination.
+type benchEnv struct {
+	srcSrv  *httptest.Server
+	dstSrv  *httptest.Server
+	payload []byte
+	dstHash [32]byte
+	assign  *filer_pb.AssignVolumeResponse
+}
+
+func newBenchEnv(b *testing.B, payloadSize int) *benchEnv {
+	b.Helper()
+	payload := make([]byte, payloadSize)
+	for i := range payload {
+		payload[i] = byte(i*31 + 7) // arbitrary repeatable pattern
+	}
+	wantHash := sha256.Sum256(payload)
+
+	srcSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Honor Range header if present (UploadPartCopy passes one).
+		start, end := int64(0), int64(len(payload)-1)
+		if rng := r.Header.Get("Range"); rng != "" {
+			var s, e int64
+			if _, err := fmt.Sscanf(rng, "bytes=%d-%d", &s, &e); err == nil {
+				start, end = s, e
+			}
+		}
+		w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10))
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(payload[start : end+1])
+	}))
+
+	dstSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mr, err := r.MultipartReader()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		part, err := mr.NextPart()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		h := sha256.New()
+		if _, err := io.Copy(h, part); err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		var got [32]byte
+		copy(got[:], h.Sum(nil))
+		if got != wantHash {
+			http.Error(w, "hash mismatch", http.StatusBadRequest)
+			return
+		}
+		w.WriteHeader(http.StatusCreated)
+		_, _ = io.WriteString(w, `{"size":`+strconv.Itoa(payloadSize)+`}`)
+	}))
+
+	dstURL, _ := neturl(dstSrv.URL)
+	assign := &filer_pb.AssignVolumeResponse{
+		FileId: "1,benchmark",
+		Location: &filer_pb.Location{
+			Url:       dstURL,
+			PublicUrl: dstURL,
+		},
+		Auth: "",
+	}
+
+	return &benchEnv{
+		srcSrv:  srcSrv,
+		dstSrv:  dstSrv,
+		payload: payload,
+		dstHash: wantHash,
+		assign:  assign,
+	}
+}
+
+func (e *benchEnv) close() {
+	e.srcSrv.Close()
+	e.dstSrv.Close()
+}
+
+// neturl extracts host:port from "http://host:port".
+func neturl(rawURL string) (string, error) {
+	const prefix = "http://"
+	if len(rawURL) < len(prefix) || rawURL[:len(prefix)] != prefix {
+		return "", fmt.Errorf("unexpected URL: %q", rawURL)
+	}
+	host := rawURL[len(prefix):]
+	if _, _, err := net.SplitHostPort(host); err != nil {
+		return "", err
+	}
+	return host, nil
+}
+
+// BenchmarkCopyChunk_Buffered measures the cost of the existing buffered
+// chunk-copy path (downloadChunkData + uploadChunkData). The path
+// allocates one chunk-sized []byte and one multipart-encoded buffer per
+// call.
+func BenchmarkCopyChunk_Buffered(b *testing.B) {
+	for _, size := range []int{1 << 20, 8 << 20, 64 << 20} {
+		b.Run(humanByteName(size), func(b *testing.B) {
+			util_http.InitGlobalHttpClient()
+			env := newBenchEnv(b, size)
+			defer env.close()
+
+			s3a := &S3ApiServer{}
+			b.ResetTimer()
+			b.SetBytes(int64(size))
+			b.ReportAllocs()
+
+			for i := 0; i < b.N; i++ {
+				data, err := s3a.downloadChunkData(env.srcSrv.URL, env.assign.FileId, 0, int64(size), nil)
+				if err != nil {
+					b.Fatalf("download: %v", err)
+				}
+				if err := s3a.uploadChunkData(data, env.assign, false); err != nil {
+					b.Fatalf("upload: %v", err)
+				}
+			}
+		})
+	}
+}
+
+// BenchmarkCopyChunk_Streamed measures the cost of the io.Pipe streaming
+// chunk-copy path. The path holds only the pipe's hand-off buffer (~32
+// KiB) plus http transport buffers per call, regardless of chunk size.
+func BenchmarkCopyChunk_Streamed(b *testing.B) {
+	for _, size := range []int{1 << 20, 8 << 20, 64 << 20} {
+		b.Run(humanByteName(size), func(b *testing.B) {
+			util_http.InitGlobalHttpClient()
+			env := newBenchEnv(b, size)
+			defer env.close()
+
+			s3a := &S3ApiServer{}
+			b.ResetTimer()
+			b.SetBytes(int64(size))
+			b.ReportAllocs()
+
+			for i := 0; i < b.N; i++ {
+				if err := s3a.streamCopyChunkRange(context.Background(),
+					env.srcSrv.URL, env.assign.FileId, 0, int64(size),
+					true /*isFullChunk*/, env.assign); err != nil {
+					b.Fatalf("stream: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func humanByteName(n int) string {
+	switch {
+	case n >= 1<<30:
+		return strconv.Itoa(n>>30) + "GiB"
+	case n >= 1<<20:
+		return strconv.Itoa(n>>20) + "MiB"
+	case n >= 1<<10:
+		return strconv.Itoa(n>>10) + "KiB"
+	default:
+		return strconv.Itoa(n) + "B"
+	}
+}
+
+// Compile-time sanity: ensure the multipart library version we depend on
+// is the one whose framing we mirror in streamCopyChunkRange.
+var _ = multipart.NewWriter