snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/testutil/helpers.go

@@ -0,0 +1,105 @@
+package testutil
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+const SeaweedMiniStartupTimeout = 45 * time.Second
+
+func HasDocker() bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "docker", "version")
+	return cmd.Run() == nil
+}
+
+func FindBindIP() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		return "127.0.0.1"
+	}
+	for _, addr := range addrs {
+		ipNet, ok := addr.(*net.IPNet)
+		if !ok || ipNet.IP == nil {
+			continue
+		}
+		ip := ipNet.IP.To4()
+		if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() {
+			continue
+		}
+		return ip.String()
+	}
+	return "127.0.0.1"
+}
+
+func WriteIAMConfig(dir, accessKey, secretKey string) (string, error) {
+	iamConfigPath := filepath.Join(dir, "iam_config.json")
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": [
+        "Admin",
+        "Read",
+        "List",
+        "Tagging",
+        "Write"
+      ]
+    }
+  ]
+}`, accessKey, secretKey)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		return "", err
+	}
+	return iamConfigPath, nil
+}
+
+func WaitForService(url string, timeout time.Duration) bool {
+	client := &http.Client{Timeout: 2 * time.Second}
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return false
+		case <-ticker.C:
+			resp, err := client.Get(url)
+			if err == nil {
+				resp.Body.Close()
+				return true
+			}
+		}
+	}
+}
+
+func WaitForPort(port int, timeout time.Duration) bool {
+	deadline := time.Now().Add(timeout)
+	address := fmt.Sprintf("127.0.0.1:%d", port)
+	for time.Now().Before(deadline) {
+		conn, err := net.DialTimeout("tcp", address, 500*time.Millisecond)
+		if err == nil {
+			_ = conn.Close()
+			return true
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	return false
+}

test/testutil/ports.go

@@ -0,0 +1,228 @@
+// Package testutil provides shared test utilities for SeaweedFS integration tests.
+package testutil
+
+import (
+	"fmt"
+	"math/rand"
+	"net"
+	"testing"
+)
+
+// GrpcPortOffset is the offset weed mini uses to derive gRPC ports from HTTP ports.
+const GrpcPortOffset = 10000
+
+// linuxEphemeralPortFloor is the Linux default ip_local_port_range lower bound.
+// Ports at or above it can be claimed as the source port of a transient
+// outbound connection during mini's own startup (e.g. volume dialing master
+// gRPC) in the window between allocation and the listener bind, surfacing as
+// "bind: address already in use".
+const linuxEphemeralPortFloor = 32768
+
+// miniPortMin/miniPortMax bound the random HTTP port range for weed mini.
+// The cap keeps both the HTTP port and its +GrpcPortOffset gRPC counterpart
+// below linuxEphemeralPortFloor, out of the range the kernel draws ephemeral
+// source ports from, so the steal above cannot happen. Deriving the cap from
+// GrpcPortOffset keeps the two in lockstep if the offset ever changes.
+const (
+	miniPortMin = 10000
+	miniPortMax = linuxEphemeralPortFloor - GrpcPortOffset
+)
+
+// miniDefaultPorts are the weed mini flag defaults (see weed/command/mini.go).
+// A test only overrides services it uses; unspecified services still bind
+// these defaults, so allocation must avoid handing them out (or any value
+// whose gRPC offset would collide with them).
+var miniDefaultPorts = []int{
+	9333,  // master.port
+	8888,  // filer.port
+	9340,  // volume.port
+	8333,  // s3.port
+	8181,  // s3.port.iceberg
+	9101,  // s3.port.lance
+	7333,  // webdav.port
+	23646, // admin.port
+}
+
+func reservedMiniPorts() map[int]bool {
+	r := make(map[int]bool, len(miniDefaultPorts)*2)
+	for _, p := range miniDefaultPorts {
+		r[p] = true
+		r[p+GrpcPortOffset] = true
+	}
+	return r
+}
+
+// AllocatePorts allocates count unique free ports atomically.
+// All listeners are held open until every port is obtained, preventing
+// the OS from recycling a port between successive allocations.
+func AllocatePorts(count int) ([]int, error) {
+	listeners := make([]net.Listener, 0, count)
+	ports := make([]int, 0, count)
+	for i := 0; i < count; i++ {
+		l, err := net.Listen("tcp", "127.0.0.1:0")
+		if err != nil {
+			for _, ll := range listeners {
+				_ = ll.Close()
+			}
+			return nil, err
+		}
+		listeners = append(listeners, l)
+		ports = append(ports, l.Addr().(*net.TCPAddr).Port)
+	}
+	for _, l := range listeners {
+		_ = l.Close()
+	}
+	return ports, nil
+}
+
+// MustAllocatePorts is a testing wrapper for AllocatePorts.
+func MustAllocatePorts(t *testing.T, count int) []int {
+	t.Helper()
+	ports, err := AllocatePorts(count)
+	if err != nil {
+		t.Fatalf("Failed to allocate %d free ports: %v", count, err)
+	}
+	return ports
+}
+
+// AllocateMiniPorts allocates n free ports where each port and its gRPC
+// counterpart (port + GrpcPortOffset) are available and don't collide
+// with any other allocated port or its gRPC counterpart. All listeners
+// are held open until the entire batch is allocated, preventing the OS
+// from recycling ports between allocations. Use this when ports will be
+// passed to weed mini without explicit gRPC port flags, so mini will
+// derive gRPC ports as HTTP + 10000.
+//
+// Listeners are bound on all interfaces (":port") rather than 127.0.0.1
+// to match weed mini's availability check (isPortAvailable). A port can
+// be free on loopback but held by another process on a different
+// interface; reserving only on loopback lets mini's check fail and
+// trigger gRPC port shifting, which then causes weed shell to dial the
+// wrong port and hang.
+func AllocateMiniPorts(count int) ([]int, error) {
+	const (
+		minPort = miniPortMin
+		maxPort = miniPortMax
+	)
+	reserved := reservedMiniPorts()
+	ports := make([]int, 0, count)
+	var listeners []net.Listener
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	for idx := 0; idx < count; idx++ {
+		found := false
+		for i := 0; i < 1000; i++ {
+			port := minPort + rand.Intn(maxPort-minPort)
+			grpcPort := port + GrpcPortOffset
+
+			if reserved[port] || reserved[grpcPort] {
+				continue
+			}
+
+			l1, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+			if err != nil {
+				continue
+			}
+
+			l2, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
+			if err != nil {
+				l1.Close()
+				continue
+			}
+
+			listeners = append(listeners, l1, l2)
+			reserved[port] = true
+			reserved[grpcPort] = true
+			ports = append(ports, port)
+			found = true
+			break
+		}
+		if !found {
+			return nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, count)
+		}
+	}
+
+	return ports, nil
+}
+
+// MustFreeMiniPorts allocates n ports suitable for weed mini, ensuring
+// each port's gRPC offset (port + 10000) doesn't collide with any other
+// allocated port. names is used only for error messages.
+func MustFreeMiniPorts(t *testing.T, names []string) []int {
+	t.Helper()
+	ports, err := AllocateMiniPorts(len(names))
+	if err != nil {
+		t.Fatalf("failed to allocate mini ports for %v: %v", names, err)
+	}
+	return ports
+}
+
+// MustFreeMiniPort allocates a single weed mini port.
+func MustFreeMiniPort(t *testing.T, name string) int {
+	t.Helper()
+	return MustFreeMiniPorts(t, []string{name})[0]
+}
+
+// AllocatePortSet atomically allocates miniCount master-style port pairs (each
+// with port and port+GrpcPortOffset reserved) plus regularCount additional
+// distinct ports. All listeners are held open until the entire batch is
+// allocated, so a mini gRPC port cannot be recycled as a regular port mid-batch.
+func AllocatePortSet(miniCount, regularCount int) (mini []int, regular []int, err error) {
+	const (
+		minPort = miniPortMin
+		maxPort = miniPortMax
+	)
+	reserved := reservedMiniPorts()
+	mini = make([]int, 0, miniCount)
+	var listeners []net.Listener
+	defer func() {
+		for _, l := range listeners {
+			l.Close()
+		}
+	}()
+
+	for idx := 0; idx < miniCount; idx++ {
+		found := false
+		for i := 0; i < 1000; i++ {
+			port := minPort + rand.Intn(maxPort-minPort)
+			grpcPort := port + GrpcPortOffset
+			if reserved[port] || reserved[grpcPort] {
+				continue
+			}
+			l1, lErr := net.Listen("tcp", fmt.Sprintf(":%d", port))
+			if lErr != nil {
+				continue
+			}
+			l2, lErr := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
+			if lErr != nil {
+				l1.Close()
+				continue
+			}
+			listeners = append(listeners, l1, l2)
+			reserved[port] = true
+			reserved[grpcPort] = true
+			mini = append(mini, port)
+			found = true
+			break
+		}
+		if !found {
+			return nil, nil, fmt.Errorf("failed to allocate mini port %d of %d", idx+1, miniCount)
+		}
+	}
+
+	regular = make([]int, 0, regularCount)
+	for i := 0; i < regularCount; i++ {
+		l, lErr := net.Listen("tcp", ":0")
+		if lErr != nil {
+			return nil, nil, lErr
+		}
+		listeners = append(listeners, l)
+		regular = append(regular, l.Addr().(*net.TCPAddr).Port)
+	}
+
+	return mini, regular, nil
+}

test/testutil/ports_test.go

@@ -0,0 +1,73 @@
+package testutil
+
+import "testing"
+
+// AllocateMiniPorts must never hand out a port that weed mini will reserve
+// for one of its default services (or that default's gRPC offset). A real
+// failure: Filer was given 33646 (Admin default 23646 + GrpcPortOffset),
+// which mini then refused as "reserved for gRPC calculation".
+func TestAllocateMiniPortsAvoidsMiniDefaults(t *testing.T) {
+	reserved := reservedMiniPorts()
+	for iter := 0; iter < 200; iter++ {
+		ports, err := AllocateMiniPorts(4)
+		if err != nil {
+			t.Fatalf("iter %d: AllocateMiniPorts: %v", iter, err)
+		}
+		for _, p := range ports {
+			if reserved[p] {
+				t.Fatalf("iter %d: allocated port %d is a mini default (or gRPC offset)", iter, p)
+			}
+			if reserved[p+GrpcPortOffset] {
+				t.Fatalf("iter %d: allocated port %d has gRPC offset %d colliding with a mini default",
+					iter, p, p+GrpcPortOffset)
+			}
+		}
+	}
+}
+
+// AllocateMiniPorts must keep every port it hands out, and that port's gRPC
+// counterpart, below 32768 - the Linux default ip_local_port_range floor.
+// A real failure: filer was allocated 44204, then lost the bind to a transient
+// outbound connection that had grabbed 44204 as an ephemeral source port
+// during mini startup ("bind: address already in use").
+func TestAllocateMiniPortsBelowEphemeralFloor(t *testing.T) {
+	const ephemeralFloor = linuxEphemeralPortFloor
+	for iter := 0; iter < 200; iter++ {
+		ports, err := AllocateMiniPorts(4)
+		if err != nil {
+			t.Fatalf("iter %d: AllocateMiniPorts: %v", iter, err)
+		}
+		for _, p := range ports {
+			if p+GrpcPortOffset >= ephemeralFloor {
+				t.Fatalf("iter %d: port %d (gRPC %d) reaches the ephemeral range floor %d",
+					iter, p, p+GrpcPortOffset, ephemeralFloor)
+			}
+		}
+	}
+}
+
+func TestAllocatePortSetNoGrpcCollision(t *testing.T) {
+	// Run a few iterations to catch the OS-recycles-just-closed-port race
+	// that previously hit regular ports when the mini gRPC offset was freed
+	// between AllocateMiniPorts and AllocatePorts calls.
+	for iter := 0; iter < 20; iter++ {
+		mini, regular, err := AllocatePortSet(1, 3)
+		if err != nil {
+			t.Fatalf("iter %d: AllocatePortSet: %v", iter, err)
+		}
+		if len(mini) != 1 || len(regular) != 3 {
+			t.Fatalf("iter %d: unexpected counts mini=%d regular=%d", iter, len(mini), len(regular))
+		}
+		reserved := map[int]bool{
+			mini[0]:                  true,
+			mini[0] + GrpcPortOffset: true,
+		}
+		for _, p := range regular {
+			if reserved[p] {
+				t.Fatalf("iter %d: regular port %d collides with mini pair %d/%d",
+					iter, p, mini[0], mini[0]+GrpcPortOffset)
+			}
+			reserved[p] = true
+		}
+	}
+}