snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/util/http/chunk_proxy.go

@@ -0,0 +1,62 @@
+package http
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ProxyChunkIdParam is the query parameter a filer reads to serve one chunk
+// from the volume server holding it, for callers that cannot reach volume
+// servers directly.
+const ProxyChunkIdParam = "proxyChunkId"
+
+var (
+	filerSigningKey            security.SigningKey
+	filerSigningKeyExpires     int
+	filerReadSigningKey        security.SigningKey
+	filerReadSigningKeyExpires int
+	loadFilerJwtConfigOnce     sync.Once
+)
+
+func loadFilerJwtConfig() {
+	v := util.GetViper()
+	filerSigningKey = security.SigningKey(v.GetString("jwt.filer_signing.key"))
+	filerSigningKeyExpires = v.GetInt("jwt.filer_signing.expires_after_seconds")
+	if filerSigningKeyExpires == 0 {
+		filerSigningKeyExpires = 10
+	}
+	filerReadSigningKey = security.SigningKey(v.GetString("jwt.filer_signing.read.key"))
+	filerReadSigningKeyExpires = v.GetInt("jwt.filer_signing.read.expires_after_seconds")
+	if filerReadSigningKeyExpires == 0 {
+		filerReadSigningKeyExpires = 60
+	}
+}
+
+// JwtForFilerServer generates a JWT for the filer's HTTP API if jwt.filer_signing
+// is configured for that access level, the way filer.JwtForVolumeServer does for
+// volume servers. Empty when the key is unset, which is also when the filer
+// serves the request unsigned.
+func JwtForFilerServer(isWrite bool) string {
+	loadFilerJwtConfigOnce.Do(loadFilerJwtConfig)
+	if isWrite {
+		return string(security.GenJwtForFilerServer(filerSigningKey, filerSigningKeyExpires))
+	}
+	return string(security.GenJwtForFilerServer(filerReadSigningKey, filerReadSigningKeyExpires))
+}
+
+// ProxyChunkUrl builds the request that reads or writes one chunk through a
+// filer instead of the volume server holding it.
+func ProxyChunkUrl(filerAddress string, fileId string) string {
+	return fmt.Sprintf("http://%s/?%s=%s", filerAddress, ProxyChunkIdParam, fileId)
+}
+
+// IsProxyChunkUrl reports whether a chunk URL addresses a filer's chunk proxy
+// rather than a volume server. Such a request is authorized by the filer, which
+// attaches the volume credential itself, so the token it carries is a filer one.
+func IsProxyChunkUrl(urlString string) bool {
+	return strings.Contains(urlString, ProxyChunkIdParam+"=")
+}

weed/util/http/chunk_proxy_test.go

@@ -0,0 +1,74 @@
+package http
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestIsProxyChunkUrl(t *testing.T) {
+	for _, tc := range []struct {
+		urlString string
+		want      bool
+	}{
+		{ProxyChunkUrl("filer:8888", "3,01637037d6"), true},
+		{"http://filer:8888/?proxyChunkId=3,01637037d6&readDeleted=true", true},
+		{"http://volume:8080/3,01637037d6", false},
+		{"http://volume:8080/3,01637037d6?readDeleted=true", false},
+		{"", false},
+	} {
+		if got := IsProxyChunkUrl(tc.urlString); got != tc.want {
+			t.Errorf("IsProxyChunkUrl(%q) = %v, want %v", tc.urlString, got, tc.want)
+		}
+	}
+}
+
+// The credential a proxied chunk carries is a filer one, signed with the key
+// for the access level the caller is about to use.
+func TestJwtForFilerServer(t *testing.T) {
+	const (
+		writeKey = "filer-write-key"
+		readKey  = "filer-read-key"
+	)
+	v := util.GetViper()
+	v.Set("jwt.filer_signing.key", writeKey)
+	v.Set("jwt.filer_signing.read.key", readKey)
+
+	for _, tc := range []struct {
+		name     string
+		isWrite  bool
+		signedBy string
+		otherKey string
+	}{
+		{"read", false, readKey, writeKey},
+		{"write", true, writeKey, readKey},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			token := security.EncodedJwt(JwtForFilerServer(tc.isWrite))
+			claims := &security.SeaweedFilerClaims{}
+			if _, err := security.DecodeJwt(security.SigningKey(tc.signedBy), token, claims); err != nil {
+				t.Fatalf("token does not validate against the %s key: %v", tc.name, err)
+			}
+			if claims.ExpiresAt == nil {
+				t.Fatal("token never expires")
+			}
+			if _, err := security.DecodeJwt(security.SigningKey(tc.otherKey), token, &security.SeaweedFilerClaims{}); err == nil {
+				t.Fatal("token also validates against the other access level's key")
+			}
+		})
+	}
+}
+
+func TestJwtForFilerServerWithoutKeys(t *testing.T) {
+	loadFilerJwtConfigOnce.Do(func() {})
+	write, read := filerSigningKey, filerReadSigningKey
+	t.Cleanup(func() { filerSigningKey, filerReadSigningKey = write, read })
+	filerSigningKey, filerReadSigningKey = nil, nil
+
+	for _, isWrite := range []bool{false, true} {
+		if token := JwtForFilerServer(isWrite); token != "" {
+			t.Errorf("unconfigured signing key still produced %q", token)
+		}
+	}
+}

weed/util/http/client/http_client.go

@@ -0,0 +1,340 @@
+package client
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/grpc/credentials/tls/certprovider"
+
+	"github.com/seaweedfs/seaweedfs/weed/security/certreload"
+	util "github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/spf13/viper"
+)
+
+var (
+	loadSecurityConfigOnce sync.Once
+)
+
+// Intra-cluster peers (volume/filer/master) answer headers within milliseconds.
+// A peer that is TCP-reachable but not answering -- a volume server still loading
+// after a restart, or a stale keep-alive to a container that returned on a new IP
+// -- otherwise blocks a chunk read or a replicated write forever. The response
+// timeout bounds that wait so the read fails over to another replica and the
+// write fails fast to retry; the idle timeout evicts sockets to a departed server.
+const (
+	responseHeaderTimeout = 30 * time.Second
+	idleConnTimeout       = 90 * time.Second
+)
+
+type HTTPClient struct {
+	Client            *http.Client
+	Transport         *http.Transport
+	expectHttpsScheme bool
+	// certProvider, when non-nil, owns a background refresh goroutine for
+	// the client mTLS cert/key pair. Close() must be called to stop it.
+	certProvider certprovider.Provider
+}
+
+// Close stops any background cert refresh goroutine. Safe to call on a
+// client that was constructed without mTLS. Existing pooled connections
+// are also closed via CloseIdleConnections.
+func (httpClient *HTTPClient) Close() {
+	if httpClient == nil {
+		return
+	}
+	if httpClient.certProvider != nil {
+		httpClient.certProvider.Close()
+		httpClient.certProvider = nil
+	}
+	if httpClient.Client != nil {
+		httpClient.Client.CloseIdleConnections()
+	}
+}
+
+func (httpClient *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = httpClient.GetHttpScheme()
+	return httpClient.Client.Do(req)
+}
+
+func (httpClient *HTTPClient) Get(url string) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Get(url)
+}
+
+func (httpClient *HTTPClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Post(url, contentType, body)
+}
+
+func (httpClient *HTTPClient) PostForm(url string, data url.Values) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.PostForm(url, data)
+}
+
+func (httpClient *HTTPClient) Head(url string) (resp *http.Response, err error) {
+	url, err = httpClient.NormalizeHttpScheme(url)
+	if err != nil {
+		return nil, err
+	}
+	return httpClient.Client.Head(url)
+}
+func (httpClient *HTTPClient) CloseIdleConnections() {
+	httpClient.Client.CloseIdleConnections()
+}
+
+func (httpClient *HTTPClient) GetClientTransport() *http.Transport {
+	return httpClient.Transport
+}
+
+func (httpClient *HTTPClient) GetHttpScheme() string {
+	if httpClient.expectHttpsScheme {
+		return "https"
+	}
+	return "http"
+}
+
+func (httpClient *HTTPClient) IsTLSVerified() bool {
+	return httpClient.expectHttpsScheme && httpClient.Transport != nil && (httpClient.Transport.TLSClientConfig == nil || !httpClient.Transport.TLSClientConfig.InsecureSkipVerify)
+}
+
+func rejectHttpsDowngrade(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return http.ErrUseLastResponse
+	}
+	if len(via) > 0 && via[0].URL.Scheme == "https" && req.URL.Scheme != "https" {
+		return http.ErrUseLastResponse
+	}
+	return nil
+}
+
+func (httpClient *HTTPClient) NormalizeHttpScheme(rawURL string) (string, error) {
+	expectedScheme := httpClient.GetHttpScheme()
+
+	if !(strings.HasPrefix(rawURL, "http://") || strings.HasPrefix(rawURL, "https://")) {
+		return expectedScheme + "://" + rawURL, nil
+	}
+
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return "", err
+	}
+
+	if expectedScheme != parsedURL.Scheme {
+		parsedURL.Scheme = expectedScheme
+	}
+	return parsedURL.String(), nil
+}
+
+func NewHttpClient(clientName ClientName, opts ...HttpClientOpt) (*HTTPClient, error) {
+	httpClient := HTTPClient{}
+	httpClient.expectHttpsScheme = checkIsHttpsClientEnabled(clientName)
+	var tlsConfig *tls.Config = nil
+
+	if httpClient.expectHttpsScheme {
+		certFileName, keyFileName, hasClientCert, err := clientCertPaths(clientName)
+		if err != nil {
+			return nil, err
+		}
+
+		clientCaCert, clientCaCertName, err := getClientCaCert(clientName)
+		if err != nil {
+			return nil, err
+		}
+
+		if hasClientCert || len(clientCaCert) != 0 {
+			caCertPool, err := createHTTPClientCertPool(clientCaCert, clientCaCertName)
+			if err != nil {
+				return nil, err
+			}
+
+			tlsConfig = &tls.Config{
+				RootCAs:            caCertPool,
+				InsecureSkipVerify: false,
+			}
+
+			if hasClientCert {
+				getClientCert, provider, err := certreload.NewClientGetCertificate(certFileName, keyFileName)
+				if err != nil {
+					return nil, fmt.Errorf("error loading client certificate and key: %s", err)
+				}
+				tlsConfig.GetClientCertificate = getClientCert
+				httpClient.certProvider = provider
+			}
+		}
+
+		if getBoolOptionFromSecurityConfiguration(clientName, "insecure_skip_verify") {
+			if tlsConfig == nil {
+				tlsConfig = &tls.Config{}
+			}
+			tlsConfig.InsecureSkipVerify = true
+		}
+	}
+
+	httpClient.Transport = &http.Transport{
+		MaxIdleConns:        1024,
+		MaxIdleConnsPerHost: 1024,
+		TLSClientConfig:     tlsConfig,
+		// Bind outbound HTTP to the -ip.bind source address.
+		DialContext:           util.OutboundDialContext,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+		IdleConnTimeout:       idleConnTimeout,
+	}
+	httpClient.Client = &http.Client{
+		Transport:     httpClient.Transport,
+		CheckRedirect: rejectHttpsDowngrade,
+	}
+
+	for _, opt := range opts {
+		opt(&httpClient)
+	}
+	return &httpClient, nil
+}
+
+func getStringOptionFromSecurityConfiguration(clientName ClientName, stringOptionName string) string {
+	util.LoadSecurityConfiguration()
+	return viper.GetString(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), stringOptionName))
+}
+
+func getBoolOptionFromSecurityConfiguration(clientName ClientName, boolOptionName string) bool {
+	util.LoadSecurityConfiguration()
+	return viper.GetBool(fmt.Sprintf("https.%s.%s", clientName.LowerCaseString(), boolOptionName))
+}
+
+func checkIsHttpsClientEnabled(clientName ClientName) bool {
+	return getBoolOptionFromSecurityConfiguration(clientName, "enabled")
+}
+
+func getFileContentFromSecurityConfiguration(clientName ClientName, fileType string) ([]byte, string, error) {
+	if fileName := getStringOptionFromSecurityConfiguration(clientName, fileType); fileName != "" {
+		fileContent, err := os.ReadFile(fileName)
+		if err != nil {
+			return nil, fileName, err
+		}
+		return fileContent, fileName, err
+	}
+	return nil, "", nil
+}
+
+// clientCertPaths reads the https.<clientName>.{cert,key} paths from the
+// security config, validates they're either both set or both empty, and
+// returns them along with a hasClientCert flag. Loading is deferred to
+// certreload so the cert/key pair is picked up from disk on rotation.
+func clientCertPaths(clientName ClientName) (certFile, keyFile string, hasClientCert bool, err error) {
+	certFile = getStringOptionFromSecurityConfiguration(clientName, "cert")
+	keyFile = getStringOptionFromSecurityConfiguration(clientName, "key")
+	if certFile == "" && keyFile == "" {
+		return "", "", false, nil
+	}
+	if certFile == "" || keyFile == "" {
+		return "", "", false, fmt.Errorf("https.%s: both cert and key must be set (got cert=%q key=%q)", clientName.LowerCaseString(), certFile, keyFile)
+	}
+	return certFile, keyFile, true, nil
+}
+
+func getClientCaCert(clientName ClientName) ([]byte, string, error) {
+	return getFileContentFromSecurityConfiguration(clientName, "ca")
+}
+
+// NewHttpClientWithTLS creates an HTTPClient with explicit TLS certificate
+// parameters instead of reading from the global security configuration.
+// This is used by filer.sync to create per-cluster HTTP clients when clusters
+// use different certificates.
+func NewHttpClientWithTLS(certFile, keyFile, caFile string, insecureSkipVerify bool, opts ...HttpClientOpt) (*HTTPClient, error) {
+	httpClient := HTTPClient{}
+	httpClient.expectHttpsScheme = true
+	var tlsConfig *tls.Config
+
+	if (certFile == "") != (keyFile == "") {
+		return nil, fmt.Errorf("both cert and key are required for mTLS, got cert=%q key=%q", certFile, keyFile)
+	}
+
+	var getClientCert func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+	if certFile != "" && keyFile != "" {
+		cb, provider, err := certreload.NewClientGetCertificate(certFile, keyFile)
+		if err != nil {
+			return nil, fmt.Errorf("error loading client certificate and key: %s", err)
+		}
+		getClientCert = cb
+		httpClient.certProvider = provider
+	}
+	// closeProviderOnError ensures the cert reloader's background refresh
+	// goroutine is shut down if any subsequent step fails before we hand
+	// the client back to the caller.
+	closeProviderOnError := func() {
+		if httpClient.certProvider != nil {
+			httpClient.certProvider.Close()
+			httpClient.certProvider = nil
+		}
+	}
+
+	var caCertPool *x509.CertPool
+	if caFile != "" {
+		caCert, err := os.ReadFile(caFile)
+		if err != nil {
+			closeProviderOnError()
+			return nil, fmt.Errorf("error reading CA cert %s: %s", caFile, err)
+		}
+		caCertPool, err = createHTTPClientCertPool(caCert, caFile)
+		if err != nil {
+			closeProviderOnError()
+			return nil, err
+		}
+	}
+
+	if getClientCert != nil || caCertPool != nil || insecureSkipVerify {
+		tlsConfig = &tls.Config{
+			GetClientCertificate: getClientCert,
+			RootCAs:              caCertPool,
+			InsecureSkipVerify:   insecureSkipVerify,
+		}
+	}
+
+	httpClient.Transport = &http.Transport{
+		MaxIdleConns:        1024,
+		MaxIdleConnsPerHost: 1024,
+		TLSClientConfig:     tlsConfig,
+		// Bind outbound HTTP to the -ip.bind source address.
+		DialContext:           util.OutboundDialContext,
+		ResponseHeaderTimeout: responseHeaderTimeout,
+		IdleConnTimeout:       idleConnTimeout,
+	}
+	httpClient.Client = &http.Client{
+		Transport:     httpClient.Transport,
+		CheckRedirect: rejectHttpsDowngrade,
+	}
+
+	for _, opt := range opts {
+		opt(&httpClient)
+	}
+	return &httpClient, nil
+}
+
+func createHTTPClientCertPool(certContent []byte, fileName string) (*x509.CertPool, error) {
+	certPool := x509.NewCertPool()
+	if len(certContent) == 0 {
+		return certPool, nil
+	}
+
+	ok := certPool.AppendCertsFromPEM(certContent)
+	if !ok {
+		return nil, fmt.Errorf("error processing certificate in %s", fileName)
+	}
+	return certPool, nil
+}

weed/util/http/client/http_client_interface.go

@@ -0,0 +1,16 @@
+package client
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+)
+
+type HTTPClientInterface interface {
+	Do(req *http.Request) (*http.Response, error)
+	Get(url string) (resp *http.Response, err error)
+	Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
+	PostForm(url string, data url.Values) (resp *http.Response, err error)
+	Head(url string) (resp *http.Response, err error)
+	CloseIdleConnections()
+}

weed/util/http/client/http_client_name.go

@@ -0,0 +1,14 @@
+package client
+
+import "strings"
+
+type ClientName int
+
+//go:generate stringer -type=ClientName -output=http_client_name_string.go
+const (
+	Client ClientName = iota
+)
+
+func (name *ClientName) LowerCaseString() string {
+	return strings.ToLower(name.String())
+}

weed/util/http/client/http_client_name_string.go

@@ -0,0 +1,23 @@
+// Code generated by "stringer -type=ClientName -output=http_client_name_string.go"; DO NOT EDIT.
+
+package client
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[Client-0]
+}
+
+const _ClientName_name = "Client"
+
+var _ClientName_index = [...]uint8{0, 6}
+
+func (i ClientName) String() string {
+	if i < 0 || i >= ClientName(len(_ClientName_index)-1) {
+		return "ClientName(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _ClientName_name[_ClientName_index[i]:_ClientName_index[i+1]]
+}

weed/util/http/client/http_client_opt.go

@@ -0,0 +1,29 @@
+package client
+
+import (
+	"net"
+	"time"
+)
+
+type HttpClientOpt = func(clientCfg *HTTPClient)
+
+func AddDialContext(httpClient *HTTPClient) {
+	dialContext := (&net.Dialer{
+		Timeout:   10 * time.Second,
+		KeepAlive: 10 * time.Second,
+	}).DialContext
+
+	httpClient.Transport.DialContext = dialContext
+	httpClient.Client.Transport = httpClient.Transport
+}
+
+// WithResponseHeaderTimeout overrides the transport's default response-header
+// timeout. Mainly for tests that need a short deadline against an unresponsive
+// peer without mutating the package default.
+func WithResponseHeaderTimeout(timeout time.Duration) HttpClientOpt {
+	return func(httpClient *HTTPClient) {
+		if httpClient.Transport != nil {
+			httpClient.Transport.ResponseHeaderTimeout = timeout
+		}
+	}
+}

weed/util/http/client/http_client_test.go

@@ -0,0 +1,127 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestNewHttpClientSetsResponseTimeouts(t *testing.T) {
+	c, err := NewHttpClient(Client)
+	if err != nil {
+		t.Fatalf("NewHttpClient: %v", err)
+	}
+	if got := c.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("ResponseHeaderTimeout = %v, want %v", got, responseHeaderTimeout)
+	}
+	if got := c.Transport.IdleConnTimeout; got != idleConnTimeout {
+		t.Errorf("IdleConnTimeout = %v, want %v", got, idleConnTimeout)
+	}
+	// AddDialContext must keep the response timeouts intact.
+	c2, err := NewHttpClient(Client, AddDialContext)
+	if err != nil {
+		t.Fatalf("NewHttpClient(AddDialContext): %v", err)
+	}
+	if got := c2.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("AddDialContext dropped ResponseHeaderTimeout: got %v", got)
+	}
+	// WithResponseHeaderTimeout overrides the default.
+	c3, err := NewHttpClient(Client, WithResponseHeaderTimeout(time.Second))
+	if err != nil {
+		t.Fatalf("NewHttpClient(WithResponseHeaderTimeout): %v", err)
+	}
+	if got := c3.Transport.ResponseHeaderTimeout; got != time.Second {
+		t.Errorf("WithResponseHeaderTimeout not applied: got %v", got)
+	}
+	// The TLS constructor must carry the same defaults.
+	cTLS, err := NewHttpClientWithTLS("", "", "", true)
+	if err != nil {
+		t.Fatalf("NewHttpClientWithTLS: %v", err)
+	}
+	if got := cTLS.Transport.ResponseHeaderTimeout; got != responseHeaderTimeout {
+		t.Errorf("NewHttpClientWithTLS ResponseHeaderTimeout = %v, want %v", got, responseHeaderTimeout)
+	}
+	if got := cTLS.Transport.IdleConnTimeout; got != idleConnTimeout {
+		t.Errorf("NewHttpClientWithTLS IdleConnTimeout = %v, want %v", got, idleConnTimeout)
+	}
+}
+
+// A peer that is TCP-reachable but never answers -- a volume server still
+// loading its volumes after a restart, or a stale keep-alive to a container
+// that came back on a new IP -- must not block a read or a replicated write
+// forever. ResponseHeaderTimeout turns that into a prompt timeout error so the
+// caller can fail over to another replica or retry.
+func TestResponseHeaderTimeoutUnblocksUnresponsivePeer(t *testing.T) {
+	server, release := newUnresponsiveServer(t, false)
+	defer server.Close()
+	defer close(release)
+
+	c, err := NewHttpClient(Client, WithResponseHeaderTimeout(200*time.Millisecond))
+	if err != nil {
+		t.Fatalf("NewHttpClient: %v", err)
+	}
+	assertTimesOut(t, c, server.URL)
+}
+
+// Same guarantee over the TLS client path (NewHttpClientWithTLS).
+func TestResponseHeaderTimeoutUnblocksUnresponsivePeerTLS(t *testing.T) {
+	server, release := newUnresponsiveServer(t, true)
+	defer server.Close()
+	defer close(release)
+
+	c, err := NewHttpClientWithTLS("", "", "", true, WithResponseHeaderTimeout(200*time.Millisecond))
+	if err != nil {
+		t.Fatalf("NewHttpClientWithTLS: %v", err)
+	}
+	assertTimesOut(t, c, server.URL)
+}
+
+// newUnresponsiveServer returns a server whose handler blocks until the returned
+// channel is closed, so it accepts the connection but never sends response
+// headers. Callers must close the channel before server.Close() -- defers run
+// LIFO, so register the Close defer first.
+func newUnresponsiveServer(t *testing.T, tls bool) (*httptest.Server, chan struct{}) {
+	t.Helper()
+	release := make(chan struct{})
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		<-release
+	})
+	if tls {
+		return httptest.NewTLSServer(handler), release
+	}
+	return httptest.NewServer(handler), release
+}
+
+func assertTimesOut(t *testing.T, c *HTTPClient, url string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+	if err != nil {
+		t.Fatalf("new request: %v", err)
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		resp, doErr := c.Do(req)
+		if resp != nil {
+			resp.Body.Close()
+		}
+		done <- doErr
+	}()
+
+	select {
+	case doErr := <-done:
+		if doErr == nil {
+			t.Fatal("expected a timeout error from an unresponsive peer, got nil")
+		}
+		var netErr net.Error
+		if !errors.As(doErr, &netErr) || !netErr.Timeout() {
+			t.Fatalf("expected a timeout error, got %v", doErr)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Do() blocked well past the response header timeout")
+	}
+}

weed/util/http/http_global_client_init.go

@@ -0,0 +1,37 @@
+package http
+
+import (
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+var (
+	globalHttpClient     *util_http_client.HTTPClient
+	globalHttpClientOnce sync.Once
+)
+
+func NewGlobalHttpClient(opt ...util_http_client.HttpClientOpt) (*util_http_client.HTTPClient, error) {
+	return util_http_client.NewHttpClient(util_http_client.Client, opt...)
+}
+
+// GetGlobalHttpClient returns the process-wide HTTP client, initializing it on
+// first use. Lazy init keeps callers that bypass weed.go's main (in-process
+// test harnesses, libraries) from dereferencing a nil client.
+func GetGlobalHttpClient() *util_http_client.HTTPClient {
+	globalHttpClientOnce.Do(initGlobalHttpClient)
+	return globalHttpClient
+}
+
+func InitGlobalHttpClient() {
+	globalHttpClientOnce.Do(initGlobalHttpClient)
+}
+
+func initGlobalHttpClient() {
+	client, err := NewGlobalHttpClient()
+	if err != nil {
+		glog.Fatalf("error init global http client: %v", err)
+	}
+	globalHttpClient = client
+}

weed/util/http/http_global_client_util.go

@@ -0,0 +1,814 @@
+package http
+
+import (
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+	"github.com/seaweedfs/seaweedfs/weed/util/request_id"
+
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client"
+)
+
+var ErrNotFound = fmt.Errorf("not found")
+var ErrTooManyRequests = fmt.Errorf("too many requests")
+
+type jwtSigningReadConfig struct {
+	key     security.SigningKey
+	expires int
+}
+
+var (
+	jwtSigningReadConfigPtr atomic.Pointer[jwtSigningReadConfig]
+	loadJwtConfigOnce       sync.Once
+)
+
+func AppendQueryParameter(rawURL, key, value string) string {
+	encoded := url.Values{key: []string{value}}.Encode()
+	fragment := ""
+	if fragmentIndex := strings.Index(rawURL, "#"); fragmentIndex >= 0 {
+		fragment = rawURL[fragmentIndex:]
+		rawURL = rawURL[:fragmentIndex]
+	}
+
+	var result string
+	switch {
+	case strings.Contains(rawURL, "?"):
+		if strings.HasSuffix(rawURL, "?") || strings.HasSuffix(rawURL, "&") {
+			result = rawURL + encoded
+		} else {
+			result = rawURL + "&" + encoded
+		}
+	default:
+		result = rawURL + "?" + encoded
+	}
+	return result + fragment
+}
+
+func loadJwtConfig() {
+	v := util.GetViper()
+	jwtSigningReadConfigPtr.Store(&jwtSigningReadConfig{
+		key:     security.SigningKey(v.GetString("jwt.signing.read.key")),
+		expires: v.GetInt("jwt.signing.read.expires_after_seconds"),
+	})
+}
+
+// ReloadJwtSigningReadConfig re-reads the volume read-signing key from the
+// already-reloaded security config, so operators can rotate it via SIGHUP
+// without restarting the process.
+func ReloadJwtSigningReadConfig() {
+	loadJwtConfig()
+}
+
+func Post(url string, values url.Values) ([]byte, error) {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Body.Close()
+	b, err := io.ReadAll(r.Body)
+	if r.StatusCode >= 400 {
+		if err != nil {
+			return nil, fmt.Errorf("%s: %d - %s", url, r.StatusCode, string(b))
+		} else {
+			return nil, fmt.Errorf("%s: %s", url, r.Status)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// github.com/seaweedfs/seaweedfs/unmaintained/repeated_vacuum/repeated_vacuum.go
+// may need increasing http.Client.Timeout
+func Get(url string) ([]byte, bool, error) {
+	return GetAuthenticated(context.Background(), url, "")
+}
+
+func GetAuthenticated(ctx context.Context, url, jwt string) ([]byte, bool, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, true, err
+	}
+	maybeAddAuth(request, jwt)
+	request.Header.Add("Accept-Encoding", "gzip")
+
+	response, err := GetGlobalHttpClient().Do(request)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(request.URL.Host)
+		}
+		return nil, true, err
+	}
+	recordReachable(request.URL.Host)
+	defer CloseResponse(response)
+
+	var reader io.ReadCloser
+	switch response.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(response.Body)
+		if err != nil {
+			return nil, true, err
+		}
+		defer reader.Close()
+	default:
+		reader = response.Body
+	}
+
+	b, err := io.ReadAll(reader)
+	if response.StatusCode >= 400 {
+		retryable := response.StatusCode >= 500
+		return nil, retryable, fmt.Errorf("%s: %s", url, response.Status)
+	}
+	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, false, ctxErr
+		}
+		return nil, false, err
+	}
+	return b, false, nil
+}
+
+func Head(url string) (http.Header, error) {
+	r, err := GetGlobalHttpClient().Head(url)
+	if err != nil {
+		return nil, err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode >= 400 {
+		return nil, fmt.Errorf("%s: %s", url, r.Status)
+	}
+	return r.Header, nil
+}
+
+func maybeAddAuth(req *http.Request, jwt string) {
+	if jwt != "" {
+		req.Header.Set("Authorization", security.BearerPrefix+string(jwt))
+	}
+}
+
+func Delete(url string, jwt string) error {
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+	maybeAddAuth(req, jwt)
+	resp, e := GetGlobalHttpClient().Do(req)
+	if e != nil {
+		return e
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	switch resp.StatusCode {
+	case http.StatusNotFound, http.StatusNoContent, http.StatusAccepted, http.StatusOK:
+		return nil
+	}
+	m := make(map[string]interface{})
+	if e := json.Unmarshal(body, &m); e == nil {
+		if s, ok := m["error"].(string); ok {
+			return errors.New(s)
+		}
+	}
+	return errors.New(string(body))
+}
+
+func DeleteProxied(url string, jwt string) (body []byte, httpStatus int, err error) {
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return
+	}
+	maybeAddAuth(req, jwt)
+	resp, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	httpStatus = resp.StatusCode
+	return
+}
+
+func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachBuffer func([]byte)) error {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode != 200 {
+		return fmt.Errorf("%s: %s", url, r.Status)
+	}
+	for {
+		n, err := r.Body.Read(allocatedBytes)
+		if n > 0 {
+			eachBuffer(allocatedBytes[:n])
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+}
+
+func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error {
+	r, err := GetGlobalHttpClient().PostForm(url, values)
+	if err != nil {
+		return err
+	}
+	defer CloseResponse(r)
+	if r.StatusCode != 200 {
+		return fmt.Errorf("%s: %s", url, r.Status)
+	}
+	return readFn(r.Body)
+}
+
+func DownloadFile(fileUrl string, jwt string, offset ...int64) (filename string, header http.Header, resp *http.Response, e error) {
+	return DownloadFileWithClient(GetGlobalHttpClient(), fileUrl, jwt, offset...)
+}
+
+// DownloadFileWithClient is like DownloadFile but uses the provided HTTP client
+// instead of the global one. This is used by filer.sync to download from
+// remote clusters that use different TLS certificates.
+func DownloadFileWithClient(client *util_http_client.HTTPClient, fileUrl string, jwt string, offset ...int64) (filename string, header http.Header, resp *http.Response, e error) {
+	if client == nil {
+		return "", nil, nil, fmt.Errorf("nil HTTP client in DownloadFileWithClient")
+	}
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	maybeAddAuth(req, jwt)
+
+	var rangeOffset int64
+	if len(offset) > 0 {
+		rangeOffset = offset[0]
+	}
+	if rangeOffset > 0 {
+		req.Header.Set("Range", fmt.Sprintf("bytes=%d-", rangeOffset))
+	}
+
+	response, err := client.Do(req)
+	if err != nil {
+		return "", nil, nil, err
+	}
+
+	if rangeOffset > 0 {
+		expected := fmt.Sprintf("bytes %d-", rangeOffset)
+		if response.StatusCode != http.StatusPartialContent ||
+			!strings.HasPrefix(response.Header.Get("Content-Range"), expected) {
+			CloseResponse(response)
+			return "", nil, nil, fmt.Errorf("range request %q to %s returned %s with Content-Range %q",
+				req.Header.Get("Range"), fileUrl, response.Status, response.Header.Get("Content-Range"))
+		}
+	}
+
+	header = response.Header
+	contentDisposition := response.Header["Content-Disposition"]
+	if len(contentDisposition) > 0 {
+		idx := strings.Index(contentDisposition[0], "filename=")
+		if idx != -1 {
+			filename = contentDisposition[0][idx+len("filename="):]
+			filename = strings.Trim(filename, "\"")
+		}
+	}
+	resp = response
+	return
+}
+
+func Do(req *http.Request) (resp *http.Response, err error) {
+	return GetGlobalHttpClient().Do(req)
+}
+
+func NormalizeUrl(url string) (string, error) {
+	return GetGlobalHttpClient().NormalizeHttpScheme(url)
+}
+
+func ReadUrl(ctx context.Context, fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, buf []byte) (int64, error) {
+
+	if cipherKey != nil {
+		var n int
+		_, err := readEncryptedUrl(ctx, fileUrl, "", cipherKey, isContentCompressed, isFullChunk, offset, size, func(data []byte) {
+			n = copy(buf, data)
+		})
+		return int64(n), err
+	}
+
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return 0, err
+	}
+	if !isFullChunk {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
+	} else {
+		req.Header.Set("Accept-Encoding", "gzip")
+	}
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return 0, err
+	}
+	defer CloseResponse(r)
+
+	if r.StatusCode >= 400 {
+		return 0, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return 0, err
+		}
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
+	var (
+		i, m int
+		n    int64
+	)
+
+	// refers to https://github.com/golang/go/blob/master/src/bytes/buffer.go#L199
+	// commit id c170b14c2c1cfb2fd853a37add92a82fd6eb4318
+	for {
+		m, err = reader.Read(buf[i:])
+		i += m
+		n += int64(m)
+		if err == io.EOF {
+			return n, nil
+		}
+		if err != nil {
+			return n, err
+		}
+		if n == int64(len(buf)) {
+			break
+		}
+	}
+	// drains the response body to avoid memory leak
+	data, _ := io.ReadAll(reader)
+	if len(data) != 0 {
+		glog.V(1).InfofCtx(ctx, "%s reader has remaining %d bytes", contentEncoding, len(data))
+	}
+	return n, err
+}
+
+func ReadUrlAsStream(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
+	if cipherKey != nil {
+		return readEncryptedUrl(ctx, fileUrl, jwt, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return false, err
+	}
+	maybeAddAuth(req, jwt)
+
+	if isFullChunk {
+		req.Header.Add("Accept-Encoding", "gzip")
+	} else {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
+	}
+	request_id.InjectToRequest(ctx, req)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(req.URL.Host)
+		}
+		return true, err
+	}
+	recordReachable(req.URL.Host)
+	defer CloseResponse(r)
+	if r.StatusCode >= 400 {
+		if r.StatusCode == http.StatusNotFound {
+			return true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			return false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests)
+		}
+		retryable = r.StatusCode >= 499
+		return retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return true, err
+		}
+		defer reader.Close()
+	default:
+		reader = r.Body
+	}
+
+	var (
+		m int
+	)
+	buf := mem.Allocate(256 * 1024)
+	defer mem.Free(buf)
+
+	for {
+		// Check for context cancellation before each read
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+
+		m, err = reader.Read(buf)
+		if m > 0 {
+			fn(buf[:m])
+		}
+		if err == io.EOF {
+			return false, nil
+		}
+		if err != nil {
+			return true, err
+		}
+	}
+
+}
+
+func readEncryptedUrl(ctx context.Context, fileUrl, jwt string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
+	encryptedData, retryable, err := GetAuthenticated(ctx, fileUrl, jwt)
+	if err != nil {
+		return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
+	}
+	decryptedData, err := util.Decrypt(encryptedData, util.CipherKey(cipherKey))
+	if err != nil {
+		return false, fmt.Errorf("decrypt %s: %v", fileUrl, err)
+	}
+	if isContentCompressed {
+		decryptedData, err = util.DecompressData(decryptedData)
+		if err != nil {
+			glog.V(0).InfofCtx(ctx, "unzip decrypt %s: %v", fileUrl, err)
+		}
+	}
+	if len(decryptedData) < int(offset)+size {
+		return false, fmt.Errorf("read decrypted %s size %d [%d, %d)", fileUrl, len(decryptedData), offset, int(offset)+size)
+	}
+	if isFullChunk {
+		fn(decryptedData)
+	} else {
+		sliceEnd := int(offset) + size
+		fn(decryptedData[int(offset):sliceEnd])
+	}
+	return false, nil
+}
+
+func ReadUrlAsReaderCloser(fileUrl string, jwt string, rangeHeader string) (*http.Response, io.ReadCloser, error) {
+
+	req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	if rangeHeader != "" {
+		req.Header.Add("Range", rangeHeader)
+	} else {
+		req.Header.Add("Accept-Encoding", "gzip")
+	}
+
+	maybeAddAuth(req, jwt)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	if r.StatusCode >= 400 {
+		CloseResponse(r)
+		return nil, nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	var reader io.ReadCloser
+	contentEncoding := r.Header.Get("Content-Encoding")
+	switch contentEncoding {
+	case "gzip":
+		reader, err = gzip.NewReader(r.Body)
+		if err != nil {
+			return nil, nil, err
+		}
+	default:
+		reader = r.Body
+	}
+
+	return r, reader, nil
+}
+
+func CloseResponse(resp *http.Response) {
+	if resp == nil || resp.Body == nil {
+		return
+	}
+	reader := &CountingReader{reader: resp.Body}
+	io.Copy(io.Discard, reader)
+	resp.Body.Close()
+	if reader.BytesRead > 0 {
+		glog.V(1).Infof("response leftover %d bytes", reader.BytesRead)
+	}
+}
+
+func CloseRequest(req *http.Request) {
+	reader := &CountingReader{reader: req.Body}
+	io.Copy(io.Discard, reader)
+	req.Body.Close()
+	if reader.BytesRead > 0 {
+		glog.V(1).Infof("request leftover %d bytes", reader.BytesRead)
+	}
+}
+
+type CountingReader struct {
+	reader    io.Reader
+	BytesRead int
+}
+
+func (r *CountingReader) Read(p []byte) (n int, err error) {
+	n, err = r.reader.Read(p)
+	r.BytesRead += n
+	return n, err
+}
+
+// refreshedUrls asks for a fresh location list and reports whether it is worth
+// retrying on: a list that comes back empty, or identical to the one that just
+// failed everywhere, says the locations were never the problem.
+func refreshedUrls(ctx context.Context, refreshUrls RefreshUrlsFunc, current []string, fileId string) ([]string, bool) {
+	if refreshUrls == nil {
+		return nil, false
+	}
+	fresh := refreshUrls()
+	if len(fresh) == 0 || SameUrls(current, fresh) {
+		return nil, false
+	}
+	glog.V(0).InfofCtx(ctx, "chunk %s failed on every known location, retrying on %d fresh ones", fileId, len(fresh))
+	return fresh, true
+}
+
+// SameUrls reports whether two location lists hold the same URLs, regardless of
+// order: lookups shuffle the locations they return, so comparing positionally
+// would read a reshuffle of the very same replicas as a fresh set.
+func SameUrls(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	counts := make(map[string]int, len(a))
+	for _, url := range a {
+		counts[url]++
+	}
+	for _, url := range b {
+		if counts[url] == 0 {
+			return false
+		}
+		counts[url]--
+	}
+	return true
+}
+
+// RefreshUrlsFunc supplies a fresh location list for a chunk. The retry loops
+// call it at most once: after every location in the list failed, to retry on
+// the fresh list at once, or after a later location answered for one that
+// failed, so the next read starts from what the cluster knows now instead of
+// trying the same dead replica again. Returning nil or the same list leaves
+// the caller on the original locations.
+type RefreshUrlsFunc func() []string
+
+// RetriedFetchChunkData reads a chunk, trying every location before backing off
+// and trying them again. refreshUrls may be nil; when it is not, a pass in which
+// every location failed is treated as a stale list rather than a slow cluster,
+// and the fresh list is tried immediately instead of after the next backoff.
+// A pass that failed on one location and succeeded on another refreshes the
+// list for the reads that follow.
+func RetriedFetchChunkData(ctx context.Context, buffer []byte, urlStrings []string, cipherKey []byte, isGzipped bool, isFullChunk bool, offset int64, fileId string, refreshUrls RefreshUrlsFunc) (n int, err error) {
+
+	var jwt security.EncodedJwt
+	if len(urlStrings) > 0 && IsProxyChunkUrl(urlStrings[0]) {
+		jwt = security.EncodedJwt(JwtForFilerServer(false))
+	} else {
+		loadJwtConfigOnce.Do(loadJwtConfig)
+		if cfg := jwtSigningReadConfigPtr.Load(); cfg != nil && len(cfg.key) > 0 {
+			jwt = security.GenJwtForVolumeServer(cfg.key, cfg.expires, fileId)
+		}
+	}
+
+	// For unencrypted, non-gzipped full chunks, use direct buffer read
+	// This avoids the 64KB intermediate buffer and callback overhead
+	if cipherKey == nil && !isGzipped && isFullChunk {
+		return retriedFetchChunkDataDirect(ctx, buffer, urlStrings, string(jwt), fileId, refreshUrls)
+	}
+
+	var shouldRetry bool
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		// Check for context cancellation before starting retry loop
+		select {
+		case <-ctx.Done():
+			return n, ctx.Err()
+		default:
+		}
+
+		var failed bool
+		for _, urlString := range ReachableFirst(urlStrings) {
+			// Check for context cancellation before each volume server request
+			select {
+			case <-ctx.Done():
+				return n, ctx.Err()
+			default:
+			}
+
+			n = 0
+			if strings.Contains(urlString, "%") {
+				urlString = url.PathEscape(urlString)
+			}
+			shouldRetry, err = ReadUrlAsStream(ctx, AppendQueryParameter(urlString, "readDeleted", "true"), string(jwt), cipherKey, isGzipped, isFullChunk, offset, len(buffer), func(data []byte) {
+				// Check for context cancellation during data processing
+				select {
+				case <-ctx.Done():
+					// Stop processing data when context is cancelled
+					return
+				default:
+				}
+
+				if n < len(buffer) {
+					x := copy(buffer[n:], data)
+					n += x
+				}
+			})
+			if !shouldRetry {
+				break
+			}
+			if err != nil {
+				failed = true
+				glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
+			} else {
+				break
+			}
+		}
+		if err == nil && failed && refreshUrls != nil {
+			refreshUrls()
+		}
+		if err != nil && shouldRetry {
+			if fresh, ok := refreshedUrls(ctx, refreshUrls, urlStrings, fileId); ok {
+				urlStrings, refreshUrls = fresh, nil
+				continue
+			}
+			refreshUrls = nil
+			glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
+			// Sleep with proper context cancellation and timer cleanup
+			timer := time.NewTimer(waitTime)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return n, ctx.Err()
+			case <-timer.C:
+				// Continue with retry
+			}
+		} else {
+			break
+		}
+	}
+
+	return n, err
+
+}
+
+// retriedFetchChunkDataDirect reads chunk data directly into the buffer without
+// intermediate buffering. This reduces memory copies and improves throughput
+// for large chunk reads.
+func retriedFetchChunkDataDirect(ctx context.Context, buffer []byte, urlStrings []string, jwt, fileId string, refreshUrls RefreshUrlsFunc) (n int, err error) {
+	var shouldRetry bool
+
+	for waitTime := time.Second; waitTime < util.RetryWaitTime; waitTime += waitTime / 2 {
+		select {
+		case <-ctx.Done():
+			return 0, ctx.Err()
+		default:
+		}
+
+		var failed bool
+		for _, urlString := range ReachableFirst(urlStrings) {
+			select {
+			case <-ctx.Done():
+				return 0, ctx.Err()
+			default:
+			}
+
+			n, shouldRetry, err = readUrlDirectToBuffer(ctx, AppendQueryParameter(urlString, "readDeleted", "true"), jwt, buffer)
+			if err == nil {
+				if failed && refreshUrls != nil {
+					refreshUrls()
+				}
+				return n, nil
+			}
+			if !shouldRetry {
+				break
+			}
+			failed = true
+			glog.V(0).InfofCtx(ctx, "read %s failed, err: %v", urlString, err)
+		}
+
+		if err != nil && shouldRetry {
+			if fresh, ok := refreshedUrls(ctx, refreshUrls, urlStrings, fileId); ok {
+				urlStrings, refreshUrls = fresh, nil
+				continue
+			}
+			refreshUrls = nil
+			glog.V(0).InfofCtx(ctx, "retry reading in %v", waitTime)
+			timer := time.NewTimer(waitTime)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return 0, ctx.Err()
+			case <-timer.C:
+			}
+		} else {
+			break
+		}
+	}
+
+	return n, err
+}
+
+// readUrlDirectToBuffer reads HTTP response directly into the provided buffer,
+// avoiding intermediate buffer allocations and copies.
+func readUrlDirectToBuffer(ctx context.Context, fileUrl, jwt string, buffer []byte) (n int, retryable bool, err error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileUrl, nil)
+	if err != nil {
+		return 0, false, err
+	}
+	maybeAddAuth(req, jwt)
+	request_id.InjectToRequest(ctx, req)
+
+	r, err := GetGlobalHttpClient().Do(req)
+	if err != nil {
+		if ctx.Err() == nil {
+			recordUnreachable(req.URL.Host)
+		}
+		return 0, true, err
+	}
+	recordReachable(req.URL.Host)
+	defer CloseResponse(r)
+
+	if r.StatusCode >= 400 {
+		if r.StatusCode == http.StatusNotFound {
+			return 0, true, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrNotFound)
+		}
+		if r.StatusCode == http.StatusTooManyRequests {
+			return 0, false, fmt.Errorf("%s: %s: %w", fileUrl, r.Status, ErrTooManyRequests)
+		}
+		retryable = r.StatusCode >= 499
+		return 0, retryable, fmt.Errorf("%s: %s", fileUrl, r.Status)
+	}
+
+	// Read directly into the buffer without intermediate copying
+	// This is significantly faster for large chunks (16MB+)
+	var totalRead int
+	for totalRead < len(buffer) {
+		select {
+		case <-ctx.Done():
+			return totalRead, false, ctx.Err()
+		default:
+		}
+
+		m, readErr := r.Body.Read(buffer[totalRead:])
+		totalRead += m
+		if readErr != nil {
+			if readErr == io.EOF {
+				// Return io.ErrUnexpectedEOF if we haven't filled the buffer
+				// This prevents silent data corruption from truncated responses
+				if totalRead < len(buffer) {
+					return totalRead, true, io.ErrUnexpectedEOF
+				}
+				return totalRead, false, nil
+			}
+			return totalRead, true, readErr
+		}
+	}
+
+	return totalRead, false, nil
+}

weed/util/http/http_global_client_util_test.go

@@ -0,0 +1,259 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAppendQueryParameter(t *testing.T) {
+	testCases := []struct {
+		name     string
+		rawURL   string
+		key      string
+		value    string
+		expected string
+	}{
+		{
+			name:     "without existing query",
+			rawURL:   "http://example.com/3,abc",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/3,abc?readDeleted=true",
+		},
+		{
+			name:     "with existing query",
+			rawURL:   "http://example.com/?proxyChunkId=3,abc",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?proxyChunkId=3,abc&readDeleted=true",
+		},
+		{
+			name:     "with trailing question mark",
+			rawURL:   "http://example.com/?",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?readDeleted=true",
+		},
+		{
+			name:     "with trailing ampersand",
+			rawURL:   "http://example.com/?proxyChunkId=3,abc&",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/?proxyChunkId=3,abc&readDeleted=true",
+		},
+		{
+			name:     "encodes values",
+			rawURL:   "http://example.com/data",
+			key:      "note",
+			value:    "space value",
+			expected: "http://example.com/data?note=space+value",
+		},
+		{
+			name:     "preserves fragment",
+			rawURL:   "http://example.com/data#frag",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "http://example.com/data?readDeleted=true#frag",
+		},
+		{
+			name:     "blank url",
+			rawURL:   "",
+			key:      "readDeleted",
+			value:    "true",
+			expected: "?readDeleted=true",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := AppendQueryParameter(tc.rawURL, tc.key, tc.value)
+			if actual != tc.expected {
+				t.Fatalf("expected %q, got %q", tc.expected, actual)
+			}
+		})
+	}
+}
+func TestReadUrlAsStreamReturnsGzipReaderError(t *testing.T) {
+	InitGlobalHttpClient()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Encoding", "gzip")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("not gzip"))
+	}))
+	defer server.Close()
+
+	_, err := ReadUrlAsStream(context.Background(), server.URL, "", nil, false, true, 0, 0, func(data []byte) {})
+	if err == nil {
+		t.Fatal("ReadUrlAsStream returned nil error for invalid gzip response")
+	}
+}
+
+func TestDeleteReturnsInvalidRequestErrorBeforeAddingAuth(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("Delete panicked before returning the request error: %v", r)
+		}
+	}()
+
+	if err := Delete("http://[::1", "jwt"); err == nil {
+		t.Fatal("expected invalid request error")
+	}
+}
+
+func TestDeleteTreatsNoContentAsSuccess(t *testing.T) {
+	InitGlobalHttpClient()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodDelete {
+			t.Errorf("expected DELETE, got %s", r.Method)
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	if err := Delete(server.URL, ""); err != nil {
+		t.Fatalf("expected 204 DELETE to succeed, got %v", err)
+	}
+}
+
+func TestDeleteProxiedReturnsInvalidRequestErrorBeforeAddingAuth(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("DeleteProxied panicked before returning the request error: %v", r)
+		}
+	}()
+
+	if _, _, err := DeleteProxied("http://[::1", "jwt"); err == nil {
+		t.Fatal("expected invalid request error")
+	}
+}
+
+// TestRetriedFetchChunkDataRetriesFreshUrlsImmediately covers the case the
+// refresh hook exists for: every location the caller knew about is gone, and
+// the data is live somewhere the caller has not heard of yet. The read must
+// land on the fresh location without first sitting through the backoff ladder.
+func TestRetriedFetchChunkDataRetriesFreshUrlsImmediately(t *testing.T) {
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+
+	// A port nothing listens on: the address is well formed, the dial fails.
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	refreshed := 0
+	buffer := make([]byte, len(payload))
+	start := time.Now()
+	n, err := RetriedFetchChunkData(context.Background(), buffer, []string{deadURL + "/3,abc"}, nil, false, true, 0, "3,abc",
+		func() []string {
+			refreshed++
+			return []string{live.URL + "/3,abc"}
+		})
+	elapsed := time.Since(start)
+
+	if err != nil {
+		t.Fatalf("fetch with a refreshed location: %v", err)
+	}
+	if n != len(payload) || string(buffer[:n]) != string(payload) {
+		t.Fatalf("got %q, want %q", buffer[:n], payload)
+	}
+	if refreshed != 1 {
+		t.Fatalf("refresh called %d times, want exactly 1", refreshed)
+	}
+	// The first backoff is a full second; landing well under it is the point.
+	if elapsed > 500*time.Millisecond {
+		t.Fatalf("took %v, expected the retry to skip the backoff", elapsed)
+	}
+}
+
+// TestRetriedFetchChunkDataKeepsBackoffWhenLocationsAreUnchanged makes sure a
+// refresh that returns the same list is treated as "the locations were never
+// the problem" rather than as a reason to spin.
+func TestRetriedFetchChunkDataKeepsBackoffWhenLocationsAreUnchanged(t *testing.T) {
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	urls := []string{deadURL + "/3,abc"}
+	refreshed := 0
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+
+	buffer := make([]byte, 4)
+	_, err := RetriedFetchChunkData(ctx, buffer, urls, nil, false, true, 0, "3,abc", func() []string {
+		refreshed++
+		return urls
+	})
+	if err == nil {
+		t.Fatal("expected the fetch to fail against a dead location")
+	}
+	if refreshed != 1 {
+		t.Fatalf("refresh called %d times, want exactly 1", refreshed)
+	}
+}
+
+// TestRetriedFetchChunkDataRefreshesLocationsAfterPartialFailure covers a read
+// that fails on a cached dead replica and succeeds on the next one: the read
+// itself is fine, but the list it came from is stale and must be refreshed so
+// later reads do not start with the dead replica again. Both the direct and
+// the streaming read paths take this branch.
+func TestRetriedFetchChunkDataRefreshesLocationsAfterPartialFailure(t *testing.T) {
+	forgetUnreachable(t)
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+	dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+	deadURL := dead.URL
+	dead.Close()
+
+	for _, isFullChunk := range []bool{true, false} {
+		forgetUnreachable(t)
+		refreshed := 0
+		refresh := func() []string {
+			refreshed++
+			return []string{live.URL + "/3,abc"}
+		}
+		buffer := make([]byte, len(payload))
+		n, err := RetriedFetchChunkData(context.Background(), buffer, []string{deadURL + "/3,abc", live.URL + "/3,abc"}, nil, false, isFullChunk, 0, "3,abc", refresh)
+		if err != nil || string(buffer[:n]) != string(payload) {
+			t.Fatalf("fullChunk=%v: got %q, %v; want %q", isFullChunk, buffer[:n], err, payload)
+		}
+		if refreshed != 1 {
+			t.Fatalf("fullChunk=%v: refresh called %d times after a partial failure, want exactly 1", isFullChunk, refreshed)
+		}
+
+		// a read answered by the first location leaves the list alone
+		refreshed = 0
+		n, err = RetriedFetchChunkData(context.Background(), buffer, []string{live.URL + "/3,abc", deadURL + "/3,abc"}, nil, false, isFullChunk, 0, "3,abc", refresh)
+		if err != nil || string(buffer[:n]) != string(payload) {
+			t.Fatalf("fullChunk=%v: got %q, %v; want %q", isFullChunk, buffer[:n], err, payload)
+		}
+		if refreshed != 0 {
+			t.Fatalf("fullChunk=%v: refresh called %d times without a failure, want none", isFullChunk, refreshed)
+		}
+	}
+}
+
+func TestSameUrlsIgnoresOrder(t *testing.T) {
+	a := []string{"http://a:8080/3,x", "http://b:8080/3,x"}
+	if !SameUrls(a, []string{a[1], a[0]}) {
+		t.Fatal("a reshuffle of the same locations should not count as fresh")
+	}
+	if SameUrls(a, []string{a[0], "http://c:8080/3,x"}) {
+		t.Fatal("a different location should count as fresh")
+	}
+	if SameUrls(a, a[:1]) {
+		t.Fatal("a shorter list should count as fresh")
+	}
+}

weed/util/http/query_middleware.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+)
+
+// EscapeSemicolonsInQuery re-encodes raw ';' in the query string. RFC 3986
+// allows ';' in a query and AWS treats it as data, but Go's url.ParseQuery
+// drops every key=value pair containing one. Presigned URLs carry
+// X-Amz-SignedHeaders=content-type%3Bhost; clients that decode the %3B would
+// otherwise lose the parameter entirely and fail signature verification.
+func EscapeSemicolonsInQuery(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.RawQuery, ";") {
+			u := *r.URL
+			u.RawQuery = strings.ReplaceAll(u.RawQuery, ";", "%3B")
+			r2 := new(http.Request)
+			*r2 = *r
+			r2.URL = &u
+			r = r2
+		}
+		next.ServeHTTP(w, r)
+	})
+}

weed/util/http/reachability.go

@@ -0,0 +1,60 @@
+package http
+
+import (
+	"net/url"
+	"sync"
+	"time"
+)
+
+// unreachableRetryInterval is how long a server that failed to answer is tried
+// last before a read probes it again.
+const unreachableRetryInterval = 30 * time.Second
+
+// unreachable maps a host to when it last failed to answer a request. A dead
+// replica sits in the location list of every volume it held, so this is kept
+// per host rather than per volume.
+var unreachable sync.Map
+
+func recordUnreachable(host string) {
+	unreachable.Store(host, time.Now())
+}
+
+func recordReachable(host string) {
+	unreachable.Delete(host)
+}
+
+// ReachableFirst orders urls so that hosts which recently failed to answer
+// come last, keeping the order within each group. Once the retry interval has
+// passed, the first read to ask claims the probe and tries that host first;
+// the others keep it last until the probe settles. A read probes at most one
+// host, so a claim is always followed by an attempt and several expired hosts
+// are probed by successive reads.
+func ReachableFirst(urls []string) []string {
+	var probe, reachable, unanswered []string
+	for _, u := range urls {
+		host := hostOf(u)
+		failedAt, failed := unreachable.Load(host)
+		switch {
+		case !failed:
+			reachable = append(reachable, u)
+		case time.Since(failedAt.(time.Time)) < unreachableRetryInterval:
+			unanswered = append(unanswered, u)
+		case probe == nil && unreachable.CompareAndSwap(host, failedAt, time.Now()):
+			probe = append(probe, u)
+		default:
+			unanswered = append(unanswered, u)
+		}
+	}
+	if probe == nil && unanswered == nil {
+		return urls
+	}
+	return append(append(probe, reachable...), unanswered...)
+}
+
+func hostOf(rawUrl string) string {
+	parsed, err := url.Parse(rawUrl)
+	if err != nil {
+		return ""
+	}
+	return parsed.Host
+}

weed/util/http/reachability_test.go

@@ -0,0 +1,112 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func forgetUnreachable(t *testing.T) {
+	t.Helper()
+	forget := func() {
+		unreachable.Range(func(host, _ any) bool {
+			unreachable.Delete(host)
+			return true
+		})
+	}
+	forget()
+	t.Cleanup(forget)
+}
+
+func assertOrder(t *testing.T, got []string, want ...string) {
+	t.Helper()
+	if len(got) != len(want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("got %v, want %v", got, want)
+		}
+	}
+}
+
+func TestReachableFirstTriesUnansweringHostLast(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"}
+
+	assertOrder(t, ReachableFirst(urls), urls...)
+	recordUnreachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[2], urls[0])
+	recordReachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls...)
+}
+
+func TestReachableFirstProbesOnceAfterRetryInterval(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://b:8080/3,x", "http://a:8080/3,x"}
+	unreachable.Store("a:8080", time.Now().Add(-unreachableRetryInterval))
+
+	// the first read to come by probes a, the next keeps it last until that settles
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[0])
+	assertOrder(t, ReachableFirst(urls), urls...)
+	recordReachable("a:8080")
+	assertOrder(t, ReachableFirst(urls), urls...)
+}
+
+func TestReachableFirstProbesOneExpiredHostPerRead(t *testing.T) {
+	forgetUnreachable(t)
+	urls := []string{"http://a:8080/3,x", "http://b:8080/3,x", "http://c:8080/3,x"}
+	expired := time.Now().Add(-unreachableRetryInterval)
+	unreachable.Store("a:8080", expired)
+	unreachable.Store("c:8080", expired)
+
+	assertOrder(t, ReachableFirst(urls), urls[0], urls[1], urls[2])
+	assertOrder(t, ReachableFirst(urls), urls[2], urls[1], urls[0])
+	assertOrder(t, ReachableFirst(urls), urls[1], urls[0], urls[2])
+}
+
+// hangupServer accepts the connection and drops it without answering, the way
+// a replica behind a broken network does, and counts how often that happened.
+func hangupServer(t *testing.T) (*httptest.Server, *int32) {
+	t.Helper()
+	var hangups int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&hangups, 1)
+		conn, _, err := w.(http.Hijacker).Hijack()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		conn.Close()
+	}))
+	t.Cleanup(srv.Close)
+	return srv, &hangups
+}
+
+func TestRetriedFetchChunkDataTriesUnansweringServerLast(t *testing.T) {
+	forgetUnreachable(t)
+	payload := []byte("chunk contents")
+	live := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write(payload)
+	}))
+	defer live.Close()
+	hangup, hangups := hangupServer(t)
+
+	urls := []string{hangup.URL + "/3,abc", live.URL + "/3,abc"}
+	for i := 0; i < 3; i++ {
+		buffer := make([]byte, len(payload))
+		n, err := RetriedFetchChunkData(context.Background(), buffer, urls, nil, false, true, 0, "3,abc", nil)
+		if err != nil {
+			t.Fatalf("read %d: %v", i, err)
+		}
+		if string(buffer[:n]) != string(payload) {
+			t.Fatalf("read %d got %q, want %q", i, buffer[:n], payload)
+		}
+	}
+	if got := atomic.LoadInt32(hangups); got != 1 {
+		t.Fatalf("the server that hung up was tried %d times, want once", got)
+	}
+}