snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/util/retry.go

@@ -0,0 +1,271 @@
+package util
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+var RetryWaitTime = 6 * time.Second
+
+// transientErrorMessages are substrings of failures that a later attempt is
+// likely to get past: connection resets, timeouts, and the throttling or
+// overload replies S3 and gRPC hand back. Cloud SDKs bury the underlying net
+// error in an opaque wrapper with no Unwrap, so the message is often all
+// that is left to match on. Compared case-insensitively, so entries are lower
+// case: the same condition reaches different layers capitalized differently
+// (a volume server relays its idle timeout as "I/O timeout" inside JSON).
+var transientErrorMessages = []string{
+	"transport",
+	"connection reset",
+	"connection refused",
+	"broken pipe",
+	"unexpected eof",
+	"i/o timeout",
+	"tls handshake timeout",
+	"no such host",
+	"no route to host",
+	"network is unreachable",
+	"client.timeout",
+	"requesterror",
+	"requesttimeout",
+	"slowdown",
+	"throttling",
+	"internalerror",
+	"resourceexhausted",
+	"unavailable",
+}
+
+// IsTransientErrorMessage reports whether an error message describes a network
+// or service condition worth retrying. Callers holding an error should use
+// IsTransientError; this is for paths that only carry the text, such as the
+// per-file status strings in a batch delete response.
+func IsTransientErrorMessage(msg string) bool {
+	lower := strings.ToLower(msg)
+	for _, transient := range transientErrorMessages {
+		if strings.Contains(lower, transient) {
+			return true
+		}
+	}
+	return false
+}
+
+// ServerStatus returns the status a gRPC server sent, when the error chain
+// still carries one. Unlike status.FromError it keeps the server's own message
+// instead of the whole wrapped string: callers routinely format a path the
+// client chose into their wrapper, so a classifier that matches substrings has
+// to read what the server said and not what the caller added around it.
+func ServerStatus(err error) (*status.Status, bool) {
+	var carrier interface{ GRPCStatus() *status.Status }
+	if !errors.As(err, &carrier) {
+		return nil, false
+	}
+	st := carrier.GRPCStatus()
+	return st, st != nil
+}
+
+// IsTransientError reports whether err is a network or service condition worth
+// retrying. A cancelled or expired context never is: the caller is already gone.
+func IsTransientError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return false
+	}
+	if errors.Is(err, io.ErrUnexpectedEOF) ||
+		errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) ||
+		errors.Is(err, syscall.ECONNREFUSED) || errors.Is(err, syscall.EPIPE) ||
+		errors.Is(err, syscall.ETIMEDOUT) {
+		return true
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return true
+	}
+	if st, ok := ServerStatus(err); ok {
+		return st.Code() == codes.Unavailable || st.Code() == codes.ResourceExhausted ||
+			IsTransientErrorMessage(st.Message())
+	}
+	return IsTransientErrorMessage(err.Error())
+}
+
+func Retry(name string, job func() error) (err error) {
+	waitTime := time.Second
+	hasErr := false
+	for waitTime < RetryWaitTime {
+		err = job()
+		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
+			waitTime = time.Second
+			break
+		}
+		if IsTransientError(err) {
+			hasErr = true
+			glog.V(0).Infof("retry %s: err: %v", name, err)
+		} else {
+			break
+		}
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}
+
+func MultiRetry(name string, errList []string, job func() error) (err error) {
+	waitTime := time.Second
+	hasErr := false
+	for waitTime < RetryWaitTime {
+		err = job()
+		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
+			waitTime = time.Second
+			break
+		}
+		if containErr(err.Error(), errList) {
+			hasErr = true
+			glog.V(0).Infof("retry %s: err: %v", name, err)
+		} else {
+			break
+		}
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}
+
+// RetryOnError retries job with the same bounded backoff as MultiRetry, but
+// decides retriability with a predicate instead of an error-substring list.
+func RetryOnError(name string, shouldRetry func(error) bool, job func() error) (err error) {
+	waitTime := time.Second
+	hasErr := false
+	for waitTime < RetryWaitTime {
+		err = job()
+		if err == nil {
+			if hasErr {
+				glog.V(0).Infof("retry %s successfully", name)
+			}
+			break
+		}
+		if shouldRetry(err) {
+			hasErr = true
+			glog.V(0).Infof("retry %s: err: %v", name, err)
+		} else {
+			break
+		}
+		time.Sleep(waitTime)
+		waitTime += waitTime / 2
+	}
+	return err
+}
+
+// RetryUntil retries until the job returns no error or onErrFn returns false
+func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) error {
+	waitTime := time.Second
+	for {
+		err := job()
+		if err == nil {
+			waitTime = time.Second
+			return nil
+		}
+		if onErrFn(err) {
+			if strings.Contains(err.Error(), "transport") || strings.Contains(err.Error(), "ResourceExhausted") || strings.Contains(err.Error(), "Unavailable") {
+				glog.V(0).Infof("retry %s: err: %v", name, err)
+			}
+			time.Sleep(waitTime)
+			if waitTime < RetryWaitTime {
+				waitTime += waitTime / 2
+			}
+			continue
+		} else {
+			return err
+		}
+	}
+}
+
+// RetryWithBackoff retries an operation on codes.Unavailable errors with exponential
+// backoff, respecting context cancellation and a maximum retry duration.
+// Returns nil on success, ctx.Err() on context cancellation, or the last error
+// when maxDuration is exceeded or a non-retriable error occurs.
+func RetryWithBackoff(ctx context.Context, name string, maxDuration time.Duration, shouldRetry func(error) bool, operation func() error) error {
+	waitTime := time.Second
+	maxWaitTime := RetryWaitTime
+	deadline := time.Now().Add(maxDuration)
+	var lastErr error
+	for {
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+		if time.Until(deadline) <= 0 {
+			if lastErr != nil {
+				glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, lastErr)
+				return lastErr
+			}
+		}
+		err := operation()
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if !shouldRetry(err) {
+			return err
+		}
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			glog.V(0).Infof("retry %s: giving up after %v: %v", name, maxDuration, err)
+			return err
+		}
+		sleepTime := waitTime
+		if sleepTime > maxWaitTime {
+			sleepTime = maxWaitTime
+		}
+		if sleepTime > remaining {
+			sleepTime = remaining
+		}
+		glog.V(1).Infof("retry %s: retrying in %v: %v", name, sleepTime, err)
+		timer := time.NewTimer(sleepTime)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			return ctx.Err()
+		case <-timer.C:
+		}
+		waitTime += waitTime / 2
+		if waitTime > maxWaitTime {
+			waitTime = maxWaitTime
+		}
+	}
+}
+
+// Nvl return the first non-empty string
+func Nvl(values ...string) string {
+	for _, s := range values {
+		if s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
+func containErr(err string, errList []string) bool {
+	for _, e := range errList {
+		if strings.Contains(err, e) {
+			return true
+		}
+	}
+	return false
+}