snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/stats/disk.go

@@ -0,0 +1,514 @@
+package stats
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+type DiskIOProbeConfig struct {
+	// enable/disable disk io probing
+	Enabled bool
+
+	// timeout for a single io probe
+	Timeout time.Duration
+
+	// probe interval
+	Interval time.Duration
+
+	// latency above this threshold is considered slow
+	SlowLatency time.Duration
+
+	// per disk type overrides for SlowLatency, keyed by the -disk tag
+	SlowLatencyByDiskType map[string]time.Duration
+
+	// rolling observation window
+	Window time.Duration
+
+	// minimum number of samples required before evaluating health
+	MinSamples int
+
+	// percentage of slow operations required to mark disk degraded
+	SlowPercent float64
+
+	// percentage of failed operations required to mark disk degraded
+	ErrorPercent float64
+
+	// recovery coef defines the hysteresis threshold for recovering from degraded state.
+	RecoveryCoef float64
+
+	// statfs failures before alerting
+	MaxStatFailures int
+}
+
+func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
+	return DiskIOProbeConfig{
+		Enabled:     false,
+		Timeout:     2 * time.Second,
+		Interval:    30 * time.Second,
+		SlowLatency: 100 * time.Millisecond,
+
+		Window:     time.Minute,
+		MinSamples: 10,
+
+		SlowPercent:  20,
+		ErrorPercent: 10,
+		RecoveryCoef: 0.5,
+
+		MaxStatFailures: 3,
+	}
+}
+
+// SlowLatencyFor returns the slow threshold configured for a disk type,
+// falling back to SlowLatency when the type has no entry of its own.
+func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
+	if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
+		return slowLatency
+	}
+	return config.SlowLatency
+}
+
+type ioSample struct {
+	ts      time.Time
+	latency time.Duration
+	failed  bool
+	slow    bool
+}
+
+type diskState struct {
+	mu sync.Mutex
+
+	// statfs probe
+	isChecking       bool
+	statFailureCount int
+	statSuccessCount int
+	lastStatErr      error
+	lastGoodStatus   volume_server_pb.DiskStatus
+	hasLastGood      bool
+
+	samples []ioSample
+
+	// IO probe
+	isIOChecking   bool
+	ioCheckID      uint64
+	lastIOCheck    time.Time
+	ioFailureCount int
+	lastIOErr      error
+	ioDegraded     bool
+}
+
+var diskRegistry sync.Map
+
+const (
+	// statfs timeout
+	diskTimeout = 500 * time.Millisecond
+)
+
+func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
+	disk = &volume_server_pb.DiskStatus{Dir: path}
+	fillInDiskStatus(disk)
+	if disk.PercentUsed > 95 {
+		glog.V(0).Infof("disk status: %v", disk)
+	}
+	return
+}
+
+func NewDiskStatusOnStart(path string, config DiskIOProbeConfig) (disk *volume_server_pb.DiskStatus) {
+	disk = &volume_server_pb.DiskStatus{Dir: path}
+	diskProbe(disk, config)
+	if disk.PercentUsed > 95 {
+		glog.V(0).Infof("disk status: %v", disk)
+	}
+	return
+}
+
+func diskProbe(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
+	actual, _ := diskRegistry.LoadOrStore(disk.Dir, &diskState{})
+	state := actual.(*diskState)
+
+	state.mu.Lock()
+	if state.isChecking {
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = errors.New("statfs still in progress")
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+	state.isChecking = true
+	state.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), diskTimeout)
+	defer cancel()
+
+	probe := &volume_server_pb.DiskStatus{Dir: disk.Dir}
+	ch := make(chan error, 1)
+	go func() {
+		err := fillInDiskStatus(probe)
+		// Clear the flag only once statfs has actually returned, so a stuck disk
+		// keeps a single outstanding probe instead of spawning a new one each tick.
+		state.mu.Lock()
+		state.isChecking = false
+		state.mu.Unlock()
+		ch <- err
+	}()
+
+	var probeErr error
+	select {
+	case probeErr = <-ch:
+	case <-ctx.Done():
+		// Leave isChecking set; the probe goroutine clears it when statfs returns.
+		state.mu.Lock()
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = errors.New("statfs timeout")
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+
+	state.mu.Lock()
+
+	if probeErr != nil {
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = probeErr
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+
+	state.observeStatSuccessLocked(config)
+	disk.All = probe.All
+	disk.Free = probe.Free
+	disk.Used = probe.Used
+	disk.PercentFree = probe.PercentFree
+	disk.PercentUsed = probe.PercentUsed
+	state.rememberGoodStatusLocked(disk)
+
+	if !config.Enabled {
+		state.ioFailureCount = 0
+		state.lastIOErr = nil
+		state.ioDegraded = false
+		state.updateErrorLocked(disk, config)
+		state.mu.Unlock()
+		return
+	}
+
+	shouldCheckIO := false
+	ioCheckID := uint64(0)
+
+	if time.Since(state.lastIOCheck) >= config.Interval {
+		if state.isIOChecking {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+			state.lastIOCheck = time.Now()
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = errors.New("disk io still in progress")
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+		} else {
+			state.isIOChecking = true
+			state.ioCheckID++
+			ioCheckID = state.ioCheckID
+			state.lastIOCheck = time.Now()
+			shouldCheckIO = true
+		}
+	}
+
+	state.updateErrorLocked(disk, config)
+	state.mu.Unlock()
+
+	if shouldCheckIO {
+		checkDiskIOLatency(state, disk, ioCheckID, config)
+	}
+}
+
+type ioProbeResult struct {
+	err     error
+	latency time.Duration
+}
+
+func checkDiskIOLatency(state *diskState, disk *volume_server_pb.DiskStatus, checkID uint64, config DiskIOProbeConfig) {
+	ctx, cancel := context.WithTimeout(context.Background(), config.Timeout)
+	defer cancel()
+
+	ch := make(chan ioProbeResult, 1)
+
+	go func() {
+		start := time.Now()
+		err := runIOProbe(disk.Dir)
+		ch <- ioProbeResult{
+			err:     err,
+			latency: time.Since(start),
+		}
+
+		state.mu.Lock()
+		if state.ioCheckID == checkID {
+			state.isIOChecking = false
+			state.lastIOCheck = time.Now()
+		}
+		state.mu.Unlock()
+	}()
+
+	select {
+	case result := <-ch:
+		state.mu.Lock()
+		defer state.mu.Unlock()
+
+		if state.ioCheckID != checkID {
+			return
+		}
+
+		state.isIOChecking = false
+		state.lastIOCheck = time.Now()
+
+		if result.err != nil {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = result.err
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+
+			state.updateErrorLocked(disk, config)
+			return
+		}
+
+		degraded := state.observeLatencyLocked(
+			result.latency,
+			false,
+			config,
+		)
+
+		state.ioDegraded = degraded
+
+		if degraded {
+			state.ioFailureCount++
+			state.lastIOErr = fmt.Errorf(
+				"disk io degradation detected latency=%v",
+				result.latency,
+			)
+		} else {
+			state.ioFailureCount = 0
+			state.lastIOErr = nil
+		}
+
+		state.updateErrorLocked(disk, config)
+
+		glog.V(1).Infof(
+			"disk io latency dir=%s latency=%v ioFailureCount=%d degraded=%v samples=%d error=%v",
+			disk.Dir,
+			result.latency,
+			state.ioFailureCount,
+			degraded,
+			len(state.samples),
+			state.lastIOErr,
+		)
+
+	case <-ctx.Done():
+		state.mu.Lock()
+
+		if state.ioCheckID == checkID {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = errors.New("disk io timeout")
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+
+			state.lastIOCheck = time.Now()
+			state.updateErrorLocked(disk, config)
+		}
+
+		state.mu.Unlock()
+	}
+}
+
+func runIOProbe(dir string) error {
+	path := filepath.Join(dir, ".disk-health-check")
+
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := f.WriteAt(make([]byte, 4096), 0); err != nil {
+		return err
+	}
+
+	return f.Sync()
+}
+
+func (s *diskState) observeLatencyLocked(latency time.Duration, failed bool, config DiskIOProbeConfig) bool {
+	now := time.Now()
+
+	sample := ioSample{
+		ts:      now,
+		latency: latency,
+		failed:  failed,
+		slow:    latency >= config.SlowLatency,
+	}
+
+	s.samples = append(s.samples, sample)
+
+	cutoff := now.Add(-config.Window)
+
+	idx := 0
+	for _, sample := range s.samples {
+		if sample.ts.After(cutoff) {
+			s.samples[idx] = sample
+			idx++
+		}
+	}
+
+	s.samples = s.samples[:idx]
+
+	total := len(s.samples)
+
+	if total < config.MinSamples {
+		return s.ioDegraded
+	}
+
+	var slowCount int
+	var failedCount int
+
+	for _, sample := range s.samples {
+		if sample.slow {
+			slowCount++
+		}
+
+		if sample.failed {
+			failedCount++
+		}
+	}
+
+	slowPercent := float64(slowCount) / float64(total) * 100
+	errorPercent := float64(failedCount) / float64(total) * 100
+
+	if !s.ioDegraded {
+		return slowPercent >= config.SlowPercent ||
+			errorPercent >= config.ErrorPercent
+	}
+
+	recoverSlowPercent := config.SlowPercent * config.RecoveryCoef
+	recoverErrorPercent := config.ErrorPercent * config.RecoveryCoef
+
+	return slowPercent > recoverSlowPercent ||
+		errorPercent > recoverErrorPercent
+}
+
+func (s *diskState) observeStatSuccessLocked(config DiskIOProbeConfig) {
+	if s.statFailureCount < config.MaxStatFailures {
+		s.statFailureCount = 0
+		s.statSuccessCount = 0
+		s.lastStatErr = nil
+		return
+	}
+
+	s.statSuccessCount++
+	if s.statSuccessCount >= statRecoverySuccesses(config) {
+		s.statFailureCount = 0
+		s.statSuccessCount = 0
+		s.lastStatErr = nil
+	}
+}
+
+func statRecoverySuccesses(config DiskIOProbeConfig) int {
+	if config.MaxStatFailures <= 1 {
+		return 1
+	}
+	if config.RecoveryCoef <= 0 {
+		return config.MaxStatFailures * 2
+	}
+
+	successes := int(float64(config.MaxStatFailures) * (1/config.RecoveryCoef - 1))
+	if successes < 1 {
+		return 1
+	}
+	return successes
+}
+
+func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
+	var problems []string
+
+	if s.statFailureCount >= config.MaxStatFailures &&
+		s.lastStatErr != nil {
+		problems = append(
+			problems,
+			fmt.Sprintf("statfs: %v", s.lastStatErr),
+		)
+	}
+
+	if config.Enabled && s.ioDegraded && s.lastIOErr != nil {
+		problems = append(
+			problems,
+			fmt.Sprintf("io: %v", s.lastIOErr),
+		)
+	}
+
+	if len(problems) == 0 {
+		disk.Error = ""
+		return
+	}
+
+	disk.Error =
+		"disk health check failed: " +
+			strings.Join(problems, "; ")
+}
+
+func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
+	s.lastGoodStatus = *disk
+	s.hasLastGood = true
+}
+
+func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
+	if !s.hasLastGood || disk.Error != "" {
+		return
+	}
+
+	dir := disk.Dir
+	errorMessage := disk.Error
+	*disk = s.lastGoodStatus
+	disk.Dir = dir
+	disk.Error = errorMessage
+}