upstream-seaweedfs/weed/stats/disk.go

package stats

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
)

type DiskIOProbeConfig struct {
	// enable/disable disk io probing
	Enabled bool

	// timeout for a single io probe
	Timeout time.Duration

	// probe interval
	Interval time.Duration

	// latency above this threshold is considered slow
	SlowLatency time.Duration

	// per disk type overrides for SlowLatency, keyed by the -disk tag
	SlowLatencyByDiskType map[string]time.Duration

	// rolling observation window
	Window time.Duration

	// minimum number of samples required before evaluating health
	MinSamples int

	// percentage of slow operations required to mark disk degraded
	SlowPercent float64

	// percentage of failed operations required to mark disk degraded
	ErrorPercent float64

	// recovery coef defines the hysteresis threshold for recovering from degraded state.
	RecoveryCoef float64

	// statfs failures before alerting
	MaxStatFailures int
}

func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
	return DiskIOProbeConfig{
		Enabled:     false,
		Timeout:     2 * time.Second,
		Interval:    30 * time.Second,
		SlowLatency: 100 * time.Millisecond,

		Window:     time.Minute,
		MinSamples: 10,

		SlowPercent:  20,
		ErrorPercent: 10,
		RecoveryCoef: 0.5,

		MaxStatFailures: 3,
	}
}

// SlowLatencyFor returns the slow threshold configured for a disk type,
// falling back to SlowLatency when the type has no entry of its own.
func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
	if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
		return slowLatency
	}
	return config.SlowLatency
}

type ioSample struct {
	ts      time.Time
	latency time.Duration
	failed  bool
	slow    bool
}

type diskState struct {
	mu sync.Mutex

	// statfs probe
	isChecking       bool
	statFailureCount int
	statSuccessCount int
	lastStatErr      error
	lastGoodStatus   volume_server_pb.DiskStatus
	hasLastGood      bool

	samples []ioSample

	// IO probe
	isIOChecking   bool
	ioCheckID      uint64
	lastIOCheck    time.Time
	ioFailureCount int
	lastIOErr      error
	ioDegraded     bool
}

var diskRegistry sync.Map

const (
	// statfs timeout
	diskTimeout = 500 * time.Millisecond
)

func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
	disk = &volume_server_pb.DiskStatus{Dir: path}
	fillInDiskStatus(disk)
	if disk.PercentUsed > 95 {
		glog.V(0).Infof("disk status: %v", disk)
	}
	return
}

func NewDiskStatusOnStart(path string, config DiskIOProbeConfig) (disk *volume_server_pb.DiskStatus) {
	disk = &volume_server_pb.DiskStatus{Dir: path}
	diskProbe(disk, config)
	if disk.PercentUsed > 95 {
		glog.V(0).Infof("disk status: %v", disk)
	}
	return
}

func diskProbe(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
	actual, _ := diskRegistry.LoadOrStore(disk.Dir, &diskState{})
	state := actual.(*diskState)

	state.mu.Lock()
	if state.isChecking {
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = errors.New("statfs still in progress")
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}
	state.isChecking = true
	state.mu.Unlock()

	ctx, cancel := context.WithTimeout(context.Background(), diskTimeout)
	defer cancel()

	probe := &volume_server_pb.DiskStatus{Dir: disk.Dir}
	ch := make(chan error, 1)
	go func() {
		err := fillInDiskStatus(probe)
		// Clear the flag only once statfs has actually returned, so a stuck disk
		// keeps a single outstanding probe instead of spawning a new one each tick.
		state.mu.Lock()
		state.isChecking = false
		state.mu.Unlock()
		ch <- err
	}()

	var probeErr error
	select {
	case probeErr = <-ch:
	case <-ctx.Done():
		// Leave isChecking set; the probe goroutine clears it when statfs returns.
		state.mu.Lock()
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = errors.New("statfs timeout")
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}

	state.mu.Lock()

	if probeErr != nil {
		state.statFailureCount++
		state.statSuccessCount = 0
		state.lastStatErr = probeErr
		state.updateErrorLocked(disk, config)
		state.applyLastGoodStatusLocked(disk)
		state.mu.Unlock()
		return
	}

	state.observeStatSuccessLocked(config)
	disk.All = probe.All
	disk.Free = probe.Free
	disk.Used = probe.Used
	disk.PercentFree = probe.PercentFree
	disk.PercentUsed = probe.PercentUsed
	state.rememberGoodStatusLocked(disk)

	if !config.Enabled {
		state.ioFailureCount = 0
		state.lastIOErr = nil
		state.ioDegraded = false
		state.updateErrorLocked(disk, config)
		state.mu.Unlock()
		return
	}

	shouldCheckIO := false
	ioCheckID := uint64(0)

	if time.Since(state.lastIOCheck) >= config.Interval {
		if state.isIOChecking {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded
			state.lastIOCheck = time.Now()

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = errors.New("disk io still in progress")
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}
		} else {
			state.isIOChecking = true
			state.ioCheckID++
			ioCheckID = state.ioCheckID
			state.lastIOCheck = time.Now()
			shouldCheckIO = true
		}
	}

	state.updateErrorLocked(disk, config)
	state.mu.Unlock()

	if shouldCheckIO {
		checkDiskIOLatency(state, disk, ioCheckID, config)
	}
}

type ioProbeResult struct {
	err     error
	latency time.Duration
}

func checkDiskIOLatency(state *diskState, disk *volume_server_pb.DiskStatus, checkID uint64, config DiskIOProbeConfig) {
	ctx, cancel := context.WithTimeout(context.Background(), config.Timeout)
	defer cancel()

	ch := make(chan ioProbeResult, 1)

	go func() {
		start := time.Now()
		err := runIOProbe(disk.Dir)
		ch <- ioProbeResult{
			err:     err,
			latency: time.Since(start),
		}

		state.mu.Lock()
		if state.ioCheckID == checkID {
			state.isIOChecking = false
			state.lastIOCheck = time.Now()
		}
		state.mu.Unlock()
	}()

	select {
	case result := <-ch:
		state.mu.Lock()
		defer state.mu.Unlock()

		if state.ioCheckID != checkID {
			return
		}

		state.isIOChecking = false
		state.lastIOCheck = time.Now()

		if result.err != nil {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = result.err
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}

			state.updateErrorLocked(disk, config)
			return
		}

		degraded := state.observeLatencyLocked(
			result.latency,
			false,
			config,
		)

		state.ioDegraded = degraded

		if degraded {
			state.ioFailureCount++
			state.lastIOErr = fmt.Errorf(
				"disk io degradation detected latency=%v",
				result.latency,
			)
		} else {
			state.ioFailureCount = 0
			state.lastIOErr = nil
		}

		state.updateErrorLocked(disk, config)

		glog.V(1).Infof(
			"disk io latency dir=%s latency=%v ioFailureCount=%d degraded=%v samples=%d error=%v",
			disk.Dir,
			result.latency,
			state.ioFailureCount,
			degraded,
			len(state.samples),
			state.lastIOErr,
		)

	case <-ctx.Done():
		state.mu.Lock()

		if state.ioCheckID == checkID {
			degraded := state.observeLatencyLocked(
				config.Timeout,
				true,
				config,
			)

			state.ioDegraded = degraded

			if degraded {
				state.ioFailureCount++
				state.lastIOErr = errors.New("disk io timeout")
			} else {
				state.ioFailureCount = 0
				state.lastIOErr = nil
			}

			state.lastIOCheck = time.Now()
			state.updateErrorLocked(disk, config)
		}

		state.mu.Unlock()
	}
}

func runIOProbe(dir string) error {
	path := filepath.Join(dir, ".disk-health-check")

	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := f.WriteAt(make([]byte, 4096), 0); err != nil {
		return err
	}

	return f.Sync()
}

func (s *diskState) observeLatencyLocked(latency time.Duration, failed bool, config DiskIOProbeConfig) bool {
	now := time.Now()

	sample := ioSample{
		ts:      now,
		latency: latency,
		failed:  failed,
		slow:    latency >= config.SlowLatency,
	}

	s.samples = append(s.samples, sample)

	cutoff := now.Add(-config.Window)

	idx := 0
	for _, sample := range s.samples {
		if sample.ts.After(cutoff) {
			s.samples[idx] = sample
			idx++
		}
	}

	s.samples = s.samples[:idx]

	total := len(s.samples)

	if total < config.MinSamples {
		return s.ioDegraded
	}

	var slowCount int
	var failedCount int

	for _, sample := range s.samples {
		if sample.slow {
			slowCount++
		}

		if sample.failed {
			failedCount++
		}
	}

	slowPercent := float64(slowCount) / float64(total) * 100
	errorPercent := float64(failedCount) / float64(total) * 100

	if !s.ioDegraded {
		return slowPercent >= config.SlowPercent ||
			errorPercent >= config.ErrorPercent
	}

	recoverSlowPercent := config.SlowPercent * config.RecoveryCoef
	recoverErrorPercent := config.ErrorPercent * config.RecoveryCoef

	return slowPercent > recoverSlowPercent ||
		errorPercent > recoverErrorPercent
}

func (s *diskState) observeStatSuccessLocked(config DiskIOProbeConfig) {
	if s.statFailureCount < config.MaxStatFailures {
		s.statFailureCount = 0
		s.statSuccessCount = 0
		s.lastStatErr = nil
		return
	}

	s.statSuccessCount++
	if s.statSuccessCount >= statRecoverySuccesses(config) {
		s.statFailureCount = 0
		s.statSuccessCount = 0
		s.lastStatErr = nil
	}
}

func statRecoverySuccesses(config DiskIOProbeConfig) int {
	if config.MaxStatFailures <= 1 {
		return 1
	}
	if config.RecoveryCoef <= 0 {
		return config.MaxStatFailures * 2
	}

	successes := int(float64(config.MaxStatFailures) * (1/config.RecoveryCoef - 1))
	if successes < 1 {
		return 1
	}
	return successes
}

func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
	var problems []string

	if s.statFailureCount >= config.MaxStatFailures &&
		s.lastStatErr != nil {
		problems = append(
			problems,
			fmt.Sprintf("statfs: %v", s.lastStatErr),
		)
	}

	if config.Enabled && s.ioDegraded && s.lastIOErr != nil {
		problems = append(
			problems,
			fmt.Sprintf("io: %v", s.lastIOErr),
		)
	}

	if len(problems) == 0 {
		disk.Error = ""
		return
	}

	disk.Error =
		"disk health check failed: " +
			strings.Join(problems, "; ")
}

func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
	s.lastGoodStatus = *disk
	s.hasLastGood = true
}

func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
	if !s.hasLastGood || disk.Error != "" {
		return
	}

	dir := disk.Dir
	errorMessage := disk.Error
	*disk = s.lastGoodStatus
	disk.Dir = dir
	disk.Error = errorMessage
}