snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/stats/disk.go

@@ -0,0 +1,514 @@
+package stats
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+type DiskIOProbeConfig struct {
+	// enable/disable disk io probing
+	Enabled bool
+
+	// timeout for a single io probe
+	Timeout time.Duration
+
+	// probe interval
+	Interval time.Duration
+
+	// latency above this threshold is considered slow
+	SlowLatency time.Duration
+
+	// per disk type overrides for SlowLatency, keyed by the -disk tag
+	SlowLatencyByDiskType map[string]time.Duration
+
+	// rolling observation window
+	Window time.Duration
+
+	// minimum number of samples required before evaluating health
+	MinSamples int
+
+	// percentage of slow operations required to mark disk degraded
+	SlowPercent float64
+
+	// percentage of failed operations required to mark disk degraded
+	ErrorPercent float64
+
+	// recovery coef defines the hysteresis threshold for recovering from degraded state.
+	RecoveryCoef float64
+
+	// statfs failures before alerting
+	MaxStatFailures int
+}
+
+func DefaultDiskIOProbeConfig() DiskIOProbeConfig {
+	return DiskIOProbeConfig{
+		Enabled:     false,
+		Timeout:     2 * time.Second,
+		Interval:    30 * time.Second,
+		SlowLatency: 100 * time.Millisecond,
+
+		Window:     time.Minute,
+		MinSamples: 10,
+
+		SlowPercent:  20,
+		ErrorPercent: 10,
+		RecoveryCoef: 0.5,
+
+		MaxStatFailures: 3,
+	}
+}
+
+// SlowLatencyFor returns the slow threshold configured for a disk type,
+// falling back to SlowLatency when the type has no entry of its own.
+func (config DiskIOProbeConfig) SlowLatencyFor(diskType string) time.Duration {
+	if slowLatency, found := config.SlowLatencyByDiskType[diskType]; found {
+		return slowLatency
+	}
+	return config.SlowLatency
+}
+
+type ioSample struct {
+	ts      time.Time
+	latency time.Duration
+	failed  bool
+	slow    bool
+}
+
+type diskState struct {
+	mu sync.Mutex
+
+	// statfs probe
+	isChecking       bool
+	statFailureCount int
+	statSuccessCount int
+	lastStatErr      error
+	lastGoodStatus   volume_server_pb.DiskStatus
+	hasLastGood      bool
+
+	samples []ioSample
+
+	// IO probe
+	isIOChecking   bool
+	ioCheckID      uint64
+	lastIOCheck    time.Time
+	ioFailureCount int
+	lastIOErr      error
+	ioDegraded     bool
+}
+
+var diskRegistry sync.Map
+
+const (
+	// statfs timeout
+	diskTimeout = 500 * time.Millisecond
+)
+
+func NewDiskStatus(path string) (disk *volume_server_pb.DiskStatus) {
+	disk = &volume_server_pb.DiskStatus{Dir: path}
+	fillInDiskStatus(disk)
+	if disk.PercentUsed > 95 {
+		glog.V(0).Infof("disk status: %v", disk)
+	}
+	return
+}
+
+func NewDiskStatusOnStart(path string, config DiskIOProbeConfig) (disk *volume_server_pb.DiskStatus) {
+	disk = &volume_server_pb.DiskStatus{Dir: path}
+	diskProbe(disk, config)
+	if disk.PercentUsed > 95 {
+		glog.V(0).Infof("disk status: %v", disk)
+	}
+	return
+}
+
+func diskProbe(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
+	actual, _ := diskRegistry.LoadOrStore(disk.Dir, &diskState{})
+	state := actual.(*diskState)
+
+	state.mu.Lock()
+	if state.isChecking {
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = errors.New("statfs still in progress")
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+	state.isChecking = true
+	state.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), diskTimeout)
+	defer cancel()
+
+	probe := &volume_server_pb.DiskStatus{Dir: disk.Dir}
+	ch := make(chan error, 1)
+	go func() {
+		err := fillInDiskStatus(probe)
+		// Clear the flag only once statfs has actually returned, so a stuck disk
+		// keeps a single outstanding probe instead of spawning a new one each tick.
+		state.mu.Lock()
+		state.isChecking = false
+		state.mu.Unlock()
+		ch <- err
+	}()
+
+	var probeErr error
+	select {
+	case probeErr = <-ch:
+	case <-ctx.Done():
+		// Leave isChecking set; the probe goroutine clears it when statfs returns.
+		state.mu.Lock()
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = errors.New("statfs timeout")
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+
+	state.mu.Lock()
+
+	if probeErr != nil {
+		state.statFailureCount++
+		state.statSuccessCount = 0
+		state.lastStatErr = probeErr
+		state.updateErrorLocked(disk, config)
+		state.applyLastGoodStatusLocked(disk)
+		state.mu.Unlock()
+		return
+	}
+
+	state.observeStatSuccessLocked(config)
+	disk.All = probe.All
+	disk.Free = probe.Free
+	disk.Used = probe.Used
+	disk.PercentFree = probe.PercentFree
+	disk.PercentUsed = probe.PercentUsed
+	state.rememberGoodStatusLocked(disk)
+
+	if !config.Enabled {
+		state.ioFailureCount = 0
+		state.lastIOErr = nil
+		state.ioDegraded = false
+		state.updateErrorLocked(disk, config)
+		state.mu.Unlock()
+		return
+	}
+
+	shouldCheckIO := false
+	ioCheckID := uint64(0)
+
+	if time.Since(state.lastIOCheck) >= config.Interval {
+		if state.isIOChecking {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+			state.lastIOCheck = time.Now()
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = errors.New("disk io still in progress")
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+		} else {
+			state.isIOChecking = true
+			state.ioCheckID++
+			ioCheckID = state.ioCheckID
+			state.lastIOCheck = time.Now()
+			shouldCheckIO = true
+		}
+	}
+
+	state.updateErrorLocked(disk, config)
+	state.mu.Unlock()
+
+	if shouldCheckIO {
+		checkDiskIOLatency(state, disk, ioCheckID, config)
+	}
+}
+
+type ioProbeResult struct {
+	err     error
+	latency time.Duration
+}
+
+func checkDiskIOLatency(state *diskState, disk *volume_server_pb.DiskStatus, checkID uint64, config DiskIOProbeConfig) {
+	ctx, cancel := context.WithTimeout(context.Background(), config.Timeout)
+	defer cancel()
+
+	ch := make(chan ioProbeResult, 1)
+
+	go func() {
+		start := time.Now()
+		err := runIOProbe(disk.Dir)
+		ch <- ioProbeResult{
+			err:     err,
+			latency: time.Since(start),
+		}
+
+		state.mu.Lock()
+		if state.ioCheckID == checkID {
+			state.isIOChecking = false
+			state.lastIOCheck = time.Now()
+		}
+		state.mu.Unlock()
+	}()
+
+	select {
+	case result := <-ch:
+		state.mu.Lock()
+		defer state.mu.Unlock()
+
+		if state.ioCheckID != checkID {
+			return
+		}
+
+		state.isIOChecking = false
+		state.lastIOCheck = time.Now()
+
+		if result.err != nil {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = result.err
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+
+			state.updateErrorLocked(disk, config)
+			return
+		}
+
+		degraded := state.observeLatencyLocked(
+			result.latency,
+			false,
+			config,
+		)
+
+		state.ioDegraded = degraded
+
+		if degraded {
+			state.ioFailureCount++
+			state.lastIOErr = fmt.Errorf(
+				"disk io degradation detected latency=%v",
+				result.latency,
+			)
+		} else {
+			state.ioFailureCount = 0
+			state.lastIOErr = nil
+		}
+
+		state.updateErrorLocked(disk, config)
+
+		glog.V(1).Infof(
+			"disk io latency dir=%s latency=%v ioFailureCount=%d degraded=%v samples=%d error=%v",
+			disk.Dir,
+			result.latency,
+			state.ioFailureCount,
+			degraded,
+			len(state.samples),
+			state.lastIOErr,
+		)
+
+	case <-ctx.Done():
+		state.mu.Lock()
+
+		if state.ioCheckID == checkID {
+			degraded := state.observeLatencyLocked(
+				config.Timeout,
+				true,
+				config,
+			)
+
+			state.ioDegraded = degraded
+
+			if degraded {
+				state.ioFailureCount++
+				state.lastIOErr = errors.New("disk io timeout")
+			} else {
+				state.ioFailureCount = 0
+				state.lastIOErr = nil
+			}
+
+			state.lastIOCheck = time.Now()
+			state.updateErrorLocked(disk, config)
+		}
+
+		state.mu.Unlock()
+	}
+}
+
+func runIOProbe(dir string) error {
+	path := filepath.Join(dir, ".disk-health-check")
+
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := f.WriteAt(make([]byte, 4096), 0); err != nil {
+		return err
+	}
+
+	return f.Sync()
+}
+
+func (s *diskState) observeLatencyLocked(latency time.Duration, failed bool, config DiskIOProbeConfig) bool {
+	now := time.Now()
+
+	sample := ioSample{
+		ts:      now,
+		latency: latency,
+		failed:  failed,
+		slow:    latency >= config.SlowLatency,
+	}
+
+	s.samples = append(s.samples, sample)
+
+	cutoff := now.Add(-config.Window)
+
+	idx := 0
+	for _, sample := range s.samples {
+		if sample.ts.After(cutoff) {
+			s.samples[idx] = sample
+			idx++
+		}
+	}
+
+	s.samples = s.samples[:idx]
+
+	total := len(s.samples)
+
+	if total < config.MinSamples {
+		return s.ioDegraded
+	}
+
+	var slowCount int
+	var failedCount int
+
+	for _, sample := range s.samples {
+		if sample.slow {
+			slowCount++
+		}
+
+		if sample.failed {
+			failedCount++
+		}
+	}
+
+	slowPercent := float64(slowCount) / float64(total) * 100
+	errorPercent := float64(failedCount) / float64(total) * 100
+
+	if !s.ioDegraded {
+		return slowPercent >= config.SlowPercent ||
+			errorPercent >= config.ErrorPercent
+	}
+
+	recoverSlowPercent := config.SlowPercent * config.RecoveryCoef
+	recoverErrorPercent := config.ErrorPercent * config.RecoveryCoef
+
+	return slowPercent > recoverSlowPercent ||
+		errorPercent > recoverErrorPercent
+}
+
+func (s *diskState) observeStatSuccessLocked(config DiskIOProbeConfig) {
+	if s.statFailureCount < config.MaxStatFailures {
+		s.statFailureCount = 0
+		s.statSuccessCount = 0
+		s.lastStatErr = nil
+		return
+	}
+
+	s.statSuccessCount++
+	if s.statSuccessCount >= statRecoverySuccesses(config) {
+		s.statFailureCount = 0
+		s.statSuccessCount = 0
+		s.lastStatErr = nil
+	}
+}
+
+func statRecoverySuccesses(config DiskIOProbeConfig) int {
+	if config.MaxStatFailures <= 1 {
+		return 1
+	}
+	if config.RecoveryCoef <= 0 {
+		return config.MaxStatFailures * 2
+	}
+
+	successes := int(float64(config.MaxStatFailures) * (1/config.RecoveryCoef - 1))
+	if successes < 1 {
+		return 1
+	}
+	return successes
+}
+
+func (s *diskState) updateErrorLocked(disk *volume_server_pb.DiskStatus, config DiskIOProbeConfig) {
+	var problems []string
+
+	if s.statFailureCount >= config.MaxStatFailures &&
+		s.lastStatErr != nil {
+		problems = append(
+			problems,
+			fmt.Sprintf("statfs: %v", s.lastStatErr),
+		)
+	}
+
+	if config.Enabled && s.ioDegraded && s.lastIOErr != nil {
+		problems = append(
+			problems,
+			fmt.Sprintf("io: %v", s.lastIOErr),
+		)
+	}
+
+	if len(problems) == 0 {
+		disk.Error = ""
+		return
+	}
+
+	disk.Error =
+		"disk health check failed: " +
+			strings.Join(problems, "; ")
+}
+
+func (s *diskState) rememberGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
+	s.lastGoodStatus = *disk
+	s.hasLastGood = true
+}
+
+func (s *diskState) applyLastGoodStatusLocked(disk *volume_server_pb.DiskStatus) {
+	if !s.hasLastGood || disk.Error != "" {
+		return
+	}
+
+	dir := disk.Dir
+	errorMessage := disk.Error
+	*disk = s.lastGoodStatus
+	disk.Dir = dir
+	disk.Error = errorMessage
+}

weed/stats/disk_notsupported.go

@@ -0,0 +1,9 @@
+//go:build netbsd || plan9
+
+package stats
+
+import "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+
+func fillInDiskStatus(status *volume_server_pb.DiskStatus) error {
+	return nil
+}

weed/stats/disk_openbsd.go

@@ -0,0 +1,24 @@
+//go:build openbsd
+
+package stats
+
+import (
+	"syscall"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func fillInDiskStatus(disk *volume_server_pb.DiskStatus) error {
+	fs := syscall.Statfs_t{}
+	if err := syscall.Statfs(disk.Dir, &fs); err != nil {
+		return err
+	}
+
+	disk.All = fs.F_blocks * uint64(fs.F_bsize)
+	disk.Free = fs.F_bfree * uint64(fs.F_bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+
+	return nil
+}

weed/stats/disk_solaris.go

@@ -0,0 +1,24 @@
+//go:build solaris
+
+package stats
+
+import (
+	"golang.org/x/sys/unix"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func fillInDiskStatus(disk *volume_server_pb.DiskStatus) error {
+	var stat unix.Statvfs_t
+	if err := unix.Statvfs(disk.Dir, &stat); err != nil {
+		return err
+	}
+
+	disk.All = stat.Blocks * uint64(stat.Bsize)
+	disk.Free = stat.Bfree * uint64(stat.Bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+
+	return nil
+}

weed/stats/disk_supported.go

@@ -0,0 +1,25 @@
+//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
+
+package stats
+
+import (
+	"syscall"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func fillInDiskStatus(disk *volume_server_pb.DiskStatus) error {
+	fs := syscall.Statfs_t{}
+	if err := syscall.Statfs(disk.Dir, &fs); err != nil {
+		return err
+	}
+	disk.All = fs.Blocks * uint64(fs.Bsize)
+	// https://man7.org/linux/man-pages/man3/statvfs.3.html
+	// fs.Bfree: Number of free blocks
+	// fs.Bavail: Number of free blocks for unprivileged users
+	disk.Free = uint64(fs.Bavail) * uint64(fs.Bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32(float64(disk.Free) / float64(disk.All) * 100)
+	disk.PercentUsed = float32(float64(disk.Used) / float64(disk.All) * 100)
+	return nil
+}

weed/stats/disk_test.go

@@ -0,0 +1,96 @@
+//go:build !netbsd && !plan9
+
+package stats
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func TestSetDiskStatusSuccess(t *testing.T) {
+	disk := &volume_server_pb.DiskStatus{Dir: t.TempDir()}
+	diskProbe(disk, DefaultDiskIOProbeConfig())
+
+	if disk.Error != "" {
+		t.Fatalf("unexpected disk error: %s", disk.Error)
+	}
+	if disk.All == 0 {
+		t.Fatalf("expected non-zero capacity for a real directory")
+	}
+}
+
+func TestSetDiskStatusReportsRepeatedFailures(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "does-not-exist")
+	config := DefaultDiskIOProbeConfig()
+	var disk *volume_server_pb.DiskStatus
+	for i := 0; i < config.MaxStatFailures; i++ {
+		disk = &volume_server_pb.DiskStatus{Dir: missing}
+		diskProbe(disk, config)
+	}
+
+	if disk.Error == "" {
+		t.Fatalf("expected disk error after %d failed checks", config.MaxStatFailures)
+	}
+}
+
+func TestSetDiskStatusRequiresRepeatedSuccessesToRecover(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "temporarily-missing")
+	config := DefaultDiskIOProbeConfig()
+	var disk *volume_server_pb.DiskStatus
+	for i := 0; i < config.MaxStatFailures; i++ {
+		disk = &volume_server_pb.DiskStatus{Dir: missing}
+		diskProbe(disk, config)
+	}
+	if disk.Error == "" {
+		t.Fatalf("expected disk error after %d failed checks", config.MaxStatFailures)
+	}
+
+	if err := os.Mkdir(missing, 0755); err != nil {
+		t.Fatalf("mkdir %s: %v", missing, err)
+	}
+
+	recoveryChecks := statRecoverySuccesses(config)
+	for i := 1; i < recoveryChecks; i++ {
+		disk = &volume_server_pb.DiskStatus{Dir: missing}
+		diskProbe(disk, config)
+		if disk.Error == "" {
+			t.Fatalf("expected disk error to persist after %d successful checks", i)
+		}
+	}
+
+	disk = &volume_server_pb.DiskStatus{Dir: missing}
+	diskProbe(disk, config)
+	if disk.Error != "" {
+		t.Fatalf("expected disk error to recover after %d successful checks: %s", recoveryChecks, disk.Error)
+	}
+}
+
+func TestSlowLatencyForFallsBackToDefault(t *testing.T) {
+	config := DiskIOProbeConfig{
+		SlowLatency: 500 * time.Millisecond,
+		SlowLatencyByDiskType: map[string]time.Duration{
+			"hdd":  500 * time.Millisecond,
+			"nvme": 50 * time.Millisecond,
+		},
+	}
+
+	for diskType, want := range map[string]time.Duration{
+		"hdd":       500 * time.Millisecond,
+		"nvme":      50 * time.Millisecond,
+		"ssd":       500 * time.Millisecond,
+		"nvme-gen5": 500 * time.Millisecond,
+	} {
+		if got := config.SlowLatencyFor(diskType); got != want {
+			t.Errorf("disk type %q: got %v, want %v", diskType, got, want)
+		}
+	}
+
+	empty := DiskIOProbeConfig{SlowLatency: 100 * time.Millisecond}
+	if got := empty.SlowLatencyFor("nvme"); got != 100*time.Millisecond {
+		t.Errorf("unconfigured table: got %v, want %v", got, 100*time.Millisecond)
+	}
+}

weed/stats/disk_windows.go

@@ -0,0 +1,48 @@
+package stats
+
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"golang.org/x/sys/windows"
+)
+
+var (
+	kernel32           = windows.NewLazySystemDLL("Kernel32.dll")
+	getDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW")
+)
+
+func fillInDiskStatus(disk *volume_server_pb.DiskStatus) error {
+
+	ptr, err := syscall.UTF16PtrFromString(disk.Dir)
+
+	if err != nil {
+		return err
+	}
+	var _temp uint64
+	/* #nosec */
+	r, _, e := syscall.Syscall6(
+		getDiskFreeSpaceEx.Addr(),
+		4,
+		uintptr(unsafe.Pointer(ptr)),
+		uintptr(unsafe.Pointer(&disk.Free)),
+		uintptr(unsafe.Pointer(&disk.All)),
+		uintptr(unsafe.Pointer(&_temp)),
+		0,
+		0,
+	)
+
+	if r == 0 {
+		if e != 0 {
+			return e
+		}
+		return fmt.Errorf("GetDiskFreeSpaceExW failed for %s", disk.Dir)
+	}
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+
+	return nil
+}

weed/stats/duration_counter.go

@@ -0,0 +1,94 @@
+package stats
+
+import (
+	"time"
+)
+
+type TimedValue struct {
+	t   time.Time
+	val int64
+}
+
+func NewTimedValue(t time.Time, val int64) *TimedValue {
+	return &TimedValue{t: t, val: val}
+}
+
+type RoundRobinCounter struct {
+	LastIndex int
+	Values    []int64
+	Counts    []int64
+}
+
+func NewRoundRobinCounter(slots int) *RoundRobinCounter {
+	return &RoundRobinCounter{LastIndex: -1, Values: make([]int64, slots), Counts: make([]int64, slots)}
+}
+func (rrc *RoundRobinCounter) Add(index int, val int64) {
+	if index >= len(rrc.Values) {
+		return
+	}
+	for rrc.LastIndex != index {
+		rrc.LastIndex = (rrc.LastIndex + 1) % len(rrc.Values)
+		rrc.Values[rrc.LastIndex] = 0
+		rrc.Counts[rrc.LastIndex] = 0
+	}
+	rrc.Values[index] += val
+	rrc.Counts[index]++
+}
+func (rrc *RoundRobinCounter) Max() (max int64) {
+	for _, val := range rrc.Values {
+		if max < val {
+			max = val
+		}
+	}
+	return
+}
+func (rrc *RoundRobinCounter) Count() (cnt int64) {
+	for _, c := range rrc.Counts {
+		cnt += c
+	}
+	return
+}
+func (rrc *RoundRobinCounter) Sum() (sum int64) {
+	for _, val := range rrc.Values {
+		sum += val
+	}
+	return
+}
+
+func (rrc *RoundRobinCounter) ToList() (ret []int64) {
+	index := rrc.LastIndex
+	step := len(rrc.Values)
+	for step > 0 {
+		step--
+		index++
+		if index >= len(rrc.Values) {
+			index = 0
+		}
+		ret = append(ret, rrc.Values[index])
+	}
+	return
+}
+
+type DurationCounter struct {
+	MinuteCounter *RoundRobinCounter
+	HourCounter   *RoundRobinCounter
+	DayCounter    *RoundRobinCounter
+	WeekCounter   *RoundRobinCounter
+}
+
+func NewDurationCounter() *DurationCounter {
+	return &DurationCounter{
+		MinuteCounter: NewRoundRobinCounter(60),
+		HourCounter:   NewRoundRobinCounter(60),
+		DayCounter:    NewRoundRobinCounter(24),
+		WeekCounter:   NewRoundRobinCounter(7),
+	}
+}
+
+// Add is for cumulative counts
+func (sc *DurationCounter) Add(tv *TimedValue) {
+	sc.MinuteCounter.Add(tv.t.Second(), tv.val)
+	sc.HourCounter.Add(tv.t.Minute(), tv.val)
+	sc.DayCounter.Add(tv.t.Hour(), tv.val)
+	sc.WeekCounter.Add(int(tv.t.Weekday()), tv.val)
+}

weed/stats/duration_counter_test.go

@@ -0,0 +1,19 @@
+package stats
+
+import "testing"
+
+func TestRobinCounter(t *testing.T) {
+	rrc := NewRoundRobinCounter(60)
+	rrc.Add(0, 1)
+	rrc.Add(50, 2)
+	if rrc.Count() != 2 {
+		t.Fatal()
+	}
+	if rrc.Sum() != 3 {
+		t.Fatal()
+	}
+	/*
+		index out of range
+	*/
+	rrc.Add(61, 1)
+}

weed/stats/http_status_recorder.go

@@ -0,0 +1,21 @@
+package stats
+
+import "net/http"
+
+type StatusRecorder struct {
+	http.ResponseWriter
+	Status int
+}
+
+func NewStatusResponseWriter(w http.ResponseWriter) *StatusRecorder {
+	return &StatusRecorder{w, http.StatusOK}
+}
+
+func (r *StatusRecorder) WriteHeader(status int) {
+	r.Status = status
+	r.ResponseWriter.WriteHeader(status)
+}
+
+func (r *StatusRecorder) Flush() {
+	r.ResponseWriter.(http.Flusher).Flush()
+}

weed/stats/memory.go

@@ -0,0 +1,20 @@
+package stats
+
+import (
+	"runtime"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func MemStat() *volume_server_pb.MemStatus {
+	mem := &volume_server_pb.MemStatus{}
+	mem.Goroutines = int32(runtime.NumGoroutine())
+	memStat := new(runtime.MemStats)
+	runtime.ReadMemStats(memStat)
+	mem.Self = memStat.Alloc
+	mem.Heap = memStat.HeapAlloc
+	mem.Stack = memStat.StackInuse
+
+	fillInMemStatus(mem)
+	return mem
+}

weed/stats/memory_notsupported.go

@@ -0,0 +1,9 @@
+//go:build !linux
+
+package stats
+
+import "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+
+func fillInMemStatus(status *volume_server_pb.MemStatus) {
+	return
+}

weed/stats/memory_supported.go

@@ -0,0 +1,20 @@
+//go:build linux
+
+package stats
+
+import (
+	"syscall"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func fillInMemStatus(mem *volume_server_pb.MemStatus) {
+	//system memory usage
+	sysInfo := new(syscall.Sysinfo_t)
+	err := syscall.Sysinfo(sysInfo)
+	if err == nil {
+		mem.All = uint64(sysInfo.Totalram) //* uint64(syscall.Getpagesize())
+		mem.Free = uint64(sysInfo.Freeram) //* uint64(syscall.Getpagesize())
+		mem.Used = mem.All - mem.Free
+	}
+}

weed/stats/metrics_bucket_quota_test.go

@@ -0,0 +1,42 @@
+package stats
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestUpdateBucketQuotaMetrics(t *testing.T) {
+	bucket := "quota-test-bucket"
+
+	UpdateBucketQuotaMetrics(bucket, 1024, true)
+	if got := testutil.ToFloat64(S3BucketQuotaBytesGauge.WithLabelValues(bucket)); got != 1024 {
+		t.Errorf("quota gauge: got %.0f, want 1024", got)
+	}
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 {
+		t.Errorf("read-only gauge: got %.0f, want 1", got)
+	}
+
+	UpdateBucketQuotaMetrics(bucket, 1024, false)
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 0 {
+		t.Errorf("read-only gauge after clearing: got %.0f, want 0", got)
+	}
+
+	// disabled quota (negative) removes the quota series but keeps reporting read-only state
+	UpdateBucketQuotaMetrics(bucket, -1024, true)
+	if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 {
+		t.Errorf("quota series after disabling: got %d, want 0", n)
+	}
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 {
+		t.Errorf("read-only gauge with disabled quota: got %.0f, want 1", got)
+	}
+
+	UpdateBucketQuotaMetrics(bucket, 2048, true)
+	DeleteBucketMetrics(bucket)
+	if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 {
+		t.Errorf("quota series after bucket deletion: got %d, want 0", n)
+	}
+	if n := testutil.CollectAndCount(S3BucketReadOnlyGauge); n != 0 {
+		t.Errorf("read-only series after bucket deletion: got %d, want 0", n)
+	}
+}

weed/stats/metrics_buildinfo_test.go

@@ -0,0 +1,81 @@
+package stats_test
+
+import (
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	_ "github.com/seaweedfs/seaweedfs/weed/util/version" // Import to trigger version init
+)
+
+func TestBuildInfo(t *testing.T) {
+	// Verify that BuildInfo metric is registered and has the expected value
+	count := testutil.CollectAndCount(stats.BuildInfo)
+	if count != 1 {
+		t.Errorf("Expected 1 BuildInfo metric, got %d", count)
+	}
+
+	// Verify the metric can be gathered
+	metrics, err := stats.Gather.Gather()
+	if err != nil {
+		t.Fatalf("Failed to gather metrics: %v", err)
+	}
+
+	// Find the build_info metric
+	found := false
+	for _, mf := range metrics {
+		if mf.GetName() == "SeaweedFS_build_info" {
+			found = true
+			metric := mf.GetMetric()[0]
+
+			// Verify the metric value is 1
+			if metric.GetGauge().GetValue() != 1 {
+				t.Errorf("Expected BuildInfo value to be 1, got %f", metric.GetGauge().GetValue())
+			}
+
+			// Verify labels exist
+			labels := metric.GetLabel()
+			labelMap := make(map[string]string)
+			for _, label := range labels {
+				labelMap[label.GetName()] = label.GetValue()
+			}
+
+			// Check required labels
+			if _, ok := labelMap["version"]; !ok {
+				t.Error("Missing 'version' label")
+			}
+			if _, ok := labelMap["commit"]; !ok {
+				t.Error("Missing 'commit' label")
+			}
+			if _, ok := labelMap["sizelimit"]; !ok {
+				t.Error("Missing 'sizelimit' label")
+			}
+			if labelMap["goos"] != runtime.GOOS {
+				t.Errorf("Expected goos='%s', got '%s'", runtime.GOOS, labelMap["goos"])
+			}
+			if labelMap["goarch"] != runtime.GOARCH {
+				t.Errorf("Expected goarch='%s', got '%s'", runtime.GOARCH, labelMap["goarch"])
+			}
+
+			// Verify version format
+			if !strings.Contains(labelMap["version"], ".") {
+				t.Errorf("Version should contain a dot: %s", labelMap["version"])
+			}
+
+			// Verify sizelimit is either 30GB or 8000GB
+			if labelMap["sizelimit"] != "30GB" && labelMap["sizelimit"] != "8000GB" {
+				t.Errorf("Expected sizelimit to be '30GB' or '8000GB', got '%s'", labelMap["sizelimit"])
+			}
+
+			t.Logf("BuildInfo metric: version=%s, commit=%s, sizelimit=%s, goos=%s, goarch=%s",
+				labelMap["version"], labelMap["commit"], labelMap["sizelimit"],
+				labelMap["goos"], labelMap["goarch"])
+		}
+	}
+
+	if !found {
+		t.Error("BuildInfo metric not found in gathered metrics")
+	}
+}

weed/stats/metrics_ec_rebuild_test.go

@@ -0,0 +1,58 @@
+package stats
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestECRebuildMetricsRegistered(t *testing.T) {
+	t.Cleanup(VolumeServerECRebuildCounter.Reset)
+	t.Cleanup(VolumeServerECRebuildHistogram.Reset)
+
+	// A HistogramVec emits no metric family until it has a child, so observe once.
+	VolumeServerECRebuildCounter.WithLabelValues("success").Inc()
+	VolumeServerECRebuildHistogram.WithLabelValues("success").Observe(0.1)
+	metrics, err := Gather.Gather()
+	if err != nil {
+		t.Fatalf("failed to gather metrics: %v", err)
+	}
+
+	counterFound := false
+	histogramFound := false
+
+	for _, mf := range metrics {
+		if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_total" {
+			counterFound = true
+		}
+		if mf.GetName() == "SeaweedFS_volumeServer_ec_rebuild_seconds" {
+			histogramFound = true
+		}
+	}
+
+	if !counterFound {
+		t.Errorf("VolumeServerECRebuildCounter: metric not registered with Gather")
+	}
+	if !histogramFound {
+		t.Errorf("VolumeServerECRebuildHistogram: metric not registered with Gather")
+	}
+}
+
+func TestECRebuildCounterIncrement(t *testing.T) {
+	VolumeServerECRebuildCounter.Reset()
+	t.Cleanup(func() {
+		VolumeServerECRebuildCounter.Reset()
+	})
+
+	VolumeServerECRebuildCounter.WithLabelValues("success").Inc()
+	VolumeServerECRebuildCounter.WithLabelValues("failure").Inc()
+
+	got := testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("success"))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+	got = testutil.ToFloat64(VolumeServerECRebuildCounter.WithLabelValues("failure"))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+}

weed/stats/metrics_names.go

@@ -0,0 +1,90 @@
+package stats
+
+// This file contains metric names for all errors
+// The naming convention is ErrorSomeThing = "error.some.thing"
+const (
+	// volume server
+	WriteToLocalDisk   = "writeToLocalDisk"
+	WriteToReplicas    = "writeToReplicas"
+	DownloadLimitCond  = "downloadLimitCondition"
+	UploadLimitCond    = "uploadLimitCondition"
+	ReadProxyReq       = "readProxyRequest"
+	ReadRedirectReq    = "readRedirectRequest"
+	EmptyReadProxyLoc  = "emptyReadProxyLocaction"
+	FailedReadProxyReq = "failedReadProxyRequest"
+
+	ErrorSizeMismatchOffsetSize = "errorSizeMismatchOffsetSize"
+	ErrorSizeMismatch           = "errorSizeMismatch"
+	ErrorCRC                    = "errorCRC"
+	ErrorIndexOutOfRange        = "errorIndexOutOfRange"
+	ErrorGetNotFound            = "errorGetNotFound"
+	ErrorGetInternal            = "errorGetInternal"
+
+	// master topology
+	ErrorWriteToLocalDisk = "errorWriteToLocalDisk"
+	ErrorUnmarshalPairs   = "errorUnmarshalPairs"
+	ErrorWriteToReplicas  = "errorWriteToReplicas"
+
+	// master client
+	FailedToKeepConnected = "failedToKeepConnected"
+	FailedToSend          = "failedToSend"
+	FailedToReceive       = "failedToReceive"
+	RedirectedToLeader    = "redirectedToLeader"
+	OnPeerUpdate          = "onPeerUpdate"
+	Failed                = "failed"
+
+	// filer handler
+	DirList            = "dirList"
+	ContentSaveToFiler = "contentSaveToFiler"
+	AutoChunk          = "autoChunk"
+	ChunkProxy         = "chunkProxy"
+	ChunkAssign        = "chunkAssign"
+	ChunkUpload        = "chunkUpload"
+	ChunkMerge         = "chunkMerge"
+
+	ChunkDoUploadRetry       = "chunkDoUploadRetry"
+	ChunkUploadRetry         = "chunkUploadRetry"
+	ChunkAssignRetry         = "chunkAssignRetry"
+	ErrorReadNotFound        = "read.notfound"
+	ErrorReadInternal        = "read.internal.error"
+	ErrorWriteEntry          = "write.entry.failed"
+	RepeatErrorUploadContent = "upload.content.repeat.failed"
+	ErrorChunkAssign         = "chunkAssign.failed"
+	ErrorReadChunk           = "read.chunk.failed"
+	ErrorReadCache           = "read.cache.failed"
+	ErrorReadStream          = "read.stream.failed"
+
+	// s3 handler
+	ErrorCompletedNoSuchUpload      = "errorCompletedNoSuchUpload"
+	ErrorCompleteEntityTooSmall     = "errorCompleteEntityTooSmall"
+	ErrorCompletedPartEmpty         = "errorCompletedPartEmpty"
+	ErrorCompletedPartNumber        = "errorCompletedPartNumber"
+	ErrorCompletedPartNotFound      = "errorCompletedPartNotFound"
+	ErrorCompletedEtagInvalid       = "errorCompletedEtagInvalid"
+	ErrorCompletedEtagMismatch      = "errorCompletedEtagMismatch"
+	ErrorCompletedPartEntryMismatch = "errorCompletedPartEntryMismatch"
+
+	// ReplicationOpWrite is the label value for write replication operations.
+	ReplicationOpWrite = "write"
+	// ReplicationOpDelete is the label value for delete replication operations.
+	ReplicationOpDelete = "delete"
+	// ReplicationSuccess is the label value for successful replication operations.
+	ReplicationSuccess = "success"
+	// ReplicationFailure is the label value for failed replication operations.
+	ReplicationFailure = "failure"
+
+	// FailureTimeout is the failure reason label for deadline-exceeded errors.
+	FailureTimeout = "timeout"
+	// FailureConnectionRefused is the failure reason label for connection-refused errors.
+	FailureConnectionRefused = "connection_refused"
+	// FailureContextCancelled is the failure reason label for context-cancelled errors.
+	FailureContextCancelled = "context_cancelled"
+	// FailureServerError is the failure reason label for generic server-side errors.
+	FailureServerError = "server_error"
+
+	// remote-mount cache read labels
+	RemoteCacheSourceFiler = "filer"
+	RemoteCacheSourceS3    = "s3"
+	RemoteCacheResultHit   = "hit"
+	RemoteCacheResultMiss  = "miss"
+)

weed/stats/metrics_replication_test.go

@@ -0,0 +1,123 @@
+package stats
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestReplicationMetricsRegistered(t *testing.T) {
+
+	VolumeServerReplicationCounter.Reset()
+	VolumeServerReplicationHistogram.Reset()
+	VolumeServerReplicationFailures.Reset()
+	MasterUnderReplicatedVolumes.Reset()
+
+	t.Cleanup(func() {
+		VolumeServerReplicationCounter.Reset()
+		VolumeServerReplicationHistogram.Reset()
+		VolumeServerReplicationFailures.Reset()
+		MasterUnderReplicatedVolumes.Reset()
+	})
+
+	// Seed CounterVec and HistogramVec with a value so collection returns them
+	VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
+	VolumeServerReplicationHistogram.WithLabelValues(ReplicationOpWrite).Observe(0.1)
+	VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
+	VolumeServerReplicationTargets.Observe(1)
+	MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(1)
+
+	metrics := []struct {
+		name string
+		c    prometheus.Collector
+	}{
+		{"VolumeServerReplicationCounter", VolumeServerReplicationCounter},
+		{"VolumeServerReplicationHistogram", VolumeServerReplicationHistogram},
+		{"VolumeServerReplicationTargets", VolumeServerReplicationTargets},
+		{"VolumeServerReplicationFailures", VolumeServerReplicationFailures},
+		{"MasterUnderReplicatedVolumes", MasterUnderReplicatedVolumes},
+	}
+	for _, m := range metrics {
+		count := testutil.CollectAndCount(m.c)
+		if count < 1 {
+			t.Errorf("%s: expected at least 1 collection, got %d", m.name, count)
+		}
+	}
+}
+
+func TestReplicationCounterIncrement(t *testing.T) {
+	VolumeServerReplicationCounter.Reset()
+
+	VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
+	VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure).Inc()
+	VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess).Inc()
+
+	got := testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+	got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+	got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+}
+
+func TestReplicationTargetsHistogram(t *testing.T) {
+	// VolumeServerReplicationTargets is a plain prometheus.Histogram
+	// (not a HistogramVec) and does not expose a Reset() method, so the
+	// histogram state is cumulative across tests that share the Gather
+	// registry.  We check >= to tolerate prior observations.
+	VolumeServerReplicationTargets.Observe(3)
+
+	metrics, err := Gather.Gather()
+	if err != nil {
+		t.Fatalf("Failed to gather metrics: %v", err)
+	}
+
+	found := false
+	for _, mf := range metrics {
+		if mf.GetName() == "SeaweedFS_volumeServer_replication_targets" {
+			found = true
+			h := mf.GetMetric()[0].GetHistogram()
+			if h.GetSampleCount() < 1 {
+				t.Errorf("expected histogram_count>=1, got %d", h.GetSampleCount())
+			}
+			if h.GetSampleSum() < 3 {
+				t.Errorf("expected histogram_sum>=3, got %f", h.GetSampleSum())
+			}
+		}
+	}
+	if !found {
+		t.Error("SeaweedFS_volumeServer_replication_targets not found in gathered metrics")
+	}
+}
+
+func TestUnderReplicatedVolumesGauge(t *testing.T) {
+	MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(5)
+	got := testutil.ToFloat64(MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", ""))
+	if got != 5 {
+		t.Errorf("expected 5.0, got %f", got)
+	}
+}
+
+func TestReplicationFailuresCounter(t *testing.T) {
+	VolumeServerReplicationFailures.Reset()
+
+	VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
+	VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused).Inc()
+
+	got := testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+
+	got = testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused))
+	if got != 1 {
+		t.Errorf("expected 1.0, got %f", got)
+	}
+}

weed/stats/stats.go

@@ -0,0 +1,105 @@
+package stats
+
+import (
+	"time"
+)
+
+type ServerStats struct {
+	Requests       *DurationCounter
+	Connections    *DurationCounter
+	AssignRequests *DurationCounter
+	ReadRequests   *DurationCounter
+	WriteRequests  *DurationCounter
+	DeleteRequests *DurationCounter
+	BytesIn        *DurationCounter
+	BytesOut       *DurationCounter
+}
+
+type Channels struct {
+	Connections    chan *TimedValue
+	Requests       chan *TimedValue
+	AssignRequests chan *TimedValue
+	ReadRequests   chan *TimedValue
+	WriteRequests  chan *TimedValue
+	DeleteRequests chan *TimedValue
+	BytesIn        chan *TimedValue
+	BytesOut       chan *TimedValue
+}
+
+var (
+	Chan *Channels
+)
+
+func init() {
+	Chan = &Channels{
+		Connections:    make(chan *TimedValue, 100),
+		Requests:       make(chan *TimedValue, 100),
+		AssignRequests: make(chan *TimedValue, 100),
+		ReadRequests:   make(chan *TimedValue, 100),
+		WriteRequests:  make(chan *TimedValue, 100),
+		DeleteRequests: make(chan *TimedValue, 100),
+		BytesIn:        make(chan *TimedValue, 100),
+		BytesOut:       make(chan *TimedValue, 100),
+	}
+}
+
+func NewServerStats() *ServerStats {
+	return &ServerStats{
+		Requests:       NewDurationCounter(),
+		Connections:    NewDurationCounter(),
+		AssignRequests: NewDurationCounter(),
+		ReadRequests:   NewDurationCounter(),
+		WriteRequests:  NewDurationCounter(),
+		DeleteRequests: NewDurationCounter(),
+		BytesIn:        NewDurationCounter(),
+		BytesOut:       NewDurationCounter(),
+	}
+}
+
+func ConnectionOpen() {
+	Chan.Connections <- NewTimedValue(time.Now(), 1)
+}
+func ConnectionClose() {
+	Chan.Connections <- NewTimedValue(time.Now(), -1)
+}
+func AssignRequest() {
+	Chan.AssignRequests <- NewTimedValue(time.Now(), 1)
+}
+func ReadRequest() {
+	Chan.ReadRequests <- NewTimedValue(time.Now(), 1)
+}
+func WriteRequest() {
+	Chan.WriteRequests <- NewTimedValue(time.Now(), 1)
+}
+func DeleteRequest() {
+	Chan.DeleteRequests <- NewTimedValue(time.Now(), 1)
+}
+func BytesIn(val int64) {
+	Chan.BytesIn <- NewTimedValue(time.Now(), val)
+}
+func BytesOut(val int64) {
+	Chan.BytesOut <- NewTimedValue(time.Now(), val)
+}
+
+func (ss *ServerStats) Start() {
+	for {
+		select {
+		case tv := <-Chan.Connections:
+			ss.Connections.Add(tv)
+		case tv := <-Chan.Requests:
+			ss.Requests.Add(tv)
+		case tv := <-Chan.AssignRequests:
+			ss.AssignRequests.Add(tv)
+		case tv := <-Chan.ReadRequests:
+			ss.ReadRequests.Add(tv)
+		case tv := <-Chan.WriteRequests:
+			ss.WriteRequests.Add(tv)
+		case tv := <-Chan.DeleteRequests:
+			ss.DeleteRequests.Add(tv)
+		case tv := <-Chan.BytesIn:
+			ss.BytesIn.Add(tv)
+		case tv := <-Chan.BytesOut:
+			ss.BytesOut.Add(tv)
+		}
+	}
+}