snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

telemetry/server/storage/confirmed_test.go

@@ -0,0 +1,87 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func report(id, version string) *proto.TelemetryData {
+	return &proto.TelemetryData{
+		TopologyId:        id,
+		Version:           version,
+		Os:                "linux/amd64",
+		VolumeServerCount: 1,
+		TotalDiskBytes:    proto.MinDiskBytes,
+		TotalVolumeCount:  1,
+	}
+}
+
+func statsOf(t *testing.T, s *PrometheusStorage) map[string]interface{} {
+	t.Helper()
+	stats, err := s.GetStats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	return stats
+}
+
+func TestConfirmedClusters(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// A single-day cluster is active but not confirmed; with no confirmed
+	// clusters yet, distributions fall back to all active clusters.
+	if err := s.StoreTelemetry(report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")); err != nil {
+		t.Fatal(err)
+	}
+	stats := statsOf(t, s)
+	if stats["active_instances"] != 1 || stats["confirmed_instances"] != 0 {
+		t.Fatalf("day one: active=%v confirmed=%v, want 1/0", stats["active_instances"], stats["confirmed_instances"])
+	}
+	if v := stats["versions"].(map[string]int); v["4.40"] != 1 {
+		t.Fatalf("fallback distribution missing active cluster: %v", v)
+	}
+
+	// Give cluster A samples from the six previous days: now seen on 7
+	// distinct days.
+	s.mu.Lock()
+	id := "aaaaaaaa-0000-0000-0000-000000000001"
+	var older []HistorySample
+	for offset := -6; offset < 0; offset++ {
+		older = append(older, HistorySample{
+			Ts:             time.Now().AddDate(0, 0, offset).Unix(),
+			TotalDiskBytes: 50,
+		})
+	}
+	s.histories[id] = append(older, s.histories[id]...)
+	s.mu.Unlock()
+
+	// A one-shot cluster B arrives (like an injected report): it counts as
+	// active, but the distributions now only reflect confirmed clusters.
+	if err := s.StoreTelemetry(report("bbbbbbbb-0000-0000-0000-000000000002", "9.99")); err != nil {
+		t.Fatal(err)
+	}
+	stats = statsOf(t, s)
+	if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 {
+		t.Fatalf("day seven: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
+	}
+	v := stats["versions"].(map[string]int)
+	if v["4.40"] != 1 {
+		t.Errorf("confirmed cluster missing from distribution: %v", v)
+	}
+	if _, ok := v["9.99"]; ok {
+		t.Errorf("one-shot cluster polluted the distribution: %v", v)
+	}
+
+	// Cluster A meets the 1/3/7-day thresholds, B only the 1-day one, and
+	// unmet thresholds are present as zero so the dashboard can show them.
+	byDays := stats["confirmed_by_days"].(map[int]int)
+	want := map[int]int{1: 2, 3: 1, 7: 1, 14: 0, 30: 0}
+	for threshold, expected := range want {
+		if got, ok := byDays[threshold]; !ok || got != expected {
+			t.Errorf("confirmed_by_days[%d] = %v (present=%v), want %d", threshold, got, ok, expected)
+		}
+	}
+}

telemetry/server/storage/floor_test.go

@@ -0,0 +1,90 @@
+package storage
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func counterValue(t *testing.T, c prometheus.Counter) float64 {
+	t.Helper()
+	var m dto.Metric
+	if err := c.Write(&m); err != nil {
+		t.Fatal(err)
+	}
+	return m.GetCounter().GetValue()
+}
+
+func TestClustersUnderTheFloorAreNotKept(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// report() sits exactly on the floor, which is enough.
+	if err := s.StoreTelemetry(report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")); err != nil {
+		t.Fatal(err)
+	}
+	small := report("bbbbbbbb-0000-0000-0000-000000000002", "4.40")
+	small.TotalDiskBytes = proto.MinDiskBytes - 1
+	if err := s.StoreTelemetry(small); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, ok := s.instances[small.TopologyId]; ok {
+		t.Fatal("cluster under the floor was kept")
+	}
+	if _, ok := s.GetHistory(small.TopologyId, 90); ok {
+		t.Fatal("history kept for a cluster under the floor")
+	}
+	if stats := statsOf(t, s); stats["active_instances"] != 1 {
+		t.Errorf("active = %v, want the one cluster on the floor", stats["active_instances"])
+	}
+	if got := counterValue(t, s.telemetryReceived); got != 2 {
+		t.Errorf("received = %v, want 2", got)
+	}
+	if got := counterValue(t, s.reportsSkipped); got != 1 {
+		t.Errorf("skipped = %v, want 1", got)
+	}
+}
+
+// State written before the floor existed carries clusters under it; loading
+// drops them and marks the state dirty so the next save sheds them from disk.
+func TestLoadStateDropsClustersUnderTheFloor(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "telemetry-state.json")
+
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	kept := report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")
+	if err := s.StoreTelemetry(kept); err != nil {
+		t.Fatal(err)
+	}
+	small := report("bbbbbbbb-0000-0000-0000-000000000002", "4.40")
+	small.TotalDiskBytes = proto.MinDiskBytes - 1
+	s.instances[small.TopologyId] = &telemetryData{TelemetryData: small, ReceivedAt: time.Now()}
+	s.histories[small.TopologyId] = []HistorySample{{Ts: time.Now().Unix(), TotalDiskBytes: small.TotalDiskBytes}}
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatal(err)
+	}
+
+	s = newPrometheusStorage(prometheus.NewRegistry())
+	n, err := s.LoadState(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 1 {
+		t.Errorf("loaded %d instances, want 1", n)
+	}
+	if _, ok := s.instances[small.TopologyId]; ok {
+		t.Error("cluster under the floor survived the load")
+	}
+	if _, ok := s.histories[small.TopologyId]; ok {
+		t.Error("history of a cluster under the floor survived the load")
+	}
+	if _, ok := s.instances[kept.TopologyId]; !ok {
+		t.Error("cluster on the floor was dropped")
+	}
+	if !s.dirty {
+		t.Error("dropping a cluster left the state clean, so it would stay on disk")
+	}
+}

telemetry/server/storage/history.go

@@ -0,0 +1,175 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+// confirmDays is how many distinct UTC days a cluster must have reported
+// on before it counts as confirmed in the aggregated stats.
+const confirmDays = 7
+
+// confirmThresholds are the confirmation windows the dashboard lets the
+// viewer pick between; confirmDays is the one everything else is built on.
+var confirmThresholds = []int{1, 3, 7, 14, 30}
+
+// activeDays is how recently a cluster must have reported to count as active.
+const activeDays = 7
+
+// HistorySample is one retained data point of a cluster's daily reports.
+// Tags are kept short because thousands of samples end up in the state file.
+type HistorySample struct {
+	Ts                int64  `json:"ts"` // unix seconds the report was received
+	TotalDiskBytes    uint64 `json:"disk"`
+	TotalVolumeCount  int32  `json:"volumes"`
+	VolumeServerCount int32  `json:"servers"`
+	Version           string `json:"ver,omitempty"` // empty in samples written before this was recorded
+}
+
+// appendHistory records the report as the cluster's sample for the day,
+// replacing an earlier sample from the same UTC day. Callers must hold s.mu.
+func (s *PrometheusStorage) appendHistory(data *proto.TelemetryData, receivedAt time.Time) {
+	sample := HistorySample{
+		Ts:                receivedAt.Unix(),
+		TotalDiskBytes:    data.TotalDiskBytes,
+		TotalVolumeCount:  data.TotalVolumeCount,
+		VolumeServerCount: data.VolumeServerCount,
+		Version:           data.Version,
+	}
+	h := s.histories[data.TopologyId]
+	if n := len(h); n > 0 && sameUTCDay(h[n-1].Ts, sample.Ts) {
+		h[n-1] = sample
+	} else {
+		h = append(h, sample)
+	}
+	s.histories[data.TopologyId] = h
+}
+
+// seriesHistories picks the clusters the fleet-wide series are built from: the
+// confirmed ones. A cluster that reported for less than a week is usually a CI
+// or test cluster, and those arrive faster than they age out, so counting them
+// makes every fleet total climb forever. Falls back to
+// all clusters while none is confirmed yet, so a fresh server still draws its
+// charts. Callers must hold s.mu.
+func (s *PrometheusStorage) seriesHistories() map[string][]HistorySample {
+	confirmed := make(map[string][]HistorySample, len(s.histories))
+	for id, history := range s.histories {
+		if len(history) >= confirmDays {
+			confirmed[id] = history
+		}
+	}
+	if len(confirmed) == 0 {
+		return s.histories
+	}
+	return confirmed
+}
+
+func sameUTCDay(a, b int64) bool {
+	ta, tb := time.Unix(a, 0).UTC(), time.Unix(b, 0).UTC()
+	return ta.Year() == tb.Year() && ta.YearDay() == tb.YearDay()
+}
+
+// dailySeries is the shared date axis of the fleet-wide time series: one slot
+// per UTC day, ending today.
+type dailySeries struct {
+	dates []string
+	dayOf map[string]int
+}
+
+// newDailySeries builds the axis of UTC days ending today, spanning `days` days
+// but starting no earlier than the first day any cluster reported on: a fresh
+// server is asked for more days than it has history for, and padding those days
+// with zeros draws a climb out of nothing that never happened.
+func newDailySeries(days int, histories map[string][]HistorySample) dailySeries {
+	today := utcDay(time.Now().Unix())
+	requested := today.AddDate(0, 0, 1-days)
+
+	// Samples are appended in receive order, so [0] is a cluster's oldest.
+	var start time.Time
+	for _, history := range histories {
+		if len(history) == 0 {
+			continue
+		}
+		if first := utcDay(history[0].Ts); start.IsZero() || first.Before(start) {
+			start = first
+		}
+	}
+	if start.IsZero() || start.Before(requested) {
+		start = requested
+	}
+
+	n := int(today.Sub(start)/(24*time.Hour)) + 1
+	d := dailySeries{
+		dates: make([]string, n),
+		dayOf: make(map[string]int, n),
+	}
+	for i := range d.dates {
+		d.dates[i] = start.AddDate(0, 0, i).Format("2006-01-02")
+		d.dayOf[d.dates[i]] = i
+	}
+	return d
+}
+
+func utcDay(ts int64) time.Time {
+	t := time.Unix(ts, 0).UTC()
+	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)
+}
+
+func diskBytes(s HistorySample) uint64     { return s.TotalDiskBytes }
+func serverCount(s HistorySample) uint64   { return uint64(s.VolumeServerCount) }
+func sampleVersion(s HistorySample) string { return s.Version }
+
+// align lays one cluster's history onto the axis, picking `value` out of each
+// sample. Clusters report roughly once a day at no fixed hour, so a day without
+// a report carries the previous value forward rather than dropping to zero; a
+// cluster that stopped reporting altogether ends at its last sample instead of
+// holding capacity forever. Reports false when the cluster has nothing in range.
+func align[T any](d dailySeries, history []HistorySample, activeSince int64, value func(HistorySample) T) ([]T, bool) {
+	out := make([]T, len(d.dates))
+	reported := make([]bool, len(d.dates))
+	first, last := -1, -1
+	for _, sample := range history {
+		i, ok := d.dayOf[time.Unix(sample.Ts, 0).UTC().Format("2006-01-02")]
+		if !ok {
+			continue
+		}
+		out[i], reported[i] = value(sample), true
+		if first < 0 {
+			first = i
+		}
+		last = i
+		if sample.Ts >= activeSince {
+			last = len(d.dates) - 1 // still reporting, so hold to the right edge
+		}
+	}
+	if first < 0 {
+		return nil, false
+	}
+	for i := first + 1; i <= last; i++ {
+		if !reported[i] {
+			out[i] = out[i-1]
+		}
+	}
+	return out, true
+}
+
+// GetHistory returns the cluster's samples from the last `days` days.
+// The second return value reports whether the cluster is known at all.
+func (s *PrometheusStorage) GetHistory(clusterId string, days int) ([]HistorySample, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	h, ok := s.histories[clusterId]
+	if !ok {
+		return nil, false
+	}
+	cutoff := time.Now().AddDate(0, 0, -days).Unix()
+	samples := make([]HistorySample, 0, len(h))
+	for _, sample := range h {
+		if sample.Ts >= cutoff {
+			samples = append(samples, sample)
+		}
+	}
+	return samples, true
+}

telemetry/server/storage/history_test.go

@@ -0,0 +1,85 @@
+package storage
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func TestClusterHistory(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	report := &proto.TelemetryData{
+		TopologyId:        "hist-cluster",
+		Version:           "4.40",
+		Os:                "linux/amd64",
+		VolumeServerCount: 3,
+		TotalDiskBytes:    proto.MinDiskBytes + 1000,
+		TotalVolumeCount:  10,
+	}
+	if err := s.StoreTelemetry(report); err != nil {
+		t.Fatalf("store: %v", err)
+	}
+
+	// A second report on the same UTC day replaces the day's sample.
+	report.TotalDiskBytes = proto.MinDiskBytes + 2000
+	if err := s.StoreTelemetry(report); err != nil {
+		t.Fatalf("store: %v", err)
+	}
+	samples, ok := s.GetHistory("hist-cluster", 90)
+	if !ok {
+		t.Fatal("cluster missing from history")
+	}
+	if len(samples) != 1 {
+		t.Fatalf("got %d samples, want 1 (same-day replace)", len(samples))
+	}
+	if samples[0].TotalDiskBytes != proto.MinDiskBytes+2000 {
+		t.Errorf("same-day sample not replaced: got %d", samples[0].TotalDiskBytes)
+	}
+
+	// An older sample from a previous day is appended and survives the
+	// state-file round trip.
+	s.mu.Lock()
+	s.histories["hist-cluster"] = append([]HistorySample{{
+		Ts:             time.Now().AddDate(0, 0, -5).Unix(),
+		TotalDiskBytes: 500,
+	}}, s.histories["hist-cluster"]...)
+	s.mu.Unlock()
+
+	path := filepath.Join(t.TempDir(), "state.json")
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	s.instances = make(map[string]*telemetryData)
+	s.histories = make(map[string][]HistorySample)
+	if _, err := s.LoadState(path); err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	samples, ok = s.GetHistory("hist-cluster", 90)
+	if !ok || len(samples) != 2 {
+		t.Fatalf("after round trip: ok=%v samples=%d, want 2", ok, len(samples))
+	}
+	if samples[0].TotalDiskBytes != 500 || samples[1].TotalDiskBytes != proto.MinDiskBytes+2000 {
+		t.Errorf("samples corrupted after round trip: %+v", samples)
+	}
+
+	// GetHistory filters by the requested window.
+	samples, _ = s.GetHistory("hist-cluster", 3)
+	if len(samples) != 1 {
+		t.Errorf("window filter: got %d samples, want 1", len(samples))
+	}
+
+	// Unknown cluster reports !ok.
+	if _, ok := s.GetHistory("nope", 90); ok {
+		t.Error("unknown cluster reported ok")
+	}
+
+	// Cleanup drops the history together with the instance.
+	s.CleanupOldInstances(0)
+	if _, ok := s.GetHistory("hist-cluster", 90); ok {
+		t.Error("history survived instance cleanup")
+	}
+}

telemetry/server/storage/metrics_test.go

@@ -0,0 +1,163 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Every cluster counts towards every day it reported on, not just towards the
+// one day it last reported on.
+func TestGetMetricsSumsEachDay(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Reported every day of the window.
+	seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
+		-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	// Stopped reporting past the active window: counts on its own days only.
+	seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+		-14, -13, -12, -11, -10, -9, -8)
+
+	metrics, err := s.GetMetrics(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got := metrics["dates"].([]string); len(got) != 10 {
+		t.Fatalf("dates = %v, want 10 days", got)
+	}
+	if got := metrics["disk_usage"].([]uint64); !equal(got,
+		[]uint64{1200, 1200, 300, 300, 300, 300, 300, 300, 300, 300}) {
+		t.Errorf("disk_usage = %v, want the fleet total per day", got)
+	}
+	if got := metrics["server_counts"].([]int64); !equalInt64(got,
+		[]int64{12, 12, 3, 3, 3, 3, 3, 3, 3, 3}) {
+		t.Errorf("server_counts = %v, want the fleet total per day", got)
+	}
+}
+
+// A cluster that skipped a day keeps its size on that day rather than dipping
+// the fleet total to zero and back.
+func TestGetMetricsCarriesSkippedDaysForward(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	seedSamples(s, "gappy", HistorySample{TotalDiskBytes: 500, VolumeServerCount: 5},
+		-8, -7, -6, -5, -4, -3, -1)
+
+	metrics, err := s.GetMetrics(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{500, 500, 500, 500}) {
+		t.Errorf("disk_usage = %v, want the skipped days carried forward", got)
+	}
+}
+
+// The window starts at the oldest sample the server actually has, so a server
+// with less history than the caller asked for does not report a fleet that grew
+// out of nothing on its first day of data.
+func TestGetMetricsWindowStartsAtOldestSample(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	seedSamples(s, "recent", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 1},
+		-6, -5, -4, -3, -2, -1, 0)
+
+	metrics, err := s.GetMetrics(30)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := metrics["dates"].([]string); len(got) != 7 {
+		t.Errorf("dates = %v, want the 7 days with history, not 30", got)
+	}
+	if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{100, 100, 100, 100, 100, 100, 100}) {
+		t.Errorf("disk_usage = %v, want no leading zero days", got)
+	}
+
+	// History reaching past the requested window still clips to the window.
+	seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1},
+		-45, -44, -43, -42, -41, -40, -39)
+	metrics, err = s.GetMetrics(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := metrics["dates"].([]string); len(got) != 10 {
+		t.Errorf("dates = %v, want 10 days", got)
+	}
+}
+
+// /api/metrics is the fleet total that /api/cluster-sizes breaks down per
+// cluster, so their last day has to add up to the same number.
+func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
+		-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
+		-8, -7, -6, -5, -4, -3, -2)
+	seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+		-14, -13, -12, -11, -10, -9, -8)
+
+	metrics, err := s.GetMetrics(10)
+	if err != nil {
+		t.Fatal(err)
+	}
+	disk := metrics["disk_usage"].([]uint64)
+	sizes := s.GetClusterSizeSeries(10, 1) // limit forces the Other fold-in too
+
+	if got, want := disk[len(disk)-1], sizes.TotalDisk; got != want {
+		t.Errorf("metrics last day = %d, cluster sizes total = %d", got, want)
+	}
+	if len(disk) != len(sizes.Dates) {
+		t.Errorf("metrics has %d days, cluster sizes has %d", len(disk), len(sizes.Dates))
+	}
+}
+
+// Short-lived clusters -- a CI run, a docker-compose demo -- report once under a
+// fresh raft topology id and never again. Held forward for the whole active
+// window they would stack up into a fleet that grows every day, so they stay out
+// of the totals until they are confirmed.
+func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
+		-6, -5, -4, -3, -2, -1, 0)
+	for _, id := range []string{"ci-1", "ci-2", "ci-3"} {
+		seedSamples(s, id, HistorySample{TotalDiskBytes: 5, VolumeServerCount: 14}, -1)
+	}
+
+	metrics, err := s.GetMetrics(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := metrics["server_counts"].([]int64); !equalInt64(got, []int64{3, 3, 3, 3}) {
+		t.Errorf("server_counts = %v, want the confirmed cluster alone", got)
+	}
+	if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{300, 300, 300, 300}) {
+		t.Errorf("disk_usage = %v, want the confirmed cluster alone", got)
+	}
+}
+
+// Until any cluster has a week of history the charts fall back to every
+// cluster, so a fresh server doesn't serve empty series.
+func TestGetMetricsFallsBackWhenNoneConfirmed(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	seedSamples(s, "new", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 2}, 0)
+
+	metrics, err := s.GetMetrics(7)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := metrics["server_counts"].([]int64); !equalInt64(got, []int64{2}) {
+		t.Errorf("server_counts = %v, want today's only cluster", got)
+	}
+}
+
+func equalInt64(a, b []int64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

telemetry/server/storage/persistence.go

@@ -0,0 +1,115 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+// persistedState is the on-disk snapshot of the in-memory instance map.
+type persistedState struct {
+	Instances map[string]*telemetryData  `json:"instances"`
+	Histories map[string][]HistorySample `json:"histories,omitempty"`
+}
+
+// LoadState restores the instance map and Prometheus gauges from a state file
+// written by SaveStateIfDirty. A missing file is not an error. Original
+// ReceivedAt timestamps are preserved so cleanup and the active-cluster
+// windows stay correct across restarts. Clusters under proto.MinDiskBytes are
+// dropped, so state written before the floor sheds them on the first restart.
+func (s *PrometheusStorage) LoadState(path string) (int, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	var state persistedState
+	if err := json.Unmarshal(b, &state); err != nil {
+		return 0, fmt.Errorf("parse %s: %w", path, err)
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	loaded := 0
+	for id, instance := range state.Instances {
+		if instance == nil || instance.TelemetryData == nil || instance.TelemetryData.TopologyId == "" {
+			continue
+		}
+		if instance.TelemetryData.TotalDiskBytes < proto.MinDiskBytes {
+			s.dirty = true // so the next save sheds it
+			continue
+		}
+		s.instances[id] = instance
+		s.setClusterMetrics(instance.TelemetryData)
+		loaded++
+	}
+	for id, h := range state.Histories {
+		instance, ok := s.instances[id]
+		if !ok || len(h) == 0 {
+			continue
+		}
+		// State written before versions were recorded carries none on its
+		// samples. The newest sample is the report the instance record itself
+		// came from, so that day's version is known and the version series can
+		// start there rather than a day after the upgrade.
+		if newest := len(h) - 1; h[newest].Version == "" {
+			h[newest].Version = instance.TelemetryData.Version
+		}
+		s.histories[id] = h
+	}
+	s.updateStats()
+	return loaded, nil
+}
+
+// SaveStateIfDirty writes the instance map to path if it changed since the
+// last successful save. The write is atomic (temp file + rename).
+func (s *PrometheusStorage) SaveStateIfDirty(path string) error {
+	s.mu.Lock()
+	if !s.dirty {
+		s.mu.Unlock()
+		return nil
+	}
+	b, err := json.Marshal(&persistedState{Instances: s.instances, Histories: s.histories})
+	if err != nil {
+		s.mu.Unlock()
+		return err
+	}
+	s.dirty = false
+	s.mu.Unlock()
+
+	if err := s.writeAtomically(path, b); err != nil {
+		s.mu.Lock()
+		s.dirty = true
+		s.mu.Unlock()
+		return err
+	}
+	return nil
+}
+
+func (s *PrometheusStorage) writeAtomically(path string, b []byte) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp-*")
+	if err != nil {
+		return err
+	}
+	if _, err := tmp.Write(b); err != nil {
+		tmp.Close()
+		os.Remove(tmp.Name())
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		os.Remove(tmp.Name())
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

telemetry/server/storage/persistence_test.go

@@ -0,0 +1,109 @@
+package storage
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func TestStateRoundTrip(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "data", "telemetry-state.json")
+
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatalf("clean save: %v", err)
+	}
+	if _, err := s.LoadState(path); err != nil {
+		t.Fatalf("load with no state file: %v", err)
+	}
+
+	report := &proto.TelemetryData{
+		TopologyId:        "test-cluster-1",
+		Version:           "4.40",
+		Os:                "linux/amd64",
+		VolumeServerCount: 5,
+		TotalDiskBytes:    123 << 30,
+		TotalVolumeCount:  42,
+		FilerCount:        2,
+		BrokerCount:       1,
+		Timestamp:         time.Now().Unix(),
+	}
+	if err := s.StoreTelemetry(report); err != nil {
+		t.Fatalf("store: %v", err)
+	}
+	receivedAt := s.instances[report.TopologyId].ReceivedAt
+
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+	if s.dirty {
+		t.Fatal("dirty flag not cleared after save")
+	}
+
+	// Simulate a restart: wipe the in-memory map, then restore.
+	s.instances = make(map[string]*telemetryData)
+	n, err := s.LoadState(path)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	if n != 1 {
+		t.Fatalf("loaded %d instances, want 1", n)
+	}
+
+	got, ok := s.instances[report.TopologyId]
+	if !ok {
+		t.Fatal("instance missing after load")
+	}
+	if !got.ReceivedAt.Equal(receivedAt) {
+		t.Errorf("ReceivedAt not preserved: got %v, want %v", got.ReceivedAt, receivedAt)
+	}
+	if got.TelemetryData.TotalDiskBytes != report.TotalDiskBytes ||
+		got.TelemetryData.Version != report.Version ||
+		got.TelemetryData.VolumeServerCount != report.VolumeServerCount {
+		t.Errorf("fields not preserved: got %+v", got.TelemetryData)
+	}
+
+	// Load must not mark state dirty.
+	if s.dirty {
+		t.Error("dirty flag set after load")
+	}
+}
+
+// State written before versions were recorded still knows the version of the
+// report its newest sample came from: the instance record's.
+func TestLoadStateFillsNewestSampleVersion(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "telemetry-state.json")
+
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	report := &proto.TelemetryData{TopologyId: "test-cluster-1", Version: "4.40", Os: "linux/amd64", TotalDiskBytes: proto.MinDiskBytes}
+	if err := s.StoreTelemetry(report); err != nil {
+		t.Fatalf("store: %v", err)
+	}
+	yesterday := time.Now().AddDate(0, 0, -1).Unix()
+	s.histories[report.TopologyId] = []HistorySample{
+		{Ts: yesterday, TotalDiskBytes: 10},
+		{Ts: time.Now().Unix(), TotalDiskBytes: 20},
+	}
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+
+	s = newPrometheusStorage(prometheus.NewRegistry())
+	if _, err := s.LoadState(path); err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	loaded := s.histories[report.TopologyId]
+	if len(loaded) != 2 {
+		t.Fatalf("history = %+v, want 2 samples", loaded)
+	}
+	if loaded[1].Version != report.Version {
+		t.Errorf("newest sample version = %q, want %q", loaded[1].Version, report.Version)
+	}
+	if loaded[0].Version != "" {
+		t.Errorf("older sample version = %q, want it left unknown", loaded[0].Version)
+	}
+}

telemetry/server/storage/prometheus.go

@@ -0,0 +1,332 @@
+package storage
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+type PrometheusStorage struct {
+	// Prometheus metrics
+	totalClusters     prometheus.Gauge
+	activeClusters    prometheus.Gauge
+	confirmedClusters prometheus.Gauge
+	volumeServerCount *prometheus.GaugeVec
+	totalDiskBytes    *prometheus.GaugeVec
+	totalVolumeCount  *prometheus.GaugeVec
+	filerCount        *prometheus.GaugeVec
+	brokerCount       *prometheus.GaugeVec
+	clusterInfo       *prometheus.GaugeVec
+	telemetryReceived prometheus.Counter
+	reportsSkipped    prometheus.Counter
+
+	// In-memory storage for API endpoints (if needed)
+	mu        sync.RWMutex
+	instances map[string]*telemetryData
+	histories map[string][]HistorySample
+	stats     map[string]interface{}
+	dirty     bool // instances changed since the last successful state save
+}
+
+// telemetryData is an internal struct that includes the received timestamp
+type telemetryData struct {
+	*proto.TelemetryData
+	ReceivedAt time.Time `json:"received_at"`
+}
+
+func NewPrometheusStorage() *PrometheusStorage {
+	return newPrometheusStorage(prometheus.DefaultRegisterer)
+}
+
+func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
+	promauto := promauto.With(reg)
+	return &PrometheusStorage{
+		totalClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_total_clusters",
+			Help: "Total number of unique SeaweedFS clusters (last 30 days)",
+		}),
+		activeClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_active_clusters",
+			Help: "Number of active SeaweedFS clusters (last 7 days)",
+		}),
+		confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_confirmed_clusters",
+			Help: "Active clusters seen on at least 7 distinct days (last 7 days)",
+		}),
+		volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_volume_servers",
+			Help: "Number of volume servers per cluster",
+		}, []string{"cluster_id"}),
+		totalDiskBytes: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_disk_bytes",
+			Help: "Total disk usage in bytes per cluster",
+		}, []string{"cluster_id"}),
+		totalVolumeCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_volume_count",
+			Help: "Total number of volumes per cluster",
+		}, []string{"cluster_id"}),
+		filerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_filer_count",
+			Help: "Number of filer servers per cluster",
+		}, []string{"cluster_id"}),
+		brokerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_broker_count",
+			Help: "Number of broker servers per cluster",
+		}, []string{"cluster_id"}),
+		clusterInfo: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_cluster_info",
+			Help: "Cluster information (always 1, labels contain metadata)",
+		}, []string{"cluster_id", "version", "os"}),
+		telemetryReceived: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "seaweedfs_telemetry_reports_received_total",
+			Help: "Total number of telemetry reports received",
+		}),
+		reportsSkipped: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "seaweedfs_telemetry_reports_skipped_total",
+			Help: fmt.Sprintf("Reports not kept because the cluster stores less than %d GiB", proto.MinDiskBytes>>30),
+		}),
+		instances: make(map[string]*telemetryData),
+		histories: make(map[string][]HistorySample),
+		stats:     make(map[string]interface{}),
+	}
+}
+
+func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.telemetryReceived.Inc()
+	if data.TotalDiskBytes < proto.MinDiskBytes {
+		s.reportsSkipped.Inc()
+		return nil
+	}
+
+	// Drop the cluster_info series recorded under the previous label set when
+	// a cluster reports back with a different version or OS, so it is not
+	// counted under two versions at once.
+	if prev, ok := s.instances[data.TopologyId]; ok &&
+		(prev.TelemetryData.Version != data.Version || prev.TelemetryData.Os != data.Os) {
+		s.clusterInfo.Delete(infoLabels(prev.TelemetryData))
+	}
+	s.setClusterMetrics(data)
+
+	// Store in memory for API endpoints
+	receivedAt := time.Now().UTC()
+	s.instances[data.TopologyId] = &telemetryData{
+		TelemetryData: data,
+		ReceivedAt:    receivedAt,
+	}
+	s.appendHistory(data, receivedAt)
+	s.dirty = true
+
+	// Update aggregated stats
+	s.updateStats()
+
+	return nil
+}
+
+// setClusterMetrics records a report's values on the Prometheus gauges.
+// Value gauges are keyed by cluster_id only so a cluster's series continues
+// across upgrades; version/os metadata lives on cluster_info (join with
+// `* on(cluster_id) group_left(version, os)`). Callers must hold s.mu.
+func (s *PrometheusStorage) setClusterMetrics(data *proto.TelemetryData) {
+	labels := prometheus.Labels{
+		"cluster_id": data.TopologyId,
+	}
+	s.volumeServerCount.With(labels).Set(float64(data.VolumeServerCount))
+	s.totalDiskBytes.With(labels).Set(float64(data.TotalDiskBytes))
+	s.totalVolumeCount.With(labels).Set(float64(data.TotalVolumeCount))
+	s.filerCount.With(labels).Set(float64(data.FilerCount))
+	s.brokerCount.With(labels).Set(float64(data.BrokerCount))
+	s.clusterInfo.With(infoLabels(data)).Set(1)
+}
+
+func (s *PrometheusStorage) GetStats() (map[string]interface{}, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Return cached stats
+	result := make(map[string]interface{})
+	for k, v := range s.stats {
+		result[k] = v
+	}
+	return result, nil
+}
+
+func (s *PrometheusStorage) GetInstances(limit int) ([]*telemetryData, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	var instances []*telemetryData
+	count := 0
+	for _, instance := range s.instances {
+		if count >= limit {
+			break
+		}
+		instances = append(instances, instance)
+		count++
+	}
+
+	return instances, nil
+}
+
+// GetMetrics returns fleet-wide daily totals across confirmed clusters for the
+// last `days` days, in the parallel-array shape the dashboard charts expect.
+// Totals come from the daily histories rather than from s.instances, which holds
+// only each cluster's most recent report and so would credit every cluster to
+// the single day it last reported on.
+func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	histories := s.seriesHistories()
+	axis := newDailySeries(days, histories)
+	activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
+
+	diskUsage := make([]uint64, len(axis.dates))
+	serverCounts := make([]int64, len(axis.dates))
+	for _, history := range histories {
+		if disk, ok := align(axis, history, activeSince, diskBytes); ok {
+			for i, v := range disk {
+				diskUsage[i] += v
+			}
+		}
+		if servers, ok := align(axis, history, activeSince, serverCount); ok {
+			for i, v := range servers {
+				serverCounts[i] += int64(v)
+			}
+		}
+	}
+
+	return map[string]interface{}{
+		"dates":         axis.dates,
+		"server_counts": serverCounts,
+		"disk_usage":    diskUsage,
+	}, nil
+}
+
+func (s *PrometheusStorage) updateStats() {
+	now := time.Now()
+	last7Days := now.AddDate(0, 0, -activeDays)
+	last30Days := now.AddDate(0, 0, -30)
+
+	totalInstances := 0
+	activeInstances := 0
+	confirmedInstances := 0
+	confirmedByDays := make(map[int]int, len(confirmThresholds))
+	for _, threshold := range confirmThresholds {
+		confirmedByDays[threshold] = 0
+	}
+	versionsAll := make(map[string]int)
+	osAll := make(map[string]int)
+	versionsConfirmed := make(map[string]int)
+	osConfirmed := make(map[string]int)
+
+	for _, instance := range s.instances {
+		if instance.ReceivedAt.After(last30Days) {
+			totalInstances++
+		}
+		if instance.ReceivedAt.After(last7Days) {
+			activeInstances++
+			versionsAll[instance.TelemetryData.Version]++
+			osAll[instance.TelemetryData.Os]++
+			// A cluster is confirmed once seen on confirmDays distinct UTC
+			// days (histories hold one sample per day), so short-lived
+			// clusters can't skew the distributions below.
+			daysSeen := len(s.histories[instance.TelemetryData.TopologyId])
+			for _, threshold := range confirmThresholds {
+				if daysSeen >= threshold {
+					confirmedByDays[threshold]++
+				}
+			}
+			if daysSeen >= confirmDays {
+				confirmedInstances++
+				versionsConfirmed[instance.TelemetryData.Version]++
+				osConfirmed[instance.TelemetryData.Os]++
+			}
+		}
+	}
+
+	// Before any cluster has a week of history (fresh server with no
+	// prior state), fall back to all active clusters so the dashboard
+	// distributions aren't empty.
+	versions, osDistribution := versionsConfirmed, osConfirmed
+	if confirmedInstances == 0 {
+		versions, osDistribution = versionsAll, osAll
+	}
+
+	// Update Prometheus gauges
+	s.totalClusters.Set(float64(totalInstances))
+	s.activeClusters.Set(float64(activeInstances))
+	s.confirmedClusters.Set(float64(confirmedInstances))
+
+	// Update cached stats for API
+	s.stats = map[string]interface{}{
+		"total_instances":     totalInstances,
+		"active_instances":    activeInstances,
+		"confirmed_instances": confirmedInstances,
+		"confirmed_by_days":   confirmedByDays,
+		"versions":            versions,
+		"os_distribution":     osDistribution,
+	}
+}
+
+// CleanupOldInstances removes instances older than the specified duration
+func (s *PrometheusStorage) CleanupOldInstances(maxAge time.Duration) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	cutoff := time.Now().Add(-maxAge)
+	for instanceID, instance := range s.instances {
+		if instance.ReceivedAt.Before(cutoff) {
+			delete(s.instances, instanceID)
+			s.deleteClusterMetrics(instance.TelemetryData)
+			s.dirty = true
+		}
+	}
+
+	for id, h := range s.histories {
+		if _, ok := s.instances[id]; !ok {
+			delete(s.histories, id)
+			s.dirty = true
+			continue
+		}
+		i := 0
+		for i < len(h) && time.Unix(h[i].Ts, 0).Before(cutoff) {
+			i++
+		}
+		if i > 0 {
+			s.histories[id] = append([]HistorySample(nil), h[i:]...)
+			s.dirty = true
+		}
+	}
+
+	s.updateStats()
+}
+
+// deleteClusterMetrics removes all gauges stored for the given report's
+// cluster. Callers must hold s.mu.
+func (s *PrometheusStorage) deleteClusterMetrics(data *proto.TelemetryData) {
+	labels := prometheus.Labels{
+		"cluster_id": data.TopologyId,
+	}
+	s.volumeServerCount.Delete(labels)
+	s.totalDiskBytes.Delete(labels)
+	s.totalVolumeCount.Delete(labels)
+	s.filerCount.Delete(labels)
+	s.brokerCount.Delete(labels)
+	s.clusterInfo.Delete(infoLabels(data))
+}
+
+// infoLabels is the full label set used by the cluster_info metric.
+func infoLabels(data *proto.TelemetryData) prometheus.Labels {
+	return prometheus.Labels{
+		"cluster_id": data.TopologyId,
+		"version":    data.Version,
+		"os":         data.Os,
+	}
+}

telemetry/server/storage/sizes.go

@@ -0,0 +1,90 @@
+package storage
+
+import (
+	"sort"
+	"time"
+)
+
+// ClusterSeries is one cluster's daily disk usage and volume server count,
+// aligned to the shared date axis of the enclosing ClusterSizeSeries.
+type ClusterSeries struct {
+	ClusterId string   `json:"cluster_id"`
+	Disk      []uint64 `json:"disk"`
+	Servers   []uint64 `json:"servers"`
+}
+
+// OtherSeries is the clusters beyond the caller's limit, summed per day so a
+// stacked chart still adds up to the fleet total.
+type OtherSeries struct {
+	Count   int      `json:"count"`
+	Disk    []uint64 `json:"disk"`
+	Servers []uint64 `json:"servers"`
+}
+
+// ClusterSizeSeries is per-cluster disk usage and volume server count over
+// time: one value per cluster per day, largest cluster first, ranked by disk on
+// their most recent day. Both metrics share one ranking so a cluster keeps its
+// place, and its colour, across the charts drawn from this.
+type ClusterSizeSeries struct {
+	Dates        []string        `json:"dates"`
+	Clusters     []ClusterSeries `json:"clusters"`
+	Other        *OtherSeries    `json:"other,omitempty"`
+	ClusterCount int             `json:"cluster_count"`
+	TotalDisk    uint64          `json:"total_disk"`    // across all clusters on the last day
+	TotalServers uint64          `json:"total_servers"` // across all clusters on the last day
+}
+
+// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage
+// and volume server counts across confirmed clusters. Clusters beyond `limit`
+// are folded into Other.
+func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSeries {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	histories := s.seriesHistories()
+	axis := newDailySeries(days, histories)
+	activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
+	last := len(axis.dates) - 1
+	series := ClusterSizeSeries{Dates: axis.dates}
+
+	for id, history := range histories {
+		disk, ok := align(axis, history, activeSince, diskBytes)
+		if !ok {
+			continue
+		}
+		servers, _ := align(axis, history, activeSince, serverCount)
+		series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk, Servers: servers})
+		series.TotalDisk += disk[last]
+		series.TotalServers += servers[last]
+	}
+	series.ClusterCount = len(series.Clusters)
+
+	// Rank by the latest day so the stack reads largest-first at its right
+	// edge, tie-breaking on id to keep the order stable across refreshes.
+	sort.Slice(series.Clusters, func(i, j int) bool {
+		a, b := series.Clusters[i], series.Clusters[j]
+		if a.Disk[last] != b.Disk[last] {
+			return a.Disk[last] > b.Disk[last]
+		}
+		return a.ClusterId < b.ClusterId
+	})
+
+	if limit > 0 && len(series.Clusters) > limit {
+		other := OtherSeries{
+			Count:   len(series.Clusters) - limit,
+			Disk:    make([]uint64, len(axis.dates)),
+			Servers: make([]uint64, len(axis.dates)),
+		}
+		for _, c := range series.Clusters[limit:] {
+			for i, v := range c.Disk {
+				other.Disk[i] += v
+			}
+			for i, v := range c.Servers {
+				other.Servers[i] += v
+			}
+		}
+		series.Clusters = series.Clusters[:limit]
+		series.Other = &other
+	}
+	return series
+}

telemetry/server/storage/sizes_test.go

@@ -0,0 +1,157 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+// seedSamples gives a cluster one sample per listed day offset (0 is today).
+// The sample's Ts is filled in per day offset.
+func seedSamples(s *PrometheusStorage, id string, sample HistorySample, dayOffsets ...int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, offset := range dayOffsets {
+		sample.Ts = time.Now().AddDate(0, 0, offset).Unix()
+		s.histories[id] = append(s.histories[id], sample)
+	}
+}
+
+func TestClusterSizeSeries(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Reported every day of the window.
+	seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
+		-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	// Stopped reporting two days ago: still active, so its size is held to
+	// the right edge instead of dropping out of the stack.
+	seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
+		-8, -7, -6, -5, -4, -3, -2)
+	// Stopped reporting past the active window: its own days only.
+	seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+		-14, -13, -12, -11, -10, -9, -8)
+	// One day of history only: unconfirmed, so it stays out of the stack.
+	seedSamples(s, "oneshot", HistorySample{TotalDiskBytes: 400, VolumeServerCount: 4}, -1)
+
+	series := s.GetClusterSizeSeries(10, 0)
+	if len(series.Dates) != 10 {
+		t.Fatalf("dates = %v, want 10 days", series.Dates)
+	}
+	if series.ClusterCount != 3 {
+		t.Fatalf("cluster_count = %d, want 3", series.ClusterCount)
+	}
+
+	byId := map[string]ClusterSeries{}
+	for _, c := range series.Clusters {
+		byId[c.ClusterId] = c
+	}
+	if got := byId["daily"].Disk; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
+		t.Errorf("daily = %v, want 300 every day", got)
+	}
+	if got := byId["lagging"].Disk; !equal(got, []uint64{0, 200, 200, 200, 200, 200, 200, 200, 200, 200}) {
+		t.Errorf("lagging = %v, want its size carried to the right edge", got)
+	}
+	if got := byId["gone"].Disk; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
+		t.Errorf("gone = %v, want no capacity after its last report", got)
+	}
+	if _, ok := byId["oneshot"]; ok {
+		t.Errorf("unconfirmed cluster in the stack: %+v", byId["oneshot"])
+	}
+
+	// Volume servers ride the same axis and the same hold-forward rule.
+	if got := byId["daily"].Servers; !equal(got, []uint64{3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) {
+		t.Errorf("daily servers = %v, want 3 every day", got)
+	}
+	if got := byId["lagging"].Servers; !equal(got, []uint64{0, 2, 2, 2, 2, 2, 2, 2, 2, 2}) {
+		t.Errorf("lagging servers = %v, want carried to the right edge", got)
+	}
+	if got := byId["gone"].Servers; !equal(got, []uint64{9, 9, 0, 0, 0, 0, 0, 0, 0, 0}) {
+		t.Errorf("gone servers = %v, want none after its last report", got)
+	}
+
+	// The totals are the last day's stack height: daily + lagging, not gone.
+	if series.TotalDisk != 500 {
+		t.Errorf("total_disk = %d, want 500", series.TotalDisk)
+	}
+	if series.TotalServers != 5 {
+		t.Errorf("total_servers = %d, want 5", series.TotalServers)
+	}
+	// Largest on the last day comes first so the stack reads top-down.
+	if series.Clusters[0].ClusterId != "daily" {
+		t.Errorf("order = %s first, want daily", series.Clusters[0].ClusterId)
+	}
+
+	// Clusters past the limit are summed into "other", per day.
+	series = s.GetClusterSizeSeries(10, 1)
+	if len(series.Clusters) != 1 || series.Clusters[0].ClusterId != "daily" {
+		t.Fatalf("limited clusters = %+v, want just daily", series.Clusters)
+	}
+	if series.Other == nil || series.Other.Count != 2 {
+		t.Fatalf("other = %+v, want 2 clusters", series.Other)
+	}
+	if !equal(series.Other.Disk, []uint64{900, 1100, 200, 200, 200, 200, 200, 200, 200, 200}) {
+		t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
+	}
+	if !equal(series.Other.Servers, []uint64{9, 11, 2, 2, 2, 2, 2, 2, 2, 2}) {
+		t.Errorf("other servers = %v, want lagging+gone summed per day", series.Other.Servers)
+	}
+	if series.ClusterCount != 3 || series.TotalDisk != 500 || series.TotalServers != 5 {
+		t.Errorf("limit changed totals: count=%d disk=%d servers=%d, want 3/500/5",
+			series.ClusterCount, series.TotalDisk, series.TotalServers)
+	}
+}
+
+// A report that lands after an earlier one on the same day replaces it, so the
+// series shows one value per cluster per day.
+func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	data := &proto.TelemetryData{
+		TopologyId:        "aaaaaaaa-0000-0000-0000-000000000001",
+		Version:           "4.40",
+		Os:                "linux/amd64",
+		TotalDiskBytes:    proto.MinDiskBytes + 100,
+		VolumeServerCount: 4,
+	}
+	if err := s.StoreTelemetry(data); err != nil {
+		t.Fatal(err)
+	}
+	data.TotalDiskBytes = proto.MinDiskBytes + 700
+	data.VolumeServerCount = 6
+	if err := s.StoreTelemetry(data); err != nil {
+		t.Fatal(err)
+	}
+
+	// Nothing was reported before today, so the axis is today alone rather
+	// than the full 3 days padded out with zeros.
+	series := s.GetClusterSizeSeries(3, 0)
+	if len(series.Clusters) != 1 {
+		t.Fatalf("clusters = %+v, want 1", series.Clusters)
+	}
+	if today := time.Now().UTC().Format("2006-01-02"); len(series.Dates) != 1 || series.Dates[0] != today {
+		t.Errorf("dates = %v, want %s only", series.Dates, today)
+	}
+	if got := series.Clusters[0].Disk; !equal(got, []uint64{proto.MinDiskBytes + 700}) {
+		t.Errorf("disk = %v, want today's latest sample only", got)
+	}
+	if got := series.Clusters[0].Servers; !equal(got, []uint64{6}) {
+		t.Errorf("servers = %v, want today's latest sample only", got)
+	}
+	if series.TotalDisk != proto.MinDiskBytes+700 || series.TotalServers != 6 {
+		t.Errorf("totals = %d disk / %d servers, want %d/6", series.TotalDisk, series.TotalServers, proto.MinDiskBytes+700)
+	}
+}
+
+func equal(a, b []uint64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}

telemetry/server/storage/versions.go

@@ -0,0 +1,163 @@
+package storage
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// VersionCounts is how many clusters ran one version per day, aligned to the
+// shared date axis of the enclosing VersionSeries.
+type VersionCounts struct {
+	Version  string   `json:"version"`
+	Clusters []uint64 `json:"clusters"`
+}
+
+// OtherVersions is the versions beyond the caller's limit, summed per day so a
+// stacked chart still adds up to the cluster total.
+type OtherVersions struct {
+	Count    int      `json:"count"`
+	Clusters []uint64 `json:"clusters"`
+}
+
+// VersionSeries is the version make-up of the fleet over time: one cluster
+// count per version per day, oldest release first.
+type VersionSeries struct {
+	Dates         []string        `json:"dates"`
+	Versions      []VersionCounts `json:"versions"`
+	Other         *OtherVersions  `json:"other,omitempty"`
+	TotalClusters uint64          `json:"total_clusters"` // across all versions on the last day
+}
+
+// GetVersionSeries returns the last `days` days of per-version cluster counts
+// across confirmed clusters. Versions beyond `limit` are folded into Other,
+// keeping the ones with the most clusters on the last day.
+func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	histories := versionedHistories(s.seriesHistories())
+	axis := newDailySeries(days, histories)
+	activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
+	last := len(axis.dates) - 1
+	series := VersionSeries{Dates: axis.dates}
+
+	daily := make(map[string][]uint64)
+	for _, history := range histories {
+		versions, ok := align(axis, history, activeSince, sampleVersion)
+		if !ok {
+			continue
+		}
+		for i, v := range versions {
+			if v == "" {
+				continue // before the cluster's first day in the window
+			}
+			counts, ok := daily[v]
+			if !ok {
+				counts = make([]uint64, len(axis.dates))
+				daily[v] = counts
+			}
+			counts[i]++
+		}
+	}
+
+	for v, counts := range daily {
+		series.Versions = append(series.Versions, VersionCounts{Version: v, Clusters: counts})
+		series.TotalClusters += counts[last]
+	}
+	// Ordered by release rather than by size: the caller stacks them, and a
+	// stack whose order changes with the counts is unreadable over time.
+	sort.Slice(series.Versions, func(i, j int) bool {
+		return versionLess(series.Versions[i].Version, series.Versions[j].Version)
+	})
+
+	if limit > 0 && len(series.Versions) > limit {
+		// Old releases are a long tail of one-cluster bands; keep the versions
+		// most of the fleet is on now and sum the rest into one band.
+		ranked := append([]VersionCounts(nil), series.Versions...)
+		sort.Slice(ranked, func(i, j int) bool {
+			if ranked[i].Clusters[last] != ranked[j].Clusters[last] {
+				return ranked[i].Clusters[last] > ranked[j].Clusters[last]
+			}
+			return versionLess(ranked[j].Version, ranked[i].Version)
+		})
+		other := OtherVersions{Count: len(ranked) - limit, Clusters: make([]uint64, len(axis.dates))}
+		for _, v := range ranked[limit:] {
+			for i, c := range v.Clusters {
+				other.Clusters[i] += c
+			}
+		}
+		kept := make(map[string]bool, limit)
+		for _, v := range ranked[:limit] {
+			kept[v.Version] = true
+		}
+		versions := series.Versions[:0]
+		for _, v := range series.Versions {
+			if kept[v.Version] {
+				versions = append(versions, v)
+			}
+		}
+		series.Versions = versions
+		series.Other = &other
+	}
+	return series
+}
+
+// versionedHistories drops the samples recorded before the reported version was
+// kept in history, so the version series spans the days it actually knows a
+// version for instead of climbing out of a run of blank days.
+func versionedHistories(histories map[string][]HistorySample) map[string][]HistorySample {
+	out := make(map[string][]HistorySample, len(histories))
+	for id, history := range histories {
+		var kept []HistorySample
+		for _, sample := range history {
+			if sample.Version != "" {
+				kept = append(kept, sample)
+			}
+		}
+		if len(kept) > 0 {
+			out[id] = kept
+		}
+	}
+	return out
+}
+
+// versionLess orders release strings like "3.97" and "4.40" by their numeric
+// components rather than lexically, which would sort "10.02" before "9.99". A
+// build suffix sits next to the release it was built from. Anything that
+// doesn't parse, such as the "unknown" a client with no version compiled in
+// reports, sorts first.
+func versionLess(a, b string) bool {
+	an, aSuffix, aok := versionParts(a)
+	bn, bSuffix, bok := versionParts(b)
+	if aok != bok {
+		return !aok
+	}
+	if !aok {
+		return a < b
+	}
+	for i := 0; i < len(an) && i < len(bn); i++ {
+		if an[i] != bn[i] {
+			return an[i] < bn[i]
+		}
+	}
+	if len(an) != len(bn) {
+		return len(an) < len(bn)
+	}
+	return aSuffix < bSuffix
+}
+
+func versionParts(v string) ([]int, string, bool) {
+	number, suffix, _ := strings.Cut(v, "-")
+	fields := strings.Split(number, ".")
+	parts := make([]int, 0, len(fields))
+	for _, field := range fields {
+		n, err := strconv.Atoi(field)
+		if err != nil {
+			return nil, "", false
+		}
+		parts = append(parts, n)
+	}
+	return parts, suffix, len(parts) > 0
+}

telemetry/server/storage/versions_test.go

@@ -0,0 +1,109 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestVersionSeries(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Upgraded mid-window: its band leaves the old version for the new one.
+	seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -6, -5, -4, -3)
+	seedSamples(s, "upgraded", HistorySample{Version: "4.40"}, -2, -1, 0)
+	// Reported every day on the same version.
+	seedSamples(s, "steady", HistorySample{Version: "4.40"}, -6, -5, -4, -3, -2, -1, 0)
+	// One day of history only: unconfirmed, so it stays out of the stack.
+	seedSamples(s, "oneshot", HistorySample{Version: "4.40"}, -1)
+	// Confirmed, but its samples predate versions being recorded.
+	seedSamples(s, "versionless", HistorySample{}, -13, -12, -11, -10, -9, -8, -7)
+
+	series := s.GetVersionSeries(10, 0)
+
+	// The versionless days are dropped before the axis is built, so the chart
+	// spans the days a version is known for instead of climbing out of blanks.
+	if len(series.Dates) != 7 {
+		t.Fatalf("dates = %v, want the 7 days with versions", series.Dates)
+	}
+	if len(series.Versions) != 2 ||
+		series.Versions[0].Version != "4.39" || series.Versions[1].Version != "4.40" {
+		t.Fatalf("versions = %+v, want 4.39 then 4.40", series.Versions)
+	}
+	if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 1, 1, 0, 0, 0}) {
+		t.Errorf("4.39 = %v, want the upgraded cluster's first four days", got)
+	}
+	if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 1, 1, 2, 2, 2}) {
+		t.Errorf("4.40 = %v, want steady plus upgraded from day 5", got)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("total_clusters = %d, want 2", series.TotalClusters)
+	}
+	if series.Other != nil {
+		t.Errorf("other = %+v, want none without a limit", series.Other)
+	}
+}
+
+func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Stopped reporting past the active window: its own days only.
+	seedSamples(s, "gone", HistorySample{Version: "3.97"}, -14, -13, -12, -11, -10, -9, -8)
+	// Reported every day of the window.
+	seedSamples(s, "daily", HistorySample{Version: "4.40"}, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	// Stopped reporting two days ago: still active, so it holds its version
+	// to the right edge instead of dropping out of the stack.
+	seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -8, -7, -6, -5, -4, -3, -2)
+
+	series := s.GetVersionSeries(10, 0)
+	if len(series.Dates) != 10 {
+		t.Fatalf("dates = %v, want 10 days", series.Dates)
+	}
+	byVersion := map[string][]uint64{}
+	for _, v := range series.Versions {
+		byVersion[v.Version] = v.Clusters
+	}
+	if got := byVersion["3.97"]; !equal(got, []uint64{1, 1, 0, 0, 0, 0, 0, 0, 0, 0}) {
+		t.Errorf("3.97 = %v, want nothing after its last report", got)
+	}
+	if got := byVersion["4.30"]; !equal(got, []uint64{0, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
+		t.Errorf("4.30 = %v, want carried to the right edge", got)
+	}
+	if got := byVersion["4.40"]; !equal(got, []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
+		t.Errorf("4.40 = %v, want 1 every day", got)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("total_clusters = %d, want the 2 clusters still reporting", series.TotalClusters)
+	}
+
+	// Past the limit, the versions the fewest clusters are on are summed into
+	// one band; ties on the last day keep the newer version.
+	series = s.GetVersionSeries(10, 1)
+	if len(series.Versions) != 1 || series.Versions[0].Version != "4.40" {
+		t.Fatalf("limited versions = %+v, want just 4.40", series.Versions)
+	}
+	if series.Other == nil || series.Other.Count != 2 {
+		t.Fatalf("other = %+v, want 2 versions", series.Other)
+	}
+	if !equal(series.Other.Clusters, []uint64{1, 2, 1, 1, 1, 1, 1, 1, 1, 1}) {
+		t.Errorf("other = %v, want 3.97+4.30 summed per day", series.Other.Clusters)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("limit changed total_clusters: %d, want 2", series.TotalClusters)
+	}
+}
+
+func TestVersionLess(t *testing.T) {
+	// Ordered oldest to newest; every pair must compare in this order.
+	ordered := []string{"unknown", "3.97", "4.02", "4.30", "4.30-enterprise", "9.99", "10.02"}
+	for i := 0; i < len(ordered); i++ {
+		for j := i + 1; j < len(ordered); j++ {
+			if !versionLess(ordered[i], ordered[j]) {
+				t.Errorf("versionLess(%q, %q) = false, want true", ordered[i], ordered[j])
+			}
+			if versionLess(ordered[j], ordered[i]) {
+				t.Errorf("versionLess(%q, %q) = true, want false", ordered[j], ordered[i])
+			}
+		}
+	}
+}