snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

telemetry/server/storage/prometheus.go

@@ -0,0 +1,332 @@
+package storage
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+type PrometheusStorage struct {
+	// Prometheus metrics
+	totalClusters     prometheus.Gauge
+	activeClusters    prometheus.Gauge
+	confirmedClusters prometheus.Gauge
+	volumeServerCount *prometheus.GaugeVec
+	totalDiskBytes    *prometheus.GaugeVec
+	totalVolumeCount  *prometheus.GaugeVec
+	filerCount        *prometheus.GaugeVec
+	brokerCount       *prometheus.GaugeVec
+	clusterInfo       *prometheus.GaugeVec
+	telemetryReceived prometheus.Counter
+	reportsSkipped    prometheus.Counter
+
+	// In-memory storage for API endpoints (if needed)
+	mu        sync.RWMutex
+	instances map[string]*telemetryData
+	histories map[string][]HistorySample
+	stats     map[string]interface{}
+	dirty     bool // instances changed since the last successful state save
+}
+
+// telemetryData is an internal struct that includes the received timestamp
+type telemetryData struct {
+	*proto.TelemetryData
+	ReceivedAt time.Time `json:"received_at"`
+}
+
+func NewPrometheusStorage() *PrometheusStorage {
+	return newPrometheusStorage(prometheus.DefaultRegisterer)
+}
+
+func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
+	promauto := promauto.With(reg)
+	return &PrometheusStorage{
+		totalClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_total_clusters",
+			Help: "Total number of unique SeaweedFS clusters (last 30 days)",
+		}),
+		activeClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_active_clusters",
+			Help: "Number of active SeaweedFS clusters (last 7 days)",
+		}),
+		confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_confirmed_clusters",
+			Help: "Active clusters seen on at least 7 distinct days (last 7 days)",
+		}),
+		volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_volume_servers",
+			Help: "Number of volume servers per cluster",
+		}, []string{"cluster_id"}),
+		totalDiskBytes: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_disk_bytes",
+			Help: "Total disk usage in bytes per cluster",
+		}, []string{"cluster_id"}),
+		totalVolumeCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_volume_count",
+			Help: "Total number of volumes per cluster",
+		}, []string{"cluster_id"}),
+		filerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_filer_count",
+			Help: "Number of filer servers per cluster",
+		}, []string{"cluster_id"}),
+		brokerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_broker_count",
+			Help: "Number of broker servers per cluster",
+		}, []string{"cluster_id"}),
+		clusterInfo: promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_cluster_info",
+			Help: "Cluster information (always 1, labels contain metadata)",
+		}, []string{"cluster_id", "version", "os"}),
+		telemetryReceived: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "seaweedfs_telemetry_reports_received_total",
+			Help: "Total number of telemetry reports received",
+		}),
+		reportsSkipped: promauto.NewCounter(prometheus.CounterOpts{
+			Name: "seaweedfs_telemetry_reports_skipped_total",
+			Help: fmt.Sprintf("Reports not kept because the cluster stores less than %d GiB", proto.MinDiskBytes>>30),
+		}),
+		instances: make(map[string]*telemetryData),
+		histories: make(map[string][]HistorySample),
+		stats:     make(map[string]interface{}),
+	}
+}
+
+func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.telemetryReceived.Inc()
+	if data.TotalDiskBytes < proto.MinDiskBytes {
+		s.reportsSkipped.Inc()
+		return nil
+	}
+
+	// Drop the cluster_info series recorded under the previous label set when
+	// a cluster reports back with a different version or OS, so it is not
+	// counted under two versions at once.
+	if prev, ok := s.instances[data.TopologyId]; ok &&
+		(prev.TelemetryData.Version != data.Version || prev.TelemetryData.Os != data.Os) {
+		s.clusterInfo.Delete(infoLabels(prev.TelemetryData))
+	}
+	s.setClusterMetrics(data)
+
+	// Store in memory for API endpoints
+	receivedAt := time.Now().UTC()
+	s.instances[data.TopologyId] = &telemetryData{
+		TelemetryData: data,
+		ReceivedAt:    receivedAt,
+	}
+	s.appendHistory(data, receivedAt)
+	s.dirty = true
+
+	// Update aggregated stats
+	s.updateStats()
+
+	return nil
+}
+
+// setClusterMetrics records a report's values on the Prometheus gauges.
+// Value gauges are keyed by cluster_id only so a cluster's series continues
+// across upgrades; version/os metadata lives on cluster_info (join with
+// `* on(cluster_id) group_left(version, os)`). Callers must hold s.mu.
+func (s *PrometheusStorage) setClusterMetrics(data *proto.TelemetryData) {
+	labels := prometheus.Labels{
+		"cluster_id": data.TopologyId,
+	}
+	s.volumeServerCount.With(labels).Set(float64(data.VolumeServerCount))
+	s.totalDiskBytes.With(labels).Set(float64(data.TotalDiskBytes))
+	s.totalVolumeCount.With(labels).Set(float64(data.TotalVolumeCount))
+	s.filerCount.With(labels).Set(float64(data.FilerCount))
+	s.brokerCount.With(labels).Set(float64(data.BrokerCount))
+	s.clusterInfo.With(infoLabels(data)).Set(1)
+}
+
+func (s *PrometheusStorage) GetStats() (map[string]interface{}, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	// Return cached stats
+	result := make(map[string]interface{})
+	for k, v := range s.stats {
+		result[k] = v
+	}
+	return result, nil
+}
+
+func (s *PrometheusStorage) GetInstances(limit int) ([]*telemetryData, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	var instances []*telemetryData
+	count := 0
+	for _, instance := range s.instances {
+		if count >= limit {
+			break
+		}
+		instances = append(instances, instance)
+		count++
+	}
+
+	return instances, nil
+}
+
+// GetMetrics returns fleet-wide daily totals across confirmed clusters for the
+// last `days` days, in the parallel-array shape the dashboard charts expect.
+// Totals come from the daily histories rather than from s.instances, which holds
+// only each cluster's most recent report and so would credit every cluster to
+// the single day it last reported on.
+func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	histories := s.seriesHistories()
+	axis := newDailySeries(days, histories)
+	activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
+
+	diskUsage := make([]uint64, len(axis.dates))
+	serverCounts := make([]int64, len(axis.dates))
+	for _, history := range histories {
+		if disk, ok := align(axis, history, activeSince, diskBytes); ok {
+			for i, v := range disk {
+				diskUsage[i] += v
+			}
+		}
+		if servers, ok := align(axis, history, activeSince, serverCount); ok {
+			for i, v := range servers {
+				serverCounts[i] += int64(v)
+			}
+		}
+	}
+
+	return map[string]interface{}{
+		"dates":         axis.dates,
+		"server_counts": serverCounts,
+		"disk_usage":    diskUsage,
+	}, nil
+}
+
+func (s *PrometheusStorage) updateStats() {
+	now := time.Now()
+	last7Days := now.AddDate(0, 0, -activeDays)
+	last30Days := now.AddDate(0, 0, -30)
+
+	totalInstances := 0
+	activeInstances := 0
+	confirmedInstances := 0
+	confirmedByDays := make(map[int]int, len(confirmThresholds))
+	for _, threshold := range confirmThresholds {
+		confirmedByDays[threshold] = 0
+	}
+	versionsAll := make(map[string]int)
+	osAll := make(map[string]int)
+	versionsConfirmed := make(map[string]int)
+	osConfirmed := make(map[string]int)
+
+	for _, instance := range s.instances {
+		if instance.ReceivedAt.After(last30Days) {
+			totalInstances++
+		}
+		if instance.ReceivedAt.After(last7Days) {
+			activeInstances++
+			versionsAll[instance.TelemetryData.Version]++
+			osAll[instance.TelemetryData.Os]++
+			// A cluster is confirmed once seen on confirmDays distinct UTC
+			// days (histories hold one sample per day), so short-lived
+			// clusters can't skew the distributions below.
+			daysSeen := len(s.histories[instance.TelemetryData.TopologyId])
+			for _, threshold := range confirmThresholds {
+				if daysSeen >= threshold {
+					confirmedByDays[threshold]++
+				}
+			}
+			if daysSeen >= confirmDays {
+				confirmedInstances++
+				versionsConfirmed[instance.TelemetryData.Version]++
+				osConfirmed[instance.TelemetryData.Os]++
+			}
+		}
+	}
+
+	// Before any cluster has a week of history (fresh server with no
+	// prior state), fall back to all active clusters so the dashboard
+	// distributions aren't empty.
+	versions, osDistribution := versionsConfirmed, osConfirmed
+	if confirmedInstances == 0 {
+		versions, osDistribution = versionsAll, osAll
+	}
+
+	// Update Prometheus gauges
+	s.totalClusters.Set(float64(totalInstances))
+	s.activeClusters.Set(float64(activeInstances))
+	s.confirmedClusters.Set(float64(confirmedInstances))
+
+	// Update cached stats for API
+	s.stats = map[string]interface{}{
+		"total_instances":     totalInstances,
+		"active_instances":    activeInstances,
+		"confirmed_instances": confirmedInstances,
+		"confirmed_by_days":   confirmedByDays,
+		"versions":            versions,
+		"os_distribution":     osDistribution,
+	}
+}
+
+// CleanupOldInstances removes instances older than the specified duration
+func (s *PrometheusStorage) CleanupOldInstances(maxAge time.Duration) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	cutoff := time.Now().Add(-maxAge)
+	for instanceID, instance := range s.instances {
+		if instance.ReceivedAt.Before(cutoff) {
+			delete(s.instances, instanceID)
+			s.deleteClusterMetrics(instance.TelemetryData)
+			s.dirty = true
+		}
+	}
+
+	for id, h := range s.histories {
+		if _, ok := s.instances[id]; !ok {
+			delete(s.histories, id)
+			s.dirty = true
+			continue
+		}
+		i := 0
+		for i < len(h) && time.Unix(h[i].Ts, 0).Before(cutoff) {
+			i++
+		}
+		if i > 0 {
+			s.histories[id] = append([]HistorySample(nil), h[i:]...)
+			s.dirty = true
+		}
+	}
+
+	s.updateStats()
+}
+
+// deleteClusterMetrics removes all gauges stored for the given report's
+// cluster. Callers must hold s.mu.
+func (s *PrometheusStorage) deleteClusterMetrics(data *proto.TelemetryData) {
+	labels := prometheus.Labels{
+		"cluster_id": data.TopologyId,
+	}
+	s.volumeServerCount.Delete(labels)
+	s.totalDiskBytes.Delete(labels)
+	s.totalVolumeCount.Delete(labels)
+	s.filerCount.Delete(labels)
+	s.brokerCount.Delete(labels)
+	s.clusterInfo.Delete(infoLabels(data))
+}
+
+// infoLabels is the full label set used by the cluster_info metric.
+func infoLabels(data *proto.TelemetryData) prometheus.Labels {
+	return prometheus.Labels{
+		"cluster_id": data.TopologyId,
+		"version":    data.Version,
+		"os":         data.Os,
+	}
+}