snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,455 @@
package dash
import (
"context"
"net"
"net/http"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iam"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// Access key status constants
const (
AccessKeyStatusActive = iam.AccessKeyStatusActive
AccessKeyStatusInactive = iam.AccessKeyStatusInactive
)
type AdminData struct {
Username string `json:"username"`
TotalVolumes int `json:"total_volumes"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
VolumeSizeLimitMB uint64 `json:"volume_size_limit_mb"`
MasterNodes []MasterNode `json:"master_nodes"`
VolumeServers []VolumeServer `json:"volume_servers"`
FilerNodes []FilerNode `json:"filer_nodes"`
MessageBrokers []MessageBrokerNode `json:"message_brokers"`
S3Nodes []S3Node `json:"s3_nodes"`
DataCenters []DataCenter `json:"datacenters"`
LastUpdated time.Time `json:"last_updated"`
// EC shard totals for dashboard
TotalEcVolumes int `json:"total_ec_volumes"` // Total number of EC volumes across all servers
TotalEcShards int `json:"total_ec_shards"` // Total number of EC shards across all servers
// TotalMountClients is the number of connected FUSE/VFS mount clients across filers.
TotalMountClients int `json:"total_mount_clients"`
// Trends holds at-a-glance sparklines built from the admin's own recent
// cluster snapshots (no Prometheus required).
Trends DashboardTrends `json:"trends"`
// TierStats breaks volumes and EC shards down by storage tier: local
// disk types plus one entry per remote storage holding tiered volumes.
TierStats []TierStats `json:"tier_stats"`
}
// Object Store Users management structures
type ObjectStoreUser struct {
Username string `json:"username"`
Email string `json:"email"`
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
Permissions []string `json:"permissions"`
PolicyNames []string `json:"policy_names"`
IsStatic bool `json:"is_static"` // loaded from static config file, not editable
}
type ObjectStoreUsersData struct {
Username string `json:"username"`
Users []ObjectStoreUser `json:"users"`
TotalUsers int `json:"total_users"`
HasAnonymousUser bool `json:"has_anonymous_user"`
LastUpdated time.Time `json:"last_updated"`
}
// User management request structures
type CreateUserRequest struct {
Username string `json:"username" binding:"required"`
Email string `json:"email"`
Actions []string `json:"actions"`
GenerateKey bool `json:"generate_key"`
PolicyNames []string `json:"policy_names"`
}
type UpdateUserRequest struct {
Email string `json:"email"`
Actions []string `json:"actions"`
PolicyNames []string `json:"policy_names"`
}
type UpdateUserPoliciesRequest struct {
Actions []string `json:"actions" binding:"required"`
}
type AccessKeyInfo struct {
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
Status string `json:"status"`
CreatedAt time.Time `json:"created_at"`
}
type CreateAccessKeyRequest struct {
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
}
type UpdateAccessKeyStatusRequest struct {
Status string `json:"status" binding:"required"`
}
type UserDetails struct {
Username string `json:"username"`
Email string `json:"email"`
Actions []string `json:"actions"`
PolicyNames []string `json:"policy_names"`
AccessKeys []AccessKeyInfo `json:"access_keys"`
Groups []string `json:"groups"`
}
// RoleReadOnly is the session role assigned to read-only (view-only) admin
// accounts. It matches the value stored by HandleLogin when the read-only
// credentials are used.
const RoleReadOnly = "readonly"
// IsReadOnlyRole reports whether the given admin session role grants only
// view-only access. Any other role (admin, or the empty role used when auth
// is disabled) is treated as non-read-only.
func IsReadOnlyRole(role string) bool {
return role == RoleReadOnly
}
// RedactSecretKey clears the plaintext S3 secret key from an object-store
// user record. The access key (a public identifier) is retained so the
// identity can still be listed; only the reusable secret is removed.
func (u *ObjectStoreUser) RedactSecretKey() {
u.SecretKey = ""
}
// RedactSecretKeys clears the plaintext S3 secret keys from a user's access
// key records. Access key identifiers are retained so the set of keys remains
// visible; only the reusable secrets are removed.
func (d *UserDetails) RedactSecretKeys() {
for i := range d.AccessKeys {
d.AccessKeys[i].SecretKey = ""
}
}
type FilerNode struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
LastUpdated time.Time `json:"last_updated"`
}
type MessageBrokerNode struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
LastUpdated time.Time `json:"last_updated"`
}
type S3Node struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
LastUpdated time.Time `json:"last_updated"`
}
// GetAdminData retrieves admin data as a struct (for reuse by both JSON and HTML handlers)
func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
if username == "" {
username = "admin"
}
// Get cluster topology
topology, err := s.GetClusterTopology()
if err != nil {
glog.Errorf("Failed to get cluster topology: %v", err)
return AdminData{}, err
}
// Get volume servers data with EC shard information
volumeServersData, err := s.GetClusterVolumeServers()
if err != nil {
glog.Errorf("Failed to get cluster volume servers: %v", err)
return AdminData{}, err
}
// Get master nodes status
masterNodes := s.getMasterNodesStatus()
// Get filer nodes status
filerNodes := s.getFilerNodesStatus()
// Get message broker nodes status
messageBrokers := s.getMessageBrokerNodesStatus()
// Get S3 nodes status
s3Nodes := s.getS3NodesStatus()
// Get volume size limit from master configuration
var volumeSizeLimitMB uint64 = 30000 // Default to 30GB
err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
volumeSizeLimitMB = uint64(resp.VolumeSizeLimitMB)
return nil
})
if err != nil {
glog.Warningf("Failed to get volume size limit from master: %v", err)
// Keep default value on error
}
// Calculate EC shard totals
var totalEcVolumes, totalEcShards int
ecVolumeSet := make(map[uint32]bool) // To avoid counting the same EC volume multiple times
for _, vs := range volumeServersData.VolumeServers {
totalEcShards += vs.EcShards
// Count unique EC volumes across all servers
for _, ecInfo := range vs.EcShardDetails {
ecVolumeSet[ecInfo.VolumeID] = true
}
}
totalEcVolumes = len(ecVolumeSet)
// Count connected FUSE/VFS mount clients (best-effort; don't fail the dashboard)
totalMountClients := 0
if mountData, mountErr := s.GetMountClients(); mountErr == nil {
totalMountClients = mountData.TotalMountClients
}
// Prepare admin data
adminData := AdminData{
Username: username,
TotalVolumes: topology.TotalVolumes,
TotalChunks: topology.TotalChunks,
TotalSize: topology.TotalSize,
VolumeSizeLimitMB: volumeSizeLimitMB,
MasterNodes: masterNodes,
VolumeServers: volumeServersData.VolumeServers,
FilerNodes: filerNodes,
MessageBrokers: messageBrokers,
S3Nodes: s3Nodes,
DataCenters: topology.DataCenters,
LastUpdated: topology.UpdatedAt,
TotalEcVolumes: totalEcVolumes,
TotalEcShards: totalEcShards,
TotalMountClients: totalMountClients,
Trends: s.GetDashboardTrends(),
TierStats: topology.TierStats,
}
return adminData, nil
}
// ShowAdmin displays the main admin page (now uses GetAdminData)
func (s *AdminServer) ShowAdmin(w http.ResponseWriter, r *http.Request) {
username := UsernameFromContext(r.Context())
adminData, err := s.GetAdminData(username)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get admin data: "+err.Error())
return
}
// Return JSON for API calls
writeJSON(w, http.StatusOK, adminData)
}
// ShowOverview displays cluster overview
func (s *AdminServer) ShowOverview(w http.ResponseWriter, r *http.Request) {
topology, err := s.GetClusterTopology()
if err != nil {
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, topology)
}
// getMasterNodesStatus returns the full set of master nodes in the cluster.
// It prefers the authoritative raft membership (RaftListClusterServers) and
// falls back to the currently-connected master if the raft call fails, so the
// dashboard never shows an empty list.
func (s *AdminServer) getMasterNodesStatus() []MasterNode {
masterMap := make(map[string]MasterNode)
raftCallSucceeded := false
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
resp, err := client.RaftListClusterServers(ctx, &master_pb.RaftListClusterServersRequest{})
if err != nil {
return err
}
raftCallSucceeded = true
for _, server := range resp.ClusterServers {
// pb.GrpcAddressToServerAddress calls glog.Fatalf on a parse
// error, so pre-validate the raft address with net.SplitHostPort
// and skip malformed entries instead of taking the process down.
if _, _, splitErr := net.SplitHostPort(server.Address); splitErr != nil {
glog.Warningf("skip master with invalid raft address %q: %v", server.Address, splitErr)
continue
}
httpAddress := pb.GrpcAddressToServerAddress(server.Address)
masterMap[httpAddress] = MasterNode{
Address: httpAddress,
IsLeader: server.IsLeader,
}
}
return nil
})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to list raft cluster masters from %s: %v", currentMaster, err)
}
if len(masterMap) == 0 {
currentMaster := s.masterClient.GetMaster(context.Background())
if currentMaster != "" {
addr := pb.ServerAddress(currentMaster).ToHttpAddress()
// A successful empty raft response means raft is not initialized
// (standalone/non-raft cluster); the only master IS the leader.
// A failed RPC means connectivity issue; do not claim leadership.
masterMap[addr] = MasterNode{
Address: addr,
IsLeader: raftCallSucceeded,
}
}
}
masterNodes := make([]MasterNode, 0, len(masterMap))
for _, m := range masterMap {
masterNodes = append(masterNodes, m)
}
sort.Slice(masterNodes, func(i, j int) bool {
return masterNodes[i].Address < masterNodes[j].Address
})
return masterNodes
}
// getFilerNodesStatus checks status of all filer nodes using master's ListClusterNodes
func (s *AdminServer) getFilerNodesStatus() []FilerNode {
var filerNodes []FilerNode
// Get filer nodes from master using ListClusterNodes
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), s.listClusterNodesRequest(cluster.FilerType))
if err != nil {
return err
}
// Process each filer node
for _, node := range resp.ClusterNodes {
filerNodes = append(filerNodes, FilerNode{
Address: pb.ServerAddress(node.Address).ToHttpAddress(),
DataCenter: node.DataCenter,
Rack: node.Rack,
LastUpdated: time.Now(),
})
}
return nil
})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to get filer nodes from master %s: %v", currentMaster, err)
// Return empty list if we can't get filer info from master
return []FilerNode{}
}
// Sort filer nodes by address for consistent ordering on page refresh
sort.Slice(filerNodes, func(i, j int) bool {
return filerNodes[i].Address < filerNodes[j].Address
})
return filerNodes
}
// getMessageBrokerNodesStatus checks status of all message broker nodes using master's ListClusterNodes
func (s *AdminServer) getMessageBrokerNodesStatus() []MessageBrokerNode {
var messageBrokers []MessageBrokerNode
// Get message broker nodes from master using ListClusterNodes
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), s.listClusterNodesRequest(cluster.BrokerType))
if err != nil {
return err
}
// Process each message broker node
for _, node := range resp.ClusterNodes {
messageBrokers = append(messageBrokers, MessageBrokerNode{
Address: node.Address,
DataCenter: node.DataCenter,
Rack: node.Rack,
LastUpdated: time.Now(),
})
}
return nil
})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to get message broker nodes from master %s: %v", currentMaster, err)
// Return empty list if we can't get broker info from master
return []MessageBrokerNode{}
}
// Sort message broker nodes by address for consistent ordering on page refresh
sort.Slice(messageBrokers, func(i, j int) bool {
return messageBrokers[i].Address < messageBrokers[j].Address
})
return messageBrokers
}
// getS3NodesStatus checks status of all S3 nodes using master's ListClusterNodes
func (s *AdminServer) getS3NodesStatus() []S3Node {
var s3Nodes []S3Node
// Get S3 nodes from master using ListClusterNodes
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), s.listClusterNodesRequest(cluster.S3Type))
if err != nil {
return err
}
// Process each S3 node
for _, node := range resp.ClusterNodes {
s3Nodes = append(s3Nodes, S3Node{
Address: pb.ServerAddress(node.Address).ToHttpAddress(),
DataCenter: node.DataCenter,
LastUpdated: time.Now(),
})
}
return nil
})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to get S3 nodes from master %s: %v", currentMaster, err)
// Return empty list if we can't get S3 info from master
return []S3Node{}
}
// Sort S3 nodes by address for consistent ordering on page refresh
sort.Slice(s3Nodes, func(i, j int) bool {
return s3Nodes[i].Address < s3Nodes[j].Address
})
return s3Nodes
}

View File

@@ -0,0 +1,195 @@
package dash
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const (
adminLockName = cluster.AdminShellLockName
adminLockClientName = "admin-plugin"
)
const (
// A competing shell client polls the master once a second, while a local
// re-acquire would win immediately. Leave the lock free for slightly more
// than one poll interval so a waiting operator gets it first.
defaultAdminLockYieldWindow = 2 * time.Second
// After the lock has been held continuously for this long (overlapping
// reference-counted holds can keep it held indefinitely), new Acquire
// calls wait for a full release so shell clients are not starved.
defaultAdminLockFairnessWindow = time.Minute
)
// adminLocker is the subset of exclusive_locks.ExclusiveLocker the manager uses.
type adminLocker interface {
RequestLock(clientName string)
ReleaseLock()
SetMessage(message string)
}
// AdminLockManager coordinates exclusive admin locks with reference counting.
// It is safe for concurrent use.
type AdminLockManager struct {
locker adminLocker
clientName string
yieldWindow time.Duration
fairnessWindow time.Duration
mu sync.Mutex
cond *sync.Cond
acquiring bool
releasing bool
holdCount int
heldSince time.Time
lastAcquiredAt time.Time
lastReleasedAt time.Time
waitingSince time.Time
waitingReason string
currentReason string
}
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
if masterClient == nil {
return nil
}
if clientName == "" {
clientName = adminLockClientName
}
manager := &AdminLockManager{
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
clientName: clientName,
yieldWindow: defaultAdminLockYieldWindow,
fairnessWindow: defaultAdminLockFairnessWindow,
}
manager.cond = sync.NewCond(&manager.mu)
return manager
}
// Acquire takes the shared cluster lock, reference counted so overlapping
// admin-side holders piggyback on one lease. It must not be called while the
// caller's own call stack already holds the lock: once the fairness window
// makes new acquires wait for a full release, a nested Acquire deadlocks.
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
if m == nil || m.locker == nil {
return func() {}, nil
}
m.mu.Lock()
if reason != "" {
m.locker.SetMessage(reason)
m.currentReason = reason
}
for m.acquiring || m.releasing || (m.holdCount > 0 && time.Since(m.heldSince) > m.fairnessWindow) {
m.cond.Wait()
}
if m.holdCount == 0 {
m.acquiring = true
m.waitingSince = time.Now().UTC()
m.waitingReason = reason
yield := time.Duration(0)
if !m.lastReleasedAt.IsZero() {
if since := time.Since(m.lastReleasedAt); since < m.yieldWindow {
yield = m.yieldWindow - since
}
}
m.mu.Unlock()
if yield > 0 {
time.Sleep(yield)
}
m.locker.RequestLock(m.clientName)
m.mu.Lock()
m.acquiring = false
m.holdCount = 1
m.heldSince = time.Now()
m.lastAcquiredAt = time.Now().UTC()
m.waitingSince = time.Time{}
m.waitingReason = ""
m.cond.Broadcast()
m.mu.Unlock()
return m.Release, nil
}
m.holdCount++
if reason != "" {
m.currentReason = reason
}
m.mu.Unlock()
return m.Release, nil
}
func (m *AdminLockManager) Release() {
if m == nil || m.locker == nil {
return
}
m.mu.Lock()
if m.holdCount <= 0 {
m.mu.Unlock()
return
}
m.holdCount--
shouldRelease := m.holdCount == 0
if shouldRelease {
// Block new acquires until ReleaseLock completes: a RequestLock
// issued while the locker still considers itself locked would no-op
// and leave a hold with no live lease.
m.releasing = true
}
m.mu.Unlock()
if shouldRelease {
m.locker.ReleaseLock()
m.mu.Lock()
m.releasing = false
m.lastReleasedAt = time.Now().UTC()
m.currentReason = ""
m.cond.Broadcast()
m.mu.Unlock()
}
}
type LockStatus struct {
Held bool `json:"held"`
HoldCount int `json:"hold_count"`
Acquiring bool `json:"acquiring"`
Message string `json:"message,omitempty"`
WaitingReason string `json:"waiting_reason,omitempty"`
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
WaitingSince *time.Time `json:"waiting_since,omitempty"`
}
func (m *AdminLockManager) Status() LockStatus {
if m == nil {
return LockStatus{}
}
m.mu.Lock()
defer m.mu.Unlock()
status := LockStatus{
Held: m.holdCount > 0,
HoldCount: m.holdCount,
Acquiring: m.acquiring,
Message: m.currentReason,
WaitingReason: m.waitingReason,
}
if !m.lastAcquiredAt.IsZero() {
at := m.lastAcquiredAt
status.LastAcquiredAt = &at
}
if !m.lastReleasedAt.IsZero() {
at := m.lastReleasedAt
status.LastReleasedAt = &at
}
if !m.waitingSince.IsZero() {
at := m.waitingSince
status.WaitingSince = &at
}
return status
}

View File

@@ -0,0 +1,156 @@
package dash
import (
"sync"
"testing"
"time"
)
type fakeAdminLocker struct {
releaseDelay time.Duration
mu sync.Mutex
requests []time.Time
releases []time.Time // recorded when ReleaseLock returns
}
func (f *fakeAdminLocker) RequestLock(clientName string) {
f.mu.Lock()
f.requests = append(f.requests, time.Now())
f.mu.Unlock()
}
func (f *fakeAdminLocker) ReleaseLock() {
if f.releaseDelay > 0 {
time.Sleep(f.releaseDelay)
}
f.mu.Lock()
f.releases = append(f.releases, time.Now())
f.mu.Unlock()
}
func (f *fakeAdminLocker) SetMessage(message string) {}
func (f *fakeAdminLocker) counts() (int, int) {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.requests), len(f.releases)
}
func newTestLockManager(locker adminLocker, yieldWindow, fairnessWindow time.Duration) *AdminLockManager {
m := &AdminLockManager{
locker: locker,
clientName: "test",
yieldWindow: yieldWindow,
fairnessWindow: fairnessWindow,
}
m.cond = sync.NewCond(&m.mu)
return m
}
func TestAdminLockManagerRefCountsOverlappingHolds(t *testing.T) {
t.Parallel()
locker := &fakeAdminLocker{}
m := newTestLockManager(locker, defaultAdminLockYieldWindow, defaultAdminLockFairnessWindow)
release1, _ := m.Acquire("a")
release2, _ := m.Acquire("b")
if requests, releases := locker.counts(); requests != 1 || releases != 0 {
t.Fatalf("expected one lease request while overlapping, got requests=%d releases=%d", requests, releases)
}
release1()
if _, releases := locker.counts(); releases != 0 {
t.Fatalf("lock released while still held by another caller")
}
release2()
if _, releases := locker.counts(); releases != 1 {
t.Fatalf("expected one release after last holder, got %d", releases)
}
}
func TestAdminLockManagerYieldsBeforeReacquire(t *testing.T) {
t.Parallel()
locker := &fakeAdminLocker{}
m := newTestLockManager(locker, 100*time.Millisecond, defaultAdminLockFairnessWindow)
release, _ := m.Acquire("first")
release()
release, _ = m.Acquire("second")
release()
locker.mu.Lock()
gap := locker.requests[1].Sub(locker.releases[0])
locker.mu.Unlock()
if gap < 90*time.Millisecond {
t.Fatalf("re-acquire did not yield to competing clients: gap=%v", gap)
}
}
func TestAdminLockManagerFairnessWindowForcesFullRelease(t *testing.T) {
t.Parallel()
locker := &fakeAdminLocker{}
m := newTestLockManager(locker, 20*time.Millisecond, 50*time.Millisecond)
release1, _ := m.Acquire("long-hold")
m.mu.Lock()
m.heldSince = time.Now().Add(-time.Second)
m.mu.Unlock()
acquired := make(chan func(), 1)
go func() {
release2, _ := m.Acquire("late-joiner")
acquired <- release2
}()
select {
case <-acquired:
t.Fatalf("late acquire joined a hold older than the fairness window")
case <-time.After(50 * time.Millisecond):
}
release1()
select {
case release2 := <-acquired:
release2()
case <-time.After(time.Second):
t.Fatalf("late acquire did not proceed after full release")
}
if requests, releases := locker.counts(); requests != 2 || releases != 2 {
t.Fatalf("expected a full release/re-acquire cycle, got requests=%d releases=%d", requests, releases)
}
}
func TestAdminLockManagerAcquireWaitsForReleaseInFlight(t *testing.T) {
t.Parallel()
locker := &fakeAdminLocker{releaseDelay: 50 * time.Millisecond}
m := newTestLockManager(locker, time.Millisecond, defaultAdminLockFairnessWindow)
release1, _ := m.Acquire("first")
releaseDone := make(chan struct{})
go func() {
release1()
close(releaseDone)
}()
// Let the release enter the slow ReleaseLock before re-acquiring.
time.Sleep(10 * time.Millisecond)
release2, _ := m.Acquire("second")
<-releaseDone
release2()
locker.mu.Lock()
defer locker.mu.Unlock()
if len(locker.requests) != 2 || len(locker.releases) != 2 {
t.Fatalf("unexpected lease traffic: requests=%d releases=%d", len(locker.requests), len(locker.releases))
}
if locker.requests[1].Before(locker.releases[0]) {
t.Fatalf("RequestLock ran while ReleaseLock was still in flight")
}
}

View File

@@ -0,0 +1,61 @@
package dash
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const adminPresenceClientName = "admin-server"
type adminPresenceLock struct {
locker *exclusive_locks.ExclusiveLocker
stopCh chan struct{}
}
func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
if masterClient == nil {
return nil
}
return &adminPresenceLock{
locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
stopCh: make(chan struct{}),
}
}
func (l *adminPresenceLock) Start() {
if l == nil || l.locker == nil {
return
}
l.locker.SetMessage("admin server connected")
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
if !l.locker.IsLocked() {
l.locker.RequestLock(adminPresenceClientName)
}
select {
case <-l.stopCh:
return
case <-ticker.C:
}
}
}()
}
func (l *adminPresenceLock) Stop() {
if l == nil {
return
}
select {
case <-l.stopCh:
default:
close(l.stopCh)
}
if l.locker != nil {
l.locker.ReleaseLock()
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,30 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/cluster"
)
func TestListClusterNodesRequestIncludesFilerGroup(t *testing.T) {
server := &AdminServer{filerGroup: "tenant-a"}
req := server.listClusterNodesRequest(cluster.FilerType)
if req.ClientType != cluster.FilerType {
t.Fatalf("ClientType = %q, want %q", req.ClientType, cluster.FilerType)
}
if req.FilerGroup != "tenant-a" {
t.Fatalf("FilerGroup = %q, want %q", req.FilerGroup, "tenant-a")
}
}
func TestListClusterNodesRequestKeepsDefaultFilerGroupEmpty(t *testing.T) {
server := &AdminServer{}
req := server.listClusterNodesRequest(cluster.FilerType)
if req.FilerGroup != "" {
t.Fatalf("FilerGroup = %q, want empty", req.FilerGroup)
}
}

View File

@@ -0,0 +1,77 @@
// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
package dash
import "testing"
func TestCleanMaintenanceScript(t *testing.T) {
tests := []struct {
name string
input string
expected string
}{
{
name: "empty",
input: "",
expected: "",
},
{
name: "only lock unlock",
input: " lock\n unlock\n",
expected: "",
},
{
name: "strips lock and unlock",
input: " lock\n ec.balance -apply\n volume.fix.replication -apply\n unlock\n",
expected: "ec.balance -apply\nvolume.fix.replication -apply",
},
{
name: "case insensitive lock",
input: "Lock\nec.balance -apply\nUNLOCK",
expected: "ec.balance -apply",
},
{
name: "preserves comments removal",
input: "lock\n# a comment\nec.balance -apply\nunlock",
expected: "ec.balance -apply",
},
{
name: "no lock unlock present",
input: "ec.balance -apply\nvolume.fix.replication -apply",
expected: "ec.balance -apply\nvolume.fix.replication -apply",
},
{
name: "windows line endings",
input: "lock\r\nec.balance -apply\r\nunlock\r\n",
expected: "ec.balance -apply",
},
{
name: "lock with inline comment",
input: "lock # migration\nec.balance -apply\nunlock # done",
expected: "ec.balance -apply",
},
{
name: "command with inline comment preserved",
input: "lock\nec.balance -apply # rebalance shards\nunlock",
expected: "ec.balance -apply",
},
{
name: "only inline comment after stripping",
input: "# full line comment\n # indented comment\n",
expected: "",
},
{
name: "typical master default",
input: "\n lock\n ec.encode -fullPercent=95 -quietFor=1h\n ec.rebuild -apply\n ec.balance -apply\n fs.log.purge -daysAgo=7\n volume.deleteEmpty -quietFor=24h -apply\n volume.balance -apply\n volume.fix.replication -apply\n s3.clean.uploads -timeAgo=24h\n unlock\n",
expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := cleanMaintenanceScript(tt.input)
if got != tt.expected {
t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
}
})
}
}

View File

@@ -0,0 +1,96 @@
package dash
import (
"crypto/subtle"
"net/http"
"github.com/gorilla/sessions"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// ShowLogin displays the login page.
func (s *AdminServer) ShowLogin(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, P(r.Context(), "/login"), http.StatusSeeOther)
}
// HandleLogin handles login form submission.
func (s *AdminServer) HandleLogin(store sessions.Store, adminUser, adminPassword, readOnlyUser, readOnlyPassword string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
prefix := URLPrefixFromContext(r.Context())
if err := r.ParseForm(); err != nil {
http.Redirect(w, r, prefix+"/login?error=Invalid form submission", http.StatusSeeOther)
return
}
session, err := store.Get(r, sessionName)
if err != nil {
http.Redirect(w, r, prefix+"/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
return
}
if err := ValidateSessionCSRFToken(session, r); err != nil {
http.Redirect(w, r, prefix+"/login?error=Invalid CSRF token", http.StatusSeeOther)
return
}
loginUsername := r.FormValue("username")
loginPassword := r.FormValue("password")
var role string
var authenticated bool
// Check admin credentials.
if adminPassword != "" && loginUsername == adminUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(adminPassword)) == 1 {
role = "admin"
authenticated = true
} else if readOnlyPassword != "" && loginUsername == readOnlyUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(readOnlyPassword)) == 1 {
// Check read-only credentials.
role = RoleReadOnly
authenticated = true
}
if authenticated {
for key := range session.Values {
delete(session.Values, key)
}
session.Values["authenticated"] = true
session.Values["username"] = loginUsername
session.Values["role"] = role
csrfToken, err := generateCSRFToken()
if err != nil {
http.Redirect(w, r, prefix+"/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
return
}
session.Values[sessionCSRFTokenKey] = csrfToken
if err := session.Save(r, w); err != nil {
// Log the detailed error server-side for diagnostics.
glog.Errorf("Failed to save session for user %s: %v", loginUsername, err)
http.Redirect(w, r, prefix+"/login?error=Unable to create session. Please try again or contact administrator.", http.StatusSeeOther)
return
}
http.Redirect(w, r, prefix+"/admin", http.StatusSeeOther)
return
}
// Authentication failed.
http.Redirect(w, r, prefix+"/login?error=Invalid credentials", http.StatusSeeOther)
}
}
// HandleLogout handles user logout.
func (s *AdminServer) HandleLogout(store sessions.Store, w http.ResponseWriter, r *http.Request) {
prefix := URLPrefixFromContext(r.Context())
session, err := store.Get(r, sessionName)
if err != nil {
http.Redirect(w, r, prefix+"/login", http.StatusSeeOther)
return
}
for key := range session.Values {
delete(session.Values, key)
}
session.Options.MaxAge = -1
if err := session.Save(r, w); err != nil {
glog.Warningf("Failed to save session during logout: %v", err)
}
http.Redirect(w, r, prefix+"/login", http.StatusSeeOther)
}

View File

@@ -0,0 +1,350 @@
package dash
import (
"errors"
"fmt"
"net/http"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
)
func minimalLifecycleRule() BucketLifecycleRule {
return BucketLifecycleRule{
ID: "rule-1",
Status: s3lifecycle.StatusEnabled,
ExpirationDays: 30,
}
}
func TestValidateBucketLifecycleRules_Valid(t *testing.T) {
if err := validateBucketLifecycleRules([]BucketLifecycleRule{minimalLifecycleRule()}); err != nil {
t.Fatalf("expected valid rule set to pass, got: %v", err)
}
}
func TestValidateBucketLifecycleRules_Empty(t *testing.T) {
if err := validateBucketLifecycleRules(nil); err != nil {
t.Fatalf("expected empty rule set to pass, got: %v", err)
}
}
func TestValidateBucketLifecycleRules_TooManyRules(t *testing.T) {
rules := make([]BucketLifecycleRule, MaxBucketLifecycleRules+1)
for i := range rules {
rules[i] = BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, ExpirationDays: 30}
}
if err := validateBucketLifecycleRules(rules); err == nil {
t.Fatal("expected error for exceeding the rule count cap")
}
}
func TestValidateBucketLifecycleRules_DuplicateIDs(t *testing.T) {
rule := minimalLifecycleRule()
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule, rule}); err == nil {
t.Fatal("expected error for duplicate rule IDs")
}
}
func TestValidateBucketLifecycleRules_IDTooLong(t *testing.T) {
rule := minimalLifecycleRule()
longID := ""
for i := 0; i < MaxBucketLifecycleRuleIDLength+1; i++ {
longID += "a"
}
rule.ID = longID
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for an ID longer than the cap")
}
}
func TestValidateBucketLifecycleRules_InvalidStatus(t *testing.T) {
rule := minimalLifecycleRule()
rule.Status = "sort-of-enabled"
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for an invalid status")
}
}
func TestValidateBucketLifecycleRules_ExpirationDaysAndDateMutuallyExclusive(t *testing.T) {
rule := minimalLifecycleRule()
rule.ExpirationDate = "2030-01-01"
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error when both expiration_days and expiration_date are set")
}
}
func TestValidateBucketLifecycleRules_DeleteMarkerWithExpirationDaysRejected(t *testing.T) {
// AWS forbids combining ExpiredObjectDeleteMarker with Days/Date in the
// same Expiration element; ruleFromCanonical would otherwise emit both
// on the same <Expiration>.
rule := minimalLifecycleRule()
rule.ExpiredObjectDeleteMarker = true
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error when expired_object_delete_marker is combined with expiration_days")
}
}
func TestValidateBucketLifecycleRules_DeleteMarkerWithExpirationDateRejected(t *testing.T) {
rule := BucketLifecycleRule{
Status: s3lifecycle.StatusEnabled,
ExpirationDate: "2030-01-01",
ExpiredObjectDeleteMarker: true,
}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error when expired_object_delete_marker is combined with expiration_date")
}
}
func TestValidateBucketLifecycleRules_DeleteMarkerAlone(t *testing.T) {
rule := BucketLifecycleRule{
Status: s3lifecycle.StatusEnabled,
ExpiredObjectDeleteMarker: true,
}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err != nil {
t.Fatalf("expected standalone expired_object_delete_marker to be accepted, got: %v", err)
}
}
func TestValidateBucketLifecycleRules_InvalidExpirationDate(t *testing.T) {
rule := BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, ExpirationDate: "01/01/2030"}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for a malformed expiration_date")
}
}
func TestValidateBucketLifecycleRules_NegativeExpirationDays(t *testing.T) {
rule := minimalLifecycleRule()
rule.ExpirationDays = -1
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for negative expiration_days")
}
}
// TestValidateBucketLifecycleRules_StandaloneNewerNoncurrentVersions guards
// against re-adding a NoncurrentVersionExpirationDays requirement:
// s3lifecycle.RuleActionKinds recognizes a standalone NewerNoncurrentVersions
// (no days) as ActionKindNewerNoncurrent, a valid count-only retention rule
// the S3 API already accepts and stores.
func TestValidateBucketLifecycleRules_StandaloneNewerNoncurrentVersions(t *testing.T) {
rule := BucketLifecycleRule{
Status: s3lifecycle.StatusEnabled,
NewerNoncurrentVersions: 2,
}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err != nil {
t.Fatalf("expected standalone newer_noncurrent_versions to be accepted, got: %v", err)
}
}
func TestValidateBucketLifecycleRules_NegativeAbortMultipartDays(t *testing.T) {
rule := minimalLifecycleRule()
rule.AbortMultipartDays = -5
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for negative abort_multipart_days")
}
}
func TestValidateBucketLifecycleRules_SizeGreaterThanNotLessThanSizeLessThan(t *testing.T) {
rule := minimalLifecycleRule()
rule.SizeGreaterThan = 2048
rule.SizeLessThan = 1024
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error when size_greater_than >= size_less_than")
}
}
func TestValidateBucketLifecycleRules_NegativeSizeBounds(t *testing.T) {
rule := minimalLifecycleRule()
rule.SizeGreaterThan = -1
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for a negative size_greater_than")
}
}
func TestValidateBucketLifecycleRules_EmptyTagKey(t *testing.T) {
rule := minimalLifecycleRule()
rule.Tags = map[string]string{"": "value"}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for an empty tag key")
}
}
func TestValidateBucketLifecycleRules_EmptyTagValue(t *testing.T) {
rule := minimalLifecycleRule()
rule.Tags = map[string]string{"env": " "}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for a blank tag value")
}
}
func TestValidateBucketLifecycleRules_NoAction(t *testing.T) {
rule := BucketLifecycleRule{Status: s3lifecycle.StatusEnabled, Prefix: "logs/"}
if err := validateBucketLifecycleRules([]BucketLifecycleRule{rule}); err == nil {
t.Fatal("expected error for a rule with no action")
}
}
func TestFromBucketLifecycleRule_MapsAllFields(t *testing.T) {
rule := BucketLifecycleRule{
ID: "rule-1",
Status: s3lifecycle.StatusEnabled,
Prefix: "logs/",
Tags: map[string]string{"env": "dev"},
SizeGreaterThan: 1024,
SizeLessThan: 2048,
ExpirationDate: "2030-01-15",
ExpiredObjectDeleteMarker: true,
NoncurrentVersionExpirationDays: 30,
NewerNoncurrentVersions: 2,
AbortMultipartDays: 7,
}
out, err := fromBucketLifecycleRule(rule)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if out.ID != rule.ID {
t.Errorf("ID = %q, want %q", out.ID, rule.ID)
}
if out.Status != rule.Status {
t.Errorf("Status = %q, want %q", out.Status, rule.Status)
}
if out.Prefix != rule.Prefix {
t.Errorf("Prefix = %q, want %q", out.Prefix, rule.Prefix)
}
if len(out.FilterTags) != 1 || out.FilterTags["env"] != "dev" {
t.Errorf("FilterTags = %v, want {env: dev}", out.FilterTags)
}
if out.FilterSizeGreaterThan != rule.SizeGreaterThan {
t.Errorf("FilterSizeGreaterThan = %d, want %d", out.FilterSizeGreaterThan, rule.SizeGreaterThan)
}
if out.FilterSizeLessThan != rule.SizeLessThan {
t.Errorf("FilterSizeLessThan = %d, want %d", out.FilterSizeLessThan, rule.SizeLessThan)
}
wantDate, _ := time.Parse(time.DateOnly, rule.ExpirationDate)
if !out.ExpirationDate.Equal(wantDate) {
t.Errorf("ExpirationDate = %v, want %v", out.ExpirationDate, wantDate)
}
if !out.ExpiredObjectDeleteMarker {
t.Error("expected ExpiredObjectDeleteMarker=true")
}
if out.NoncurrentVersionExpirationDays != rule.NoncurrentVersionExpirationDays {
t.Errorf("NoncurrentVersionExpirationDays = %d, want %d", out.NoncurrentVersionExpirationDays, rule.NoncurrentVersionExpirationDays)
}
if out.NewerNoncurrentVersions != rule.NewerNoncurrentVersions {
t.Errorf("NewerNoncurrentVersions = %d, want %d", out.NewerNoncurrentVersions, rule.NewerNoncurrentVersions)
}
if out.AbortMPUDaysAfterInitiation != rule.AbortMultipartDays {
t.Errorf("AbortMPUDaysAfterInitiation = %d, want %d", out.AbortMPUDaysAfterInitiation, rule.AbortMultipartDays)
}
}
func TestFromBucketLifecycleRule_EmptyExpirationDate(t *testing.T) {
rule := minimalLifecycleRule()
rule.ExpirationDate = ""
out, err := fromBucketLifecycleRule(rule)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !out.ExpirationDate.IsZero() {
t.Errorf("expected zero ExpirationDate, got %v", out.ExpirationDate)
}
}
func TestFromBucketLifecycleRule_InvalidExpirationDate(t *testing.T) {
rule := minimalLifecycleRule()
rule.ExpirationDate = "not-a-date"
if _, err := fromBucketLifecycleRule(rule); err == nil {
t.Fatal("expected error for a malformed expiration date")
}
}
func TestBucketLifecycleMutation_SetsXML(t *testing.T) {
m := bucketLifecycleMutation("/buckets", "mybucket", []byte("<LifecycleConfiguration/>"))
if m.Type != filer_pb.ObjectMutation_PATCH_EXTENDED {
t.Fatalf("expected a PATCH_EXTENDED mutation, got %v", m.Type)
}
if m.Directory != "/buckets" || m.Name != "mybucket" {
t.Fatalf("expected the mutation to target /buckets/mybucket, got %s/%s", m.Directory, m.Name)
}
if got := string(m.SetExtended[scheduler.BucketLifecycleConfigurationXMLKey]); got != "<LifecycleConfiguration/>" {
t.Fatalf("expected the XML key to carry the marshaled config, got %q", got)
}
if len(m.DeleteExtended) != 0 {
t.Fatalf("expected no key deletions when saving rules, got %v", m.DeleteExtended)
}
}
func TestBucketLifecycleMutation_ClearsBothKeys(t *testing.T) {
m := bucketLifecycleMutation("/buckets", "mybucket", nil)
if len(m.SetExtended) != 0 {
t.Fatalf("expected no key writes when clearing, got %v", m.SetExtended)
}
want := map[string]bool{
scheduler.BucketLifecycleConfigurationXMLKey: true,
scheduler.BucketLifecycleTransitionMinimumObjectSizeKey: true,
}
if len(m.DeleteExtended) != len(want) {
t.Fatalf("expected both lifecycle keys to be cleared, got %v", m.DeleteExtended)
}
for _, k := range m.DeleteExtended {
if !want[k] {
t.Fatalf("unexpected key cleared: %s", k)
}
}
}
// A whole-entry write would have carried the rest of the bucket entry with it;
// the patch must name only the keys it owns, so a concurrent owner or quota
// change survives.
func TestBucketLifecycleMutation_TouchesOnlyLifecycleKeys(t *testing.T) {
for _, m := range []*filer_pb.ObjectMutation{
bucketLifecycleMutation("/buckets", "mybucket", []byte("<LifecycleConfiguration/>")),
bucketLifecycleMutation("/buckets", "mybucket", nil),
} {
if m.Entry != nil {
t.Fatal("expected the mutation to carry no entry snapshot")
}
if m.SetContent {
t.Fatal("expected the mutation to leave entry content alone")
}
for k := range m.SetExtended {
if k != scheduler.BucketLifecycleConfigurationXMLKey {
t.Fatalf("unexpected key written: %s", k)
}
}
}
}
func TestSetBucketLifecycle_RejectsOversizedConfiguration(t *testing.T) {
// A single rule ID long enough to blow the cap: this must fail before any
// filer call, which is what makes it testable without one.
rule := minimalLifecycleRule()
rule.ID = strings.Repeat("x", scheduler.MaxBucketLifecycleConfigurationSize+1)
err := (&AdminServer{}).SetBucketLifecycle("mybucket", []BucketLifecycleRule{rule})
if err == nil {
t.Fatal("expected an oversized lifecycle configuration to be rejected")
}
if !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("expected a size-limit error, got: %v", err)
}
}
func TestBucketLifecycleErrorStatus(t *testing.T) {
if got := bucketLifecycleErrorStatus(fmt.Errorf("%w: mybucket", ErrBucketNotFound)); got != http.StatusNotFound {
t.Fatalf("expected a missing bucket to map to 404, got %d", got)
}
if got := bucketLifecycleErrorStatus(errors.New("filer unreachable")); got != http.StatusInternalServerError {
t.Fatalf("expected an unrelated failure to stay 500, got %d", got)
}
}

View File

@@ -0,0 +1,803 @@
package dash
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
)
// MaxBucketLifecycleRules mirrors AWS S3's limit of 1000 lifecycle rules per bucket.
const MaxBucketLifecycleRules = 1000
// MaxBucketLifecycleRuleIDLength mirrors the S3 API's limit on a lifecycle rule's <ID>.
const MaxBucketLifecycleRuleIDLength = 255
// MaxOwnerNameLength is the maximum allowed length for bucket owner identity names.
// This is a reasonable limit to prevent abuse; AWS IAM user names are limited to 64 chars,
// but we use 256 to allow for more complex identity formats (e.g., email addresses).
const MaxOwnerNameLength = 256
// S3 Bucket management data structures for templates
type S3BucketsData struct {
Username string `json:"username"`
Buckets []S3Bucket `json:"buckets"`
TotalBuckets int `json:"total_buckets"`
TotalSize int64 `json:"total_size"`
LastUpdated time.Time `json:"last_updated"`
// Pagination
CurrentPage int `json:"current_page"`
TotalPages int `json:"total_pages"`
PageSize int `json:"page_size"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
}
type CreateBucketRequest struct {
Name string `json:"name"` // validated manually in CreateBucket
Region string `json:"region"`
QuotaSize int64 `json:"quota_size"` // Quota size in bytes
QuotaUnit string `json:"quota_unit"` // Unit: MB, GB, TB
QuotaEnabled bool `json:"quota_enabled"` // Whether quota is enabled
VersioningEnabled bool `json:"versioning_enabled"` // Whether versioning is enabled
ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled
ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
SetDefaultRetention bool `json:"set_default_retention"` // Whether to set default retention
ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days
Owner string `json:"owner"` // Bucket owner identity (for S3 IAM authentication)
}
// S3 Bucket Management Handlers
// ShowS3Buckets displays the Object Store buckets management page
func (s *AdminServer) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
username := UsernameFromContext(r.Context())
data, err := s.GetS3BucketsData(1, 100, "name", "asc")
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get Object Store buckets: "+err.Error())
return
}
data.Username = username
writeJSON(w, http.StatusOK, data)
}
// ShowBucketDetails displays detailed information about a specific bucket
func (s *AdminServer) ShowBucketDetails(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
details, err := s.GetBucketDetails(bucketName)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get bucket details: "+err.Error())
return
}
writeJSON(w, http.StatusOK, details)
}
// ShowBucketLifecycle returns the lifecycle configuration for a specific bucket
func (s *AdminServer) ShowBucketLifecycle(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
lifecycle, err := s.GetBucketLifecycle(bucketName)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get bucket lifecycle: "+err.Error())
return
}
writeJSON(w, http.StatusOK, lifecycle)
}
// UpdateBucketLifecycle replaces the entire lifecycle configuration for a bucket.
func (s *AdminServer) UpdateBucketLifecycle(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
var req struct {
Rules []BucketLifecycleRule `json:"rules"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
if err := validateBucketLifecycleRules(req.Rules); err != nil {
writeJSONError(w, http.StatusBadRequest, err.Error())
return
}
if err := s.SetBucketLifecycle(bucketName, req.Rules); err != nil {
writeJSONError(w, bucketLifecycleErrorStatus(err), "Failed to update bucket lifecycle: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket lifecycle updated successfully",
"bucket": bucketName,
})
}
// DeleteBucketLifecycle clears the entire lifecycle configuration for a bucket.
func (s *AdminServer) DeleteBucketLifecycle(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
if err := s.SetBucketLifecycle(bucketName, nil); err != nil {
writeJSONError(w, bucketLifecycleErrorStatus(err), "Failed to delete bucket lifecycle: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket lifecycle deleted successfully",
"bucket": bucketName,
})
}
// bucketLifecycleErrorStatus keeps a request for a bucket that does not exist
// out of the 5xx bucket, where a client would retry it.
func bucketLifecycleErrorStatus(err error) int {
if errors.Is(err, ErrBucketNotFound) {
return http.StatusNotFound
}
return http.StatusInternalServerError
}
// validateBucketLifecycleRules rejects a rule set before any write is
// attempted, so a malformed PUT can't half-apply. Mirrors the constraints
// AWS enforces on PutBucketLifecycleConfiguration plus the local rule cap.
func validateBucketLifecycleRules(rules []BucketLifecycleRule) error {
if len(rules) > MaxBucketLifecycleRules {
return fmt.Errorf("a bucket may have at most %d lifecycle rules, got %d", MaxBucketLifecycleRules, len(rules))
}
seenIDs := make(map[string]bool, len(rules))
for i, rule := range rules {
label := fmt.Sprintf("rule %d", i+1)
if rule.ID != "" {
label = fmt.Sprintf("rule %q", rule.ID)
if len(rule.ID) > MaxBucketLifecycleRuleIDLength {
return fmt.Errorf("%s: ID must be %d characters or less", label, MaxBucketLifecycleRuleIDLength)
}
if seenIDs[rule.ID] {
return fmt.Errorf("duplicate rule ID %q", rule.ID)
}
seenIDs[rule.ID] = true
}
switch rule.Status {
case s3lifecycle.StatusEnabled, s3lifecycle.StatusDisabled:
default:
return fmt.Errorf("%s: status must be %q or %q, got %q", label, s3lifecycle.StatusEnabled, s3lifecycle.StatusDisabled, rule.Status)
}
if rule.ExpirationDays > 0 && rule.ExpirationDate != "" {
return fmt.Errorf("%s: expiration_days and expiration_date are mutually exclusive", label)
}
if rule.ExpirationDays < 0 {
return fmt.Errorf("%s: expiration_days must be positive", label)
}
if rule.ExpirationDate != "" {
if _, err := time.Parse(time.DateOnly, rule.ExpirationDate); err != nil {
return fmt.Errorf("%s: invalid expiration_date %q, expected YYYY-MM-DD", label, rule.ExpirationDate)
}
}
if rule.ExpiredObjectDeleteMarker && (rule.ExpirationDays > 0 || rule.ExpirationDate != "") {
return fmt.Errorf("%s: expired_object_delete_marker cannot be combined with expiration_days or expiration_date", label)
}
if rule.NoncurrentVersionExpirationDays < 0 {
return fmt.Errorf("%s: noncurrent_version_expiration_days must be positive", label)
}
if rule.NewerNoncurrentVersions < 0 {
return fmt.Errorf("%s: newer_noncurrent_versions must be positive", label)
}
if rule.AbortMultipartDays < 0 {
return fmt.Errorf("%s: abort_multipart_days must be positive", label)
}
if rule.SizeGreaterThan < 0 || rule.SizeLessThan < 0 {
return fmt.Errorf("%s: size_greater_than and size_less_than must be positive", label)
}
if rule.SizeGreaterThan > 0 && rule.SizeLessThan > 0 && rule.SizeGreaterThan >= rule.SizeLessThan {
return fmt.Errorf("%s: size_greater_than must be less than size_less_than", label)
}
for k, v := range rule.Tags {
if strings.TrimSpace(k) == "" {
return fmt.Errorf("%s: tag keys must not be empty", label)
}
if strings.TrimSpace(v) == "" {
return fmt.Errorf("%s: tag value for key %q must not be empty", label, k)
}
}
hasAction := rule.ExpirationDays > 0 || rule.ExpirationDate != "" || rule.ExpiredObjectDeleteMarker ||
rule.NoncurrentVersionExpirationDays > 0 || rule.NewerNoncurrentVersions > 0 || rule.AbortMultipartDays > 0
if !hasAction {
return fmt.Errorf("%s: must specify at least one action (expiration, noncurrent version expiration, or abort incomplete multipart upload)", label)
}
}
return nil
}
// ShowBucketPolicy returns the policy document for a specific bucket, or
// {"bucket": ..., "policy": null} if the bucket has none.
func (s *AdminServer) ShowBucketPolicy(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
policy, raw, err := s.GetBucketPolicy(bucketName)
if err != nil {
writeJSONError(w, bucketPolicyErrorStatus(err), "Failed to get bucket policy: "+err.Error())
return
}
resp := map[string]interface{}{
"bucket": bucketName,
"policy": policy,
}
if policy == nil && len(raw) > 0 {
// Stored bytes the decoder rejects: hand them to the JSON tab so
// the operator can fix or delete the document.
resp["policy_text"] = string(raw)
}
writeJSON(w, http.StatusOK, resp)
}
// UpdateBucketPolicy replaces the bucket policy for a bucket.
func (s *AdminServer) UpdateBucketPolicy(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
var req struct {
Policy *policy_engine.PolicyDocument `json:"policy"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
if req.Policy == nil {
writeJSONError(w, http.StatusBadRequest, "policy is required; use DELETE to clear a bucket policy")
return
}
if err := s.SetBucketPolicy(bucketName, req.Policy); err != nil {
writeJSONError(w, bucketPolicyErrorStatus(err), "Failed to update bucket policy: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket policy updated successfully",
"bucket": bucketName,
})
}
// RemoveBucketPolicy clears the bucket policy for a bucket. Named
// "Remove", not "Delete", because (*AdminServer).DeleteBucketPolicy is the
// data-layer method this handler calls.
func (s *AdminServer) RemoveBucketPolicy(w http.ResponseWriter, r *http.Request) {
if !requireSessionCSRFToken(w, r) {
return
}
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
if err := s.DeleteBucketPolicy(bucketName); err != nil {
writeJSONError(w, bucketPolicyErrorStatus(err), "Failed to delete bucket policy: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket policy deleted successfully",
"bucket": bucketName,
})
}
// bucketPolicyErrorStatus keeps a request for a bucket that does not exist,
// or an invalid policy document, out of the 5xx bucket where a client
// would retry it. Mirrors bucketLifecycleErrorStatus.
func bucketPolicyErrorStatus(err error) int {
if errors.Is(err, ErrBucketNotFound) {
return http.StatusNotFound
}
if errors.Is(err, ErrInvalidBucketPolicy) {
return http.StatusBadRequest
}
return http.StatusInternalServerError
}
// CreateBucket creates a new S3 bucket
func (s *AdminServer) CreateBucket(w http.ResponseWriter, r *http.Request) {
var req CreateBucketRequest
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
if strings.TrimSpace(req.Name) == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
// Validate bucket name (basic validation)
if len(req.Name) < 3 || len(req.Name) > 63 {
writeJSONError(w, http.StatusBadRequest, "Bucket name must be between 3 and 63 characters")
return
}
// Validate object lock settings
if req.ObjectLockEnabled {
// Object lock requires versioning to be enabled
req.VersioningEnabled = true
// Validate object lock mode
if req.ObjectLockMode != "GOVERNANCE" && req.ObjectLockMode != "COMPLIANCE" {
writeJSONError(w, http.StatusBadRequest, "Object lock mode must be either GOVERNANCE or COMPLIANCE")
return
}
// Validate retention duration if default retention is enabled
if req.SetDefaultRetention {
if req.ObjectLockDuration <= 0 {
writeJSONError(w, http.StatusBadRequest, "Object lock duration must be greater than 0 days when default retention is enabled")
return
}
}
}
normalizedUnit, err := normalizeQuotaUnit(req.QuotaUnit)
if err != nil {
writeJSONError(w, http.StatusBadRequest, err.Error())
return
}
req.QuotaUnit = normalizedUnit
quotaBytes := convertQuotaToBytes(req.QuotaSize, normalizedUnit)
// Validate quota: if enabled, size must be greater than 0
if req.QuotaEnabled && quotaBytes <= 0 {
writeJSONError(w, http.StatusBadRequest, "Quota size must be greater than 0 when quota is enabled")
return
}
// Sanitize owner: trim whitespace and enforce max length
owner := strings.TrimSpace(req.Owner)
if len(owner) > MaxOwnerNameLength {
writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength))
return
}
err = s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration, owner)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to create bucket: "+err.Error())
return
}
writeJSON(w, http.StatusCreated, map[string]interface{}{
"message": "Bucket created successfully",
"bucket": req.Name,
"quota_size": req.QuotaSize,
"quota_unit": req.QuotaUnit,
"quota_enabled": req.QuotaEnabled,
"versioning_enabled": req.VersioningEnabled,
"object_lock_enabled": req.ObjectLockEnabled,
"object_lock_mode": req.ObjectLockMode,
"object_lock_duration": req.ObjectLockDuration,
"owner": owner,
})
}
// UpdateBucketQuota updates the quota settings for a bucket
func (s *AdminServer) UpdateBucketQuota(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
var req struct {
QuotaSize int64 `json:"quota_size"`
QuotaUnit string `json:"quota_unit"`
QuotaEnabled bool `json:"quota_enabled"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
if req.QuotaEnabled && req.QuotaSize <= 0 {
writeJSONError(w, http.StatusBadRequest, "quota_size must be > 0 when quota_enabled is true")
return
}
normalizedUnit, err := normalizeQuotaUnit(req.QuotaUnit)
if err != nil {
writeJSONError(w, http.StatusBadRequest, err.Error())
return
}
req.QuotaUnit = normalizedUnit
// Convert quota to bytes
quotaBytes := convertQuotaToBytes(req.QuotaSize, normalizedUnit)
err = s.SetBucketQuota(bucketName, quotaBytes, req.QuotaEnabled)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to update bucket quota: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket quota updated successfully",
"bucket": bucketName,
"quota_size": req.QuotaSize,
"quota_unit": req.QuotaUnit,
"quota_enabled": req.QuotaEnabled,
})
}
// DeleteBucket deletes an S3 bucket
func (s *AdminServer) DeleteBucket(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
err := s.DeleteS3Bucket(bucketName)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to delete bucket: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket deleted successfully",
"bucket": bucketName,
})
}
// UpdateBucketOwner updates the owner of an S3 bucket
func (s *AdminServer) UpdateBucketOwner(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucket"]
if bucketName == "" {
writeJSONError(w, http.StatusBadRequest, "Bucket name is required")
return
}
// Use pointer to detect if owner field was explicitly provided
var req struct {
Owner *string `json:"owner"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
return
}
// Require owner field to be explicitly provided
if req.Owner == nil {
writeJSONError(w, http.StatusBadRequest, "Owner field is required (use empty string to clear owner)")
return
}
// Trim and validate owner
owner := strings.TrimSpace(*req.Owner)
if len(owner) > MaxOwnerNameLength {
writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength))
return
}
err := s.SetBucketOwner(bucketName, owner)
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to update bucket owner: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"message": "Bucket owner updated successfully",
"bucket": bucketName,
"owner": owner,
})
}
// SetBucketOwner sets the owner of a bucket
func (s *AdminServer) SetBucketOwner(bucketName string, owner string) error {
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Get the current bucket entry
lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err != nil {
return fmt.Errorf("lookup bucket %s: %w", bucketName, err)
}
bucketEntry := lookupResp.Entry
// Initialize Extended map if nil
if bucketEntry.Extended == nil {
bucketEntry.Extended = make(map[string][]byte)
}
// Set or remove the owner. The account id recorded by the S3 API goes with
// it: the S3 API derives it from the owning identity when it is absent, so
// leaving the old one behind would keep the previous owner of the objects.
delete(bucketEntry.Extended, s3_constants.ExtAmzOwnerKey)
if owner == "" {
delete(bucketEntry.Extended, s3_constants.AmzIdentityId)
} else {
bucketEntry.Extended[s3_constants.AmzIdentityId] = []byte(owner)
}
// Update the entry
_, err = client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
Directory: "/buckets",
Entry: bucketEntry,
})
if err != nil {
return fmt.Errorf("failed to update bucket owner: %w", err)
}
return nil
})
}
// ListBucketsAPI returns the list of buckets as JSON
func (s *AdminServer) ListBucketsAPI(w http.ResponseWriter, r *http.Request) {
buckets, err := s.GetS3Buckets()
if err != nil {
writeJSONError(w, http.StatusInternalServerError, "Failed to get buckets: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"buckets": buckets,
"total": len(buckets),
})
}
// Helper function to convert quota size and unit to bytes
func convertQuotaToBytes(size int64, unit string) int64 {
if size <= 0 {
return 0
}
switch unit {
case "TB":
return size * 1024 * 1024 * 1024 * 1024
case "GB":
return size * 1024 * 1024 * 1024
case "MB":
return size * 1024 * 1024
case "KB":
return size * 1024
case "B":
return size
default:
return 0
}
}
func normalizeQuotaUnit(unit string) (string, error) {
normalized := strings.ToUpper(strings.TrimSpace(unit))
if normalized == "" {
return "MB", nil
}
switch normalized {
case "B", "KB", "MB", "GB", "TB":
return normalized, nil
default:
return "", fmt.Errorf("unsupported quota unit: %s", unit)
}
}
// SetBucketQuota sets the quota for a bucket
func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Get the current bucket entry
lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err != nil {
return fmt.Errorf("bucket not found: %w", err)
}
bucketEntry := lookupResp.Entry
// Determine quota value (negative if disabled)
var quota int64
if quotaEnabled && quotaBytes > 0 {
quota = quotaBytes
} else if !quotaEnabled && quotaBytes > 0 {
quota = -quotaBytes
} else {
quota = 0
}
// Update the quota
bucketEntry.Quota = quota
// Update the entry
_, err = client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
Directory: "/buckets",
Entry: bucketEntry,
})
if err != nil {
return fmt.Errorf("failed to update bucket quota: %w", err)
}
if quota <= 0 {
// with no active quota, quota enforcement can no longer clear a
// read-only flag it turned on, so lift it here
if _, err := filer.ClearBucketReadOnly(context.Background(), client, "/buckets", bucketName); err != nil {
return fmt.Errorf("failed to clear bucket read-only flag: %w", err)
}
}
return nil
})
}
// CreateS3BucketWithQuota creates a new S3 bucket with quota settings
func (s *AdminServer) CreateS3BucketWithQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
return s.CreateS3BucketWithObjectLock(bucketName, quotaBytes, quotaEnabled, false, false, "", false, 0, "")
}
// CreateS3BucketWithObjectLock creates a new S3 bucket with quota, versioning, object lock settings, and owner
func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes int64, quotaEnabled, versioningEnabled, objectLockEnabled bool, objectLockMode string, setDefaultRetention bool, objectLockDuration int32, owner string) error {
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// First ensure /buckets directory exists
_, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
Directory: "/",
Entry: &filer_pb.Entry{
Name: "buckets",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{
FileMode: uint32(0755 | os.ModeDir), // Directory mode
Uid: uint32(1000),
Gid: uint32(1000),
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
TtlSec: 0,
},
},
})
// Ignore error if directory already exists
if err != nil && !strings.Contains(err.Error(), "already exists") && !strings.Contains(err.Error(), "existing entry") {
return fmt.Errorf("failed to create /buckets directory: %w", err)
}
// Check if bucket already exists
_, err = client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err == nil {
return fmt.Errorf("bucket %s already exists", bucketName)
}
// Determine quota value (negative if disabled)
var quota int64
if quotaEnabled && quotaBytes > 0 {
quota = quotaBytes
} else if !quotaEnabled && quotaBytes > 0 {
quota = -quotaBytes
} else {
quota = 0
}
// Prepare bucket attributes with versioning and object lock metadata
attributes := &filer_pb.FuseAttributes{
FileMode: uint32(0755 | os.ModeDir), // Directory mode
Uid: filer_pb.OS_UID,
Gid: filer_pb.OS_GID,
Crtime: time.Now().Unix(),
Mtime: time.Now().Unix(),
TtlSec: 0,
}
// Create extended attributes map for versioning and owner
extended := make(map[string][]byte)
// Set bucket owner if specified
if owner != "" {
extended[s3_constants.AmzIdentityId] = []byte(owner)
}
// Create bucket entry
bucketEntry := &filer_pb.Entry{
Name: bucketName,
IsDirectory: true,
Attributes: attributes,
Extended: extended,
Quota: quota,
}
// Handle versioning using shared utilities
if err := s3api.StoreVersioningInExtended(bucketEntry, versioningEnabled); err != nil {
return fmt.Errorf("failed to store versioning configuration: %w", err)
}
// Handle Object Lock configuration using shared utilities
if objectLockEnabled {
var duration int32 = 0
var mode string = ""
if setDefaultRetention {
// Validate Object Lock parameters only when setting default retention
if err := s3api.ValidateObjectLockParameters(objectLockEnabled, objectLockMode, objectLockDuration); err != nil {
return fmt.Errorf("invalid Object Lock parameters: %w", err)
}
duration = objectLockDuration
mode = objectLockMode
}
// Create Object Lock configuration using shared utility
objectLockConfig := s3api.CreateObjectLockConfigurationFromParams(objectLockEnabled, mode, duration)
// Store Object Lock configuration in extended attributes using shared utility
if err := s3api.StoreObjectLockConfigurationInExtended(bucketEntry, objectLockConfig); err != nil {
return fmt.Errorf("failed to store Object Lock configuration: %w", err)
}
}
// Create bucket directory under /buckets
_, err = client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
Directory: "/buckets",
Entry: bucketEntry,
})
if err != nil {
return fmt.Errorf("failed to create bucket directory: %w", err)
}
return nil
})
}

View File

@@ -0,0 +1,161 @@
package dash
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
)
// ErrInvalidBucketPolicy wraps a validation failure from SetBucketPolicy so
// callers (the HTTP handler) can map it to 400 instead of 500 without
// resorting to matching on the error string.
var ErrInvalidBucketPolicy = errors.New("invalid bucket policy")
// GetBucketPolicy returns the policy document stored on a bucket's filer
// entry, or (nil, nil, nil) if the bucket has no policy — that is not an
// error, it just means the caller (e.g. the admin UI) should show an empty
// editor instead of special-casing a 404. Stored bytes the current decoder
// rejects come back as (nil, raw, nil): a 500 here would leave the UI
// unable to show, fix, or even delete the one policy an operator most
// needs to remove — and DeleteBucketPolicy never reads the document.
func (s *AdminServer) GetBucketPolicy(bucketName string) (*policy_engine.PolicyDocument, []byte, error) {
filerConfig, err := s.getFilerConfig()
if err != nil {
return nil, nil, fmt.Errorf("get filer configuration: %w", err)
}
var doc *policy_engine.PolicyDocument
var raw []byte
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: filerConfig.BucketsPath,
Name: bucketName,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return fmt.Errorf("%w: %s", ErrBucketNotFound, bucketName)
}
return fmt.Errorf("look up bucket %s: %w", bucketName, err)
}
policyJSON := resp.Entry.Extended[s3api.BUCKET_POLICY_METADATA_KEY]
if len(policyJSON) == 0 {
return nil
}
raw = policyJSON
var parsed policy_engine.PolicyDocument
if err := json.Unmarshal(policyJSON, &parsed); err == nil {
doc = &parsed
}
return nil
})
if err != nil {
return nil, nil, err
}
return doc, raw, nil
}
// SetBucketPolicy validates and stores a bucket policy, applying the exact
// same validation the S3 gateway's PutBucketPolicy enforces
// (policy_engine.ValidatePolicy + policy_engine.ValidateBucketPolicy), so
// the admin UI and the S3 API never disagree about what's a valid policy.
//
// Propagation to every S3 gateway is automatic: writing the
// s3-bucket-policy extended attribute drives the filer metadata log, which
// each gateway's onBucketMetadataChange subscription watches to rebuild its
// bucket policy cache and to maintain the advanced-IAM
// "bucket-policy:<bucket>" mirror (mirrorBucketPolicyToIAM in
// weed/s3api/s3api_bucket_policy_handlers.go). No separate notify step is
// needed here.
func (s *AdminServer) SetBucketPolicy(bucketName string, doc *policy_engine.PolicyDocument) error {
if err := policy_engine.ValidatePolicy(doc); err != nil {
return fmt.Errorf("%w: %w", ErrInvalidBucketPolicy, err)
}
if err := policy_engine.ValidateBucketPolicy(doc, bucketName); err != nil {
return fmt.Errorf("%w: %w", ErrInvalidBucketPolicy, err)
}
policyJSON, err := json.Marshal(doc)
if err != nil {
return fmt.Errorf("marshal policy document: %w", err)
}
if len(policyJSON) > policy_engine.MaxBucketPolicySize {
return fmt.Errorf("%w: bucket policy is %d bytes, which exceeds the %d byte limit", ErrInvalidBucketPolicy, len(policyJSON), policy_engine.MaxBucketPolicySize)
}
return s.writeBucketPolicy(bucketName, policyJSON)
}
// DeleteBucketPolicy clears the bucket policy stored on a bucket's filer
// entry. Deleting a policy that doesn't exist is a success, matching
// DeleteBucketLifecycle's idempotent behavior. This cannot go through
// SetBucketPolicy: validation there rejects a nil document.
func (s *AdminServer) DeleteBucketPolicy(bucketName string) error {
return s.writeBucketPolicy(bucketName, nil)
}
// writeBucketPolicy patches the policy key on the bucket's filer entry; a
// nil policyJSON clears it.
func (s *AdminServer) writeBucketPolicy(bucketName string, policyJSON []byte) error {
filerConfig, err := s.getFilerConfig()
if err != nil {
return fmt.Errorf("get filer configuration: %w", err)
}
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// PATCH_EXTENDED is a no-op on a missing entry, so the existence
// check has to happen here rather than fall out of the write.
if _, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: filerConfig.BucketsPath,
Name: bucketName,
}); err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return fmt.Errorf("%w: %s", ErrBucketNotFound, bucketName)
}
return fmt.Errorf("look up bucket %s: %w", bucketName, err)
}
bucketPath := filerConfig.BucketsPath + "/" + bucketName
resp, err := client.ObjectTransaction(context.Background(), &filer_pb.ObjectTransactionRequest{
LockKey: bucketPath,
RouteKey: s3_constants.ObjectWriteRouteKeyPrefix + bucketPath,
Mutations: []*filer_pb.ObjectMutation{bucketPolicyMutation(filerConfig.BucketsPath, bucketName, policyJSON)},
})
if err != nil {
return fmt.Errorf("write bucket policy: %w", err)
}
if resp.Error != "" {
return fmt.Errorf("write bucket policy: %s", resp.Error)
}
return nil
})
}
// bucketPolicyMutation patches only the policy key rather than writing the
// whole entry back: the filer re-reads and merges under the bucket path
// lock, so a concurrent owner/quota/versioning/lifecycle change is
// preserved instead of being reverted by a stale snapshot. Same pattern as
// bucketLifecycleMutation. A nil/empty policyJSON clears the key.
func bucketPolicyMutation(bucketsPath, bucketName string, policyJSON []byte) *filer_pb.ObjectMutation {
mutation := &filer_pb.ObjectMutation{
Type: filer_pb.ObjectMutation_PATCH_EXTENDED,
Directory: bucketsPath,
Name: bucketName,
}
if len(policyJSON) > 0 {
mutation.SetExtended = map[string][]byte{
s3api.BUCKET_POLICY_METADATA_KEY: policyJSON,
}
return mutation
}
mutation.DeleteExtended = []string{s3api.BUCKET_POLICY_METADATA_KEY}
return mutation
}

View File

@@ -0,0 +1,168 @@
package dash
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
func validBucketPolicyJSON(bucket string) []byte {
return []byte(fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket))
}
func validBucketPolicyDoc(bucket string) *policy_engine.PolicyDocument {
var doc policy_engine.PolicyDocument
if err := json.Unmarshal(validBucketPolicyJSON(bucket), &doc); err != nil {
panic(err)
}
return &doc
}
func TestBucketPolicyMutation_SetsPolicy(t *testing.T) {
policyJSON := validBucketPolicyJSON("mybucket")
m := bucketPolicyMutation("/buckets", "mybucket", policyJSON)
if m.Type != filer_pb.ObjectMutation_PATCH_EXTENDED {
t.Fatalf("expected a PATCH_EXTENDED mutation, got %v", m.Type)
}
if m.Directory != "/buckets" || m.Name != "mybucket" {
t.Fatalf("expected the mutation to target /buckets/mybucket, got %s/%s", m.Directory, m.Name)
}
if got := m.SetExtended[s3api.BUCKET_POLICY_METADATA_KEY]; string(got) != string(policyJSON) {
t.Fatalf("expected the policy key to carry the marshaled document, got %q", got)
}
if len(m.DeleteExtended) != 0 {
t.Fatalf("expected no key deletions when saving a policy, got %v", m.DeleteExtended)
}
}
func TestBucketPolicyMutation_ClearsKey(t *testing.T) {
m := bucketPolicyMutation("/buckets", "mybucket", nil)
if len(m.SetExtended) != 0 {
t.Fatalf("expected no key writes when clearing, got %v", m.SetExtended)
}
if len(m.DeleteExtended) != 1 || m.DeleteExtended[0] != s3api.BUCKET_POLICY_METADATA_KEY {
t.Fatalf("expected only the policy key to be cleared, got %v", m.DeleteExtended)
}
}
// A whole-entry write would have carried the rest of the bucket entry with
// it; the patch must name only the key it owns, so a concurrent owner,
// quota, or lifecycle change survives.
func TestBucketPolicyMutation_TouchesOnlyPolicyKey(t *testing.T) {
for _, m := range []*filer_pb.ObjectMutation{
bucketPolicyMutation("/buckets", "mybucket", validBucketPolicyJSON("mybucket")),
bucketPolicyMutation("/buckets", "mybucket", nil),
} {
if m.Entry != nil {
t.Fatal("expected the mutation to carry no entry snapshot")
}
if m.SetContent {
t.Fatal("expected the mutation to leave entry content alone")
}
for k := range m.SetExtended {
if k != s3api.BUCKET_POLICY_METADATA_KEY {
t.Fatalf("unexpected key written: %s", k)
}
}
for _, k := range m.DeleteExtended {
if k != s3api.BUCKET_POLICY_METADATA_KEY {
t.Fatalf("unexpected key deleted: %s", k)
}
}
}
}
func TestExtractPolicyStatementCountFromEntry(t *testing.T) {
tests := []struct {
name string
entry *filer_pb.Entry
want int
}{
{"no extended attrs", &filer_pb.Entry{}, 0},
{"absent key", &filer_pb.Entry{Extended: map[string][]byte{"other": []byte("x")}}, 0},
{"one statement", &filer_pb.Entry{Extended: map[string][]byte{
s3api.BUCKET_POLICY_METADATA_KEY: validBucketPolicyJSON("b"),
}}, 1},
{"three statements", &filer_pb.Entry{Extended: map[string][]byte{
s3api.BUCKET_POLICY_METADATA_KEY: []byte(`{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"},
{"Effect":"Allow","Principal":"*","Action":"s3:PutObject","Resource":"arn:aws:s3:::b/*"},
{"Effect":"Deny","Principal":"*","Action":"s3:DeleteObject","Resource":"arn:aws:s3:::b/*"}
]}`),
}}, 3},
{"garbage bytes", &filer_pb.Entry{Extended: map[string][]byte{
s3api.BUCKET_POLICY_METADATA_KEY: []byte("not json"),
}}, 0},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := extractPolicyStatementCountFromEntry(tt.entry); got != tt.want {
t.Errorf("extractPolicyStatementCountFromEntry() = %d, want %d", got, tt.want)
}
})
}
}
func TestBucketPolicyErrorStatus(t *testing.T) {
if got := bucketPolicyErrorStatus(fmt.Errorf("%w: mybucket", ErrBucketNotFound)); got != http.StatusNotFound {
t.Fatalf("expected a missing bucket to map to 404, got %d", got)
}
if got := bucketPolicyErrorStatus(fmt.Errorf("%w: bad statement", ErrInvalidBucketPolicy)); got != http.StatusBadRequest {
t.Fatalf("expected an invalid policy to map to 400, got %d", got)
}
if got := bucketPolicyErrorStatus(errors.New("filer unreachable")); got != http.StatusInternalServerError {
t.Fatalf("expected an unrelated failure to stay 500, got %d", got)
}
}
func TestSetBucketPolicy_RejectsOversized(t *testing.T) {
// A resource list long enough to blow the cap: this must fail before
// any filer call, which is what makes it testable without one.
doc := validBucketPolicyDoc("mybucket")
doc.Statement[0].Sid = strings.Repeat("x", policy_engine.MaxBucketPolicySize+1)
err := (&AdminServer{}).SetBucketPolicy("mybucket", doc)
if err == nil {
t.Fatal("expected an oversized bucket policy to be rejected")
}
if !errors.Is(err, ErrInvalidBucketPolicy) {
t.Fatalf("expected an ErrInvalidBucketPolicy, got: %v", err)
}
}
func TestSetBucketPolicy_RejectsForeignResource(t *testing.T) {
// Proves the shared policy_engine.ValidateBucketPolicy validator is
// actually wired in: this is exactly the check the S3 gateway applies.
doc := validBucketPolicyDoc("mybucket")
doc.Statement[0].Resource = policy_engine.NewStringOrStringSlicePtr("arn:aws:s3:::other-bucket/*")
err := (&AdminServer{}).SetBucketPolicy("mybucket", doc)
if err == nil {
t.Fatal("expected a policy referencing a different bucket to be rejected")
}
if !errors.Is(err, ErrInvalidBucketPolicy) {
t.Fatalf("expected an ErrInvalidBucketPolicy, got: %v", err)
}
}
func TestSetBucketPolicy_RejectsMissingPrincipal(t *testing.T) {
doc := validBucketPolicyDoc("mybucket")
doc.Statement[0].Principal = nil
err := (&AdminServer{}).SetBucketPolicy("mybucket", doc)
if err == nil {
t.Fatal("expected a policy with no Principal to be rejected")
}
if !errors.Is(err, ErrInvalidBucketPolicy) {
t.Fatalf("expected an ErrInvalidBucketPolicy, got: %v", err)
}
}

View File

@@ -0,0 +1,124 @@
package dash
import (
"context"
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
)
// WithMasterClient executes a function with a master client connection
func (s *AdminServer) WithMasterClient(f func(client master_pb.SeaweedClient) error) error {
return s.masterClient.WithClient(context.Background(), false, f)
}
// WithFilerClient executes a function with a filer client connection
func (s *AdminServer) WithFilerClient(f func(client filer_pb.SeaweedFilerClient) error) error {
filerAddr := s.GetFilerAddress()
if filerAddr == "" {
return fmt.Errorf("no filer available")
}
return pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddr), s.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
return f(client)
})
}
// WithVolumeServerClient executes a function with a volume server client connection
func (s *AdminServer) WithVolumeServerClient(address pb.ServerAddress, f func(client volume_server_pb.VolumeServerClient) error) error {
return operation.WithVolumeServerClient(false, address, s.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
return f(client)
})
}
// GetMasterClient returns the admin server's wdclient.MasterClient. It is used
// by file browser download paths that stream chunks straight from the volume
// servers via filer.PrepareStreamContent so they keep working when the filer
// has -disableHttp=true.
func (s *AdminServer) GetMasterClient() *wdclient.MasterClient {
return s.masterClient
}
// GetGrpcDialOption returns the dial option used for all admin-originated
// gRPC connections (TLS or insecure). File browser uploads need this when
// they perform the assign + volume HTTP POST + create-entry flow.
func (s *AdminServer) GetGrpcDialOption() grpc.DialOption {
return s.grpcDialOption
}
// VolumeServerReadJwt mints a per-fileId Bearer token for reads against a
// volume server when jwt.signing.read.key is configured. The volume servers
// are unaware of jwt.filer_signing.read.key — that one only gates the filer
// HTTP surface, which this code path doesn't touch.
func VolumeServerReadJwt(fileId string) string {
v := util.GetViper()
signingKey := security.SigningKey(v.GetString("jwt.signing.read.key"))
if len(signingKey) == 0 {
return ""
}
expiresAfterSec := v.GetInt("jwt.signing.read.expires_after_seconds")
return string(security.GenJwtForVolumeServer(signingKey, expiresAfterSec, fileId))
}
// GetFilerAddress returns a filer address, discovering from masters if needed
func (s *AdminServer) GetFilerAddress() string {
// Discover filers from masters
filers := s.getDiscoveredFilers()
if len(filers) > 0 {
return filers[0] // Return the first available filer
}
return ""
}
// getDiscoveredFilers returns cached filers or discovers them from masters
func (s *AdminServer) getDiscoveredFilers() []string {
// Check if cache is still valid
if time.Since(s.lastFilerUpdate) < s.filerCacheExpiration && len(s.cachedFilers) > 0 {
return s.cachedFilers
}
// Discover filers from masters
var filers []string
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), s.listClusterNodesRequest(cluster.FilerType))
if err != nil {
return err
}
for _, node := range resp.ClusterNodes {
filers = append(filers, node.Address)
}
return nil
})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Warningf("Failed to discover filers from master %s: %v", currentMaster, err)
// Return cached filers even if expired, better than nothing
return s.cachedFilers
}
// Update cache
s.cachedFilers = filers
s.lastFilerUpdate = time.Now()
return filers
}
// GetAllFilers returns all discovered filers
func (s *AdminServer) GetAllFilers() []string {
return s.getDiscoveredFilers()
}

View File

@@ -0,0 +1,248 @@
package dash
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
const dirStatusTimeout = 5 * time.Second
// GetClusterTopology returns the current cluster topology with caching
func (s *AdminServer) GetClusterTopology() (*ClusterTopology, error) {
now := time.Now()
if s.cachedTopology != nil && now.Sub(s.lastCacheUpdate) < s.cacheExpiration {
return s.cachedTopology, nil
}
topology := &ClusterTopology{
UpdatedAt: now,
}
// Use gRPC only
err := s.getTopologyViaGRPC(topology)
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to connect to master server %s: %v", currentMaster, err)
return nil, fmt.Errorf("gRPC topology request failed: %w", err)
}
// Cache the result
s.cachedTopology = topology
s.lastCacheUpdate = now
return topology, nil
}
// fetchPublicUrlMap queries the master's /dir/status HTTP endpoint and returns
// a map from data node ID (ip:port) to its PublicUrl.
func (s *AdminServer) fetchPublicUrlMap() map[string]string {
currentMaster := s.masterClient.GetMaster(context.Background())
if currentMaster == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), dirStatusTimeout)
defer cancel()
url := fmt.Sprintf("http://%s/dir/status", currentMaster.ToHttpAddress())
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
glog.V(1).Infof("Failed to build /dir/status request for %s: %v", currentMaster, err)
return nil
}
resp, err := util_http.GetGlobalHttpClient().Do(req)
if err != nil {
glog.V(1).Infof("Failed to fetch /dir/status from %s: %v", currentMaster, err)
return nil
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
glog.V(1).Infof("Non-OK response from /dir/status: %d", resp.StatusCode)
return nil
}
body, err := io.ReadAll(resp.Body)
if err != nil {
glog.V(1).Infof("Failed to read /dir/status response body: %v", err)
return nil
}
// Parse the JSON response to extract PublicUrl for each data node
var status struct {
Topology struct {
DataCenters []struct {
Racks []struct {
DataNodes []struct {
Url string `json:"Url"`
PublicUrl string `json:"PublicUrl"`
} `json:"DataNodes"`
} `json:"Racks"`
} `json:"DataCenters"`
} `json:"Topology"`
}
if err := json.Unmarshal(body, &status); err != nil {
glog.V(1).Infof("Failed to parse /dir/status response: %v", err)
return nil
}
publicUrls := make(map[string]string)
for _, dc := range status.Topology.DataCenters {
for _, rack := range dc.Racks {
for _, dn := range rack.DataNodes {
if dn.PublicUrl != "" {
publicUrls[dn.Url] = dn.PublicUrl
}
}
}
}
return publicUrls
}
// getTopologyViaGRPC gets topology using gRPC (original method)
func (s *AdminServer) getTopologyViaGRPC(topology *ClusterTopology) error {
// Fetch public URL mapping from master HTTP API
// The gRPC DataNodeInfo does not include PublicUrl, so we supplement it.
publicUrls := s.fetchPublicUrlMap()
// Get cluster status from master
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
currentMaster := s.masterClient.GetMaster(context.Background())
glog.Errorf("Failed to get volume list from master %s: %v", currentMaster, err)
return err
}
if resp.TopologyInfo != nil {
// Get volume size limit from response, default to 30GB if not set
volumeSizeLimitMb := resp.VolumeSizeLimitMb
if volumeSizeLimitMb == 0 {
volumeSizeLimitMb = 30000
}
// Process gRPC response
for _, dc := range resp.TopologyInfo.DataCenterInfos {
dataCenter := DataCenter{
ID: dc.Id,
Racks: []Rack{},
}
for _, rack := range dc.RackInfos {
rackObj := Rack{
ID: rack.Id,
Nodes: []VolumeServer{},
}
for _, node := range rack.DataNodeInfos {
// Calculate totals from disk infos
var totalVolumes int64
var totalMaxVolumes int64
var totalSize int64
var remoteSize int64
// Prefer the real physical disk capacity the volume server
// reports per disk; the slot-based estimate overstates capacity
// when maxVolumeCount is configured higher than the disk holds.
var diskCapacity int64
for _, diskInfo := range node.DiskInfos {
totalVolumes += diskInfo.VolumeCount
totalMaxVolumes += diskInfo.MaxVolumeCount
if diskInfo.DiskTotalBytes > 0 {
diskCapacity += int64(diskInfo.DiskTotalBytes)
} else {
diskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024
}
// A remote-tiered volume reports its cloud object's
// size; keep those bytes out of the local disk usage
// that is compared against diskCapacity.
for _, volInfo := range diskInfo.VolumeInfos {
if volInfo.RemoteStorageName != "" {
remoteSize += int64(volInfo.Size)
} else {
totalSize += int64(volInfo.Size)
}
}
// ShardSizes is local to this node, so summing
// across nodes gives the physical footprint.
for _, ecShardInfo := range diskInfo.EcShardInfos {
for _, shardSize := range ecShardInfo.ShardSizes {
totalSize += shardSize
}
}
}
// Look up PublicUrl from master HTTP API
// Use node.Address (ip:port) as the key, matching the Url field in /dir/status
nodeAddr := node.Address
if nodeAddr == "" {
nodeAddr = node.Id
}
publicUrl := publicUrls[nodeAddr]
if publicUrl == "" {
publicUrl = nodeAddr
}
vs := VolumeServer{
ID: node.Id,
Address: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
PublicURL: publicUrl,
Volumes: int(totalVolumes),
MaxVolumes: int(totalMaxVolumes),
DiskUsage: totalSize,
DiskCapacity: diskCapacity,
LastHeartbeat: time.Now(),
RemoteSize: remoteSize,
}
rackObj.Nodes = append(rackObj.Nodes, vs)
topology.VolumeServers = append(topology.VolumeServers, vs)
topology.TotalVolumes += vs.Volumes
// TotalSize is the logical data size, wherever the
// bytes live, so remote-tiered volumes still count.
topology.TotalSize += totalSize + remoteSize
}
dataCenter.Racks = append(dataCenter.Racks, rackObj)
}
topology.DataCenters = append(topology.DataCenters, dataCenter)
}
// Chunk counts come from the shared collection aggregation, which
// nets out tombstones and counts a chunk once no matter how many
// volume replicas or EC shard holders report it.
topology.TotalChunks = totalCollectionFileCount(resp.TopologyInfo)
topology.TierStats = CollectTierStats(resp.TopologyInfo, volumeSizeLimitMb)
}
return nil
})
return err
}
// InvalidateCache forces a refresh of cached data
func (s *AdminServer) InvalidateCache() {
s.lastCacheUpdate = time.Now().Add(-s.cacheExpiration)
s.cachedTopology = nil
s.lastFilerUpdate = time.Now().Add(-s.filerCacheExpiration)
s.cachedFilers = nil
s.lastCollectionStatsUpdate = time.Now().Add(-s.collectionStatsCacheThreshold)
s.collectionStatsCache = nil
}

View File

@@ -0,0 +1,330 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// TestCollectCollectionStatsECUnevenShards verifies that EC shards spread
// across multiple disks are aggregated correctly, including the case where
// the last data shard is smaller than the rest.
func TestCollectCollectionStatsECUnevenShards(t *testing.T) {
// Standard 10+4 EC layout. Shards 0..9 are data, 10..13 are parity.
// Data shards 0..8 are 1000 bytes; data shard 9 is 500 bytes (uneven tail).
// Parity shards 10..13 are 1000 bytes each.
//
// Physical (raw) = 9*1000 + 500 + 4*1000 = 13500
// Logical (data only) = 9*1000 + 500 = 9500
nodeA := &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 42,
Collection: "bucket-a",
// Shards 0..6 held here.
EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
},
},
},
},
}
nodeB := &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 42,
Collection: "bucket-a",
// Shards 7..13 held here. Shard 9 (data) is the short tail.
EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
ShardSizes: []int64{1000, 1000, 500, 1000, 1000, 1000, 1000},
},
},
},
},
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
},
},
},
},
}
stats := collectCollectionStats(topo)
got, ok := stats["bucket-a"]
if !ok {
t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
}
const wantPhysical int64 = 13500
const wantLogical int64 = 9500
if got.PhysicalSize != wantPhysical {
t.Errorf("PhysicalSize: got %d, want %d", got.PhysicalSize, wantPhysical)
}
if got.LogicalSize != wantLogical {
t.Errorf("LogicalSize: got %d, want %d", got.LogicalSize, wantLogical)
}
}
// TestCollectCollectionStatsECEmptyCollection verifies that EC shards with
// an empty collection name are bucketed under "default".
func TestCollectCollectionStatsECEmptyCollection(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 1,
Collection: "",
EcIndexBits: (1 << 0) | (1 << 10),
ShardSizes: []int64{2000, 2000},
},
},
},
},
},
},
},
},
},
},
}
stats := collectCollectionStats(topo)
got, ok := stats["default"]
if !ok {
t.Fatalf("expected collection default in stats, got: %v", stats)
}
if got.PhysicalSize != 4000 {
t.Errorf("PhysicalSize: got %d, want 4000", got.PhysicalSize)
}
// Only shard 0 is a data shard; shard 10 is parity.
if got.LogicalSize != 2000 {
t.Errorf("LogicalSize: got %d, want 2000", got.LogicalSize)
}
}
// TestCollectCollectionStatsECFileAndDeleteCountAggregation verifies that
// FileCount for an EC volume is deduped across nodes (every shard holder has
// an identical .ecx, so the total entry count is taken once) while
// DeleteCount is summed (each needle delete tombstones exactly one node's
// .ecx, so the true delete total is the sum of every holder's local count).
func TestCollectCollectionStatsECFileAndDeleteCountAggregation(t *testing.T) {
// Volume id=7 reported by three nodes. Every node reports file_count=100
// (same .ecx). Local delete counts: 5 + 3 + 2 = 10 deletes total.
// Expected live object count = 100 - 10 = 90.
makeNode := func(bits uint32, sizes []int64, deleteCount uint64) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 7,
Collection: "bucket-a",
EcIndexBits: bits,
ShardSizes: sizes,
FileCount: 100,
DeleteCount: deleteCount,
},
},
},
},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3), []int64{1000, 1000, 1000, 1000}, 5),
makeNode((1<<4)|(1<<5)|(1<<6)|(1<<7), []int64{1000, 1000, 1000, 1000}, 3),
makeNode((1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1000, 1000, 1000, 1000, 1000, 1000}, 2),
},
},
},
},
},
}
stats := collectCollectionStats(topo)
got, ok := stats["bucket-a"]
if !ok {
t.Fatalf("expected collection bucket-a in stats, got: %v", stats)
}
if got.FileCount != 90 {
t.Errorf("FileCount: got %d, want 90 (100 total - 10 deletes summed)", got.FileCount)
}
// Sanity: 14 shards × 1000 bytes physical, 10 data shards logical.
if got.PhysicalSize != 14000 {
t.Errorf("PhysicalSize: got %d, want 14000", got.PhysicalSize)
}
if got.LogicalSize != 10000 {
t.Errorf("LogicalSize: got %d, want 10000", got.LogicalSize)
}
}
// TestCollectCollectionStatsECFileCountMaxDedupe verifies that EC file_count
// is taken as the max across reporting nodes rather than the first-seen
// value. A node that has not yet finished loading .ecx reports file_count=0,
// which previously poisoned the aggregate and rendered buckets backed by EC
// volumes as "0 objects".
func TestCollectCollectionStatsECFileCountMaxDedupe(t *testing.T) {
makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo {
return &master_pb.DataNodeInfo{
DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 11,
Collection: "bucket-b",
EcIndexBits: bits,
ShardSizes: sizes,
FileCount: fileCount,
},
},
},
},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
// First-reporting node has a stale fileCount of 0,
// second node reports the authoritative 6.
DataNodeInfos: []*master_pb.DataNodeInfo{
makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0),
makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6),
},
},
},
},
},
}
stats := collectCollectionStats(topo)
got, ok := stats["bucket-b"]
if !ok {
t.Fatalf("expected collection bucket-b in stats, got: %v", stats)
}
if got.FileCount != 6 {
t.Errorf("FileCount: got %d, want 6 (max across reporters)", got.FileCount)
}
}
// TestTotalCollectionFileCount verifies the cluster-wide chunk count counts a
// chunk once per replicated volume and per EC volume, and nets out deletes.
func TestTotalCollectionFileCount(t *testing.T) {
// Volume 1 has replication 001 (two copies), so both nodes report the same
// 101 chunks with 10 deleted: 91 live chunks in total, not 182 and not the
// 90 that halving each report would give.
replica := func() *master_pb.DiskInfo {
return &master_pb.DiskInfo{
VolumeInfos: []*master_pb.VolumeInformationMessage{
{
Id: 1,
Collection: "bucket-a",
ReplicaPlacement: 1,
FileCount: 101,
DeleteCount: 10,
},
},
}
}
// EC volume 2 has its 20 chunks reported by every shard holder.
ecShards := func(bits uint32, sizes []int64) *master_pb.DiskInfo {
return &master_pb.DiskInfo{
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{
Id: 2,
Collection: "bucket-b",
EcIndexBits: bits,
ShardSizes: sizes,
FileCount: 20,
},
},
}
}
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
{DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": replica(),
"disk2": ecShards((1<<0)|(1<<1), []int64{1, 1}),
}},
{DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": replica(),
"disk2": ecShards((1<<2)|(1<<3), []int64{1, 1}),
}},
},
},
},
},
},
}
if got := totalCollectionFileCount(topo); got != 111 {
t.Errorf("totalCollectionFileCount: got %d, want 111 (91 replicated + 20 EC)", got)
}
}
// TestTotalCollectionFileCountUnderReplicated verifies a volume whose second
// replica has not reported yet still contributes its full live count.
func TestTotalCollectionFileCountUnderReplicated(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
RackInfos: []*master_pb.RackInfo{
{
DataNodeInfos: []*master_pb.DataNodeInfo{
{DiskInfos: map[string]*master_pb.DiskInfo{
"disk1": {
VolumeInfos: []*master_pb.VolumeInformationMessage{
{
Id: 1,
Collection: "bucket-a",
ReplicaPlacement: 1,
FileCount: 50,
},
},
},
}},
},
},
},
},
},
}
if got := totalCollectionFileCount(topo); got != 50 {
t.Errorf("totalCollectionFileCount: got %d, want 50 (one replica reporting, not halved)", got)
}
}

View File

@@ -0,0 +1,395 @@
package dash
import (
"context"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// GetClusterCollections retrieves cluster collections data
func (s *AdminServer) GetClusterCollections() (*ClusterCollectionsData, error) {
var collections []CollectionInfo
var totalVolumes int
var totalEcVolumes int
var totalChunks int64
var totalSize int64
collectionMap := make(map[string]*CollectionInfo)
// Get actual collection information from volume data
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
// Process regular volumes
for _, volInfo := range diskInfo.VolumeInfos {
// Extract collection name from volume info
collectionName := volInfo.Collection
if collectionName == "" {
collectionName = "default" // Default collection for volumes without explicit collection
}
// Get disk type from volume info, default to hdd if empty
diskType := volInfo.DiskType
if diskType == "" {
diskType = "hdd"
}
// Get or create collection info
if collection, exists := collectionMap[collectionName]; exists {
collection.VolumeCount++
collection.TotalSize += int64(volInfo.Size)
// Update data center if this collection spans multiple DCs
if collection.DataCenter != dc.Id && collection.DataCenter != "multi" {
collection.DataCenter = "multi"
}
// Add disk type if not already present
diskTypeExists := false
for _, existingDiskType := range collection.DiskTypes {
if existingDiskType == diskType {
diskTypeExists = true
break
}
}
if !diskTypeExists {
collection.DiskTypes = append(collection.DiskTypes, diskType)
}
totalVolumes++
totalSize += int64(volInfo.Size)
} else {
newCollection := CollectionInfo{
Name: collectionName,
DataCenter: dc.Id,
VolumeCount: 1,
EcVolumeCount: 0,
TotalSize: int64(volInfo.Size),
DiskTypes: []string{diskType},
}
collectionMap[collectionName] = &newCollection
totalVolumes++
totalSize += int64(volInfo.Size)
}
}
// Process EC volumes
ecVolumeMap := make(map[uint32]bool) // Track unique EC volumes to avoid double counting
for _, ecShardInfo := range diskInfo.EcShardInfos {
// Extract collection name from EC shard info
collectionName := ecShardInfo.Collection
if collectionName == "" {
collectionName = "default" // Default collection for EC volumes without explicit collection
}
// Only count each EC volume once (not per shard)
if !ecVolumeMap[ecShardInfo.Id] {
ecVolumeMap[ecShardInfo.Id] = true
// Get disk type from disk info, default to hdd if empty
diskType := diskInfo.Type
if diskType == "" {
diskType = "hdd"
}
// Get or create collection info
if collection, exists := collectionMap[collectionName]; exists {
collection.EcVolumeCount++
// Update data center if this collection spans multiple DCs
if collection.DataCenter != dc.Id && collection.DataCenter != "multi" {
collection.DataCenter = "multi"
}
// Add disk type if not already present
diskTypeExists := false
for _, existingDiskType := range collection.DiskTypes {
if existingDiskType == diskType {
diskTypeExists = true
break
}
}
if !diskTypeExists {
collection.DiskTypes = append(collection.DiskTypes, diskType)
}
totalEcVolumes++
} else {
newCollection := CollectionInfo{
Name: collectionName,
DataCenter: dc.Id,
VolumeCount: 0,
EcVolumeCount: 1,
TotalSize: 0,
DiskTypes: []string{diskType},
}
collectionMap[collectionName] = &newCollection
totalEcVolumes++
}
}
}
}
}
}
}
// Chunk counts come from the shared collection aggregation, which
// nets out tombstones and counts a chunk once no matter how many
// volume replicas or EC shard holders report it.
for collectionName, stats := range collectCollectionStats(resp.TopologyInfo) {
if collection, exists := collectionMap[collectionName]; exists {
collection.ChunkCount = stats.FileCount
totalChunks += stats.FileCount
}
}
}
return nil
})
if err != nil {
return nil, err
}
// Convert map to slice
for _, collection := range collectionMap {
collections = append(collections, *collection)
}
// Sort collections alphabetically by name
sort.Slice(collections, func(i, j int) bool {
return collections[i].Name < collections[j].Name
})
// If no collections found, show a message indicating no collections exist
if len(collections) == 0 {
// Return empty collections data instead of creating fake ones
return &ClusterCollectionsData{
Collections: []CollectionInfo{},
TotalCollections: 0,
TotalVolumes: 0,
TotalEcVolumes: 0,
TotalChunks: 0,
TotalSize: 0,
LastUpdated: time.Now(),
}, nil
}
return &ClusterCollectionsData{
Collections: collections,
TotalCollections: len(collections),
TotalVolumes: totalVolumes,
TotalEcVolumes: totalEcVolumes,
TotalChunks: totalChunks,
TotalSize: totalSize,
LastUpdated: time.Now(),
}, nil
}
// GetCollectionDetails retrieves detailed information for a specific collection including volumes and EC volumes
func (s *AdminServer) GetCollectionDetails(collectionName string, page int, pageSize int, sortBy string, sortOrder string) (*CollectionDetailsData, error) {
// Set defaults
if page < 1 {
page = 1
}
if pageSize < 1 || pageSize > 1000 {
pageSize = 25
}
if sortBy == "" {
sortBy = "volume_id"
}
if sortOrder == "" {
sortOrder = "asc"
}
var regularVolumes []VolumeWithTopology
var ecVolumes []EcVolumeWithShards
var totalSize int64
dataCenters := make(map[string]bool)
diskTypes := make(map[string]bool)
// Get regular volumes for this collection
regularVolumeData, err := s.GetClusterVolumes(1, 10000, "volume_id", "asc", collectionName) // Get all volumes
if err != nil {
return nil, err
}
regularVolumes = regularVolumeData.Volumes
totalSize = regularVolumeData.TotalSize
// Chunk counts come from the shared collection aggregation, which nets out
// tombstones and counts a chunk once no matter how many volume replicas or
// EC shard holders report it. Fail rather than render a zero that reads
// like an empty collection.
stats, err := s.getCollectionStats()
if err != nil {
return nil, err
}
totalChunks := stats[collectionName].FileCount
// Collect data centers and disk types from regular volumes
for _, vol := range regularVolumes {
dataCenters[vol.DataCenter] = true
diskTypes[vol.DiskType] = true
}
// Get EC volumes for this collection
ecVolumeData, err := s.GetClusterEcVolumes(1, 10000, "volume_id", "asc", collectionName) // Get all EC volumes
if err != nil {
return nil, err
}
ecVolumes = ecVolumeData.EcVolumes
// Collect data centers from EC volumes
for _, ecVol := range ecVolumes {
for _, dc := range ecVol.DataCenters {
dataCenters[dc] = true
}
}
// Combine all volumes for sorting and pagination
type VolumeForSorting struct {
Type string // "regular" or "ec"
RegularVolume *VolumeWithTopology
EcVolume *EcVolumeWithShards
}
var allVolumes []VolumeForSorting
for i := range regularVolumes {
allVolumes = append(allVolumes, VolumeForSorting{
Type: "regular",
RegularVolume: &regularVolumes[i],
})
}
for i := range ecVolumes {
allVolumes = append(allVolumes, VolumeForSorting{
Type: "ec",
EcVolume: &ecVolumes[i],
})
}
// Sort all volumes
sort.Slice(allVolumes, func(i, j int) bool {
var less bool
switch sortBy {
case "volume_id":
var idI, idJ uint32
if allVolumes[i].Type == "regular" {
idI = allVolumes[i].RegularVolume.Id
} else {
idI = allVolumes[i].EcVolume.VolumeID
}
if allVolumes[j].Type == "regular" {
idJ = allVolumes[j].RegularVolume.Id
} else {
idJ = allVolumes[j].EcVolume.VolumeID
}
less = idI < idJ
case "type":
// Sort by type first (regular before ec), then by volume ID
if allVolumes[i].Type == allVolumes[j].Type {
var idI, idJ uint32
if allVolumes[i].Type == "regular" {
idI = allVolumes[i].RegularVolume.Id
} else {
idI = allVolumes[i].EcVolume.VolumeID
}
if allVolumes[j].Type == "regular" {
idJ = allVolumes[j].RegularVolume.Id
} else {
idJ = allVolumes[j].EcVolume.VolumeID
}
less = idI < idJ
} else {
less = allVolumes[i].Type < allVolumes[j].Type // "ec" < "regular"
}
default:
// Default to volume ID sort
var idI, idJ uint32
if allVolumes[i].Type == "regular" {
idI = allVolumes[i].RegularVolume.Id
} else {
idI = allVolumes[i].EcVolume.VolumeID
}
if allVolumes[j].Type == "regular" {
idJ = allVolumes[j].RegularVolume.Id
} else {
idJ = allVolumes[j].EcVolume.VolumeID
}
less = idI < idJ
}
if sortOrder == "desc" {
return !less
}
return less
})
// Apply pagination
totalVolumesAndEc := len(allVolumes)
totalPages := (totalVolumesAndEc + pageSize - 1) / pageSize
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if endIndex > totalVolumesAndEc {
endIndex = totalVolumesAndEc
}
if startIndex >= totalVolumesAndEc {
startIndex = 0
endIndex = 0
}
// Extract paginated results
var paginatedRegularVolumes []VolumeWithTopology
var paginatedEcVolumes []EcVolumeWithShards
for i := startIndex; i < endIndex; i++ {
if allVolumes[i].Type == "regular" {
paginatedRegularVolumes = append(paginatedRegularVolumes, *allVolumes[i].RegularVolume)
} else {
paginatedEcVolumes = append(paginatedEcVolumes, *allVolumes[i].EcVolume)
}
}
// Convert maps to slices
var dcList []string
for dc := range dataCenters {
dcList = append(dcList, dc)
}
sort.Strings(dcList)
var diskTypeList []string
for diskType := range diskTypes {
diskTypeList = append(diskTypeList, diskType)
}
sort.Strings(diskTypeList)
return &CollectionDetailsData{
CollectionName: collectionName,
RegularVolumes: paginatedRegularVolumes,
EcVolumes: paginatedEcVolumes,
TotalVolumes: len(regularVolumes),
TotalEcVolumes: len(ecVolumes),
TotalChunks: totalChunks,
TotalSize: totalSize,
DataCenters: dcList,
DiskTypes: diskTypeList,
LastUpdated: time.Now(),
Page: page,
PageSize: pageSize,
TotalPages: totalPages,
SortBy: sortBy,
SortOrder: sortOrder,
}, nil
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,313 @@
package dash
import (
"fmt"
"math"
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)
// TomlConfig is the subset of viper used to read admin.toml values.
type TomlConfig interface {
IsSet(key string) bool
GetBool(key string) bool
GetInt(key string) int
GetFloat64(key string) float64
GetString(key string) string
GetStringSlice(key string) []string
}
// ApplyMaintenanceConfigFromToml writes maintenance task settings declared in
// admin.toml through to the persisted task configs, so they survive data
// directory loss and override admin UI edits on restart. Absent keys keep
// their persisted values.
func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error {
var maintenanceConf *MaintenanceConfig
maintenanceChanged := false
if k := "maintenance.enabled"; v.IsSet(k) {
conf, err := cp.LoadMaintenanceConfig()
if err != nil {
return fmt.Errorf("load maintenance config: %w", err)
}
conf.Enabled = proto.Bool(v.GetBool(k))
// the policy lives in the per-task config files; don't snapshot it here
conf.Policy = nil
maintenanceConf = conf
maintenanceChanged = true
}
vacuumConf := vacuum.LoadConfigFromPersistence(cp)
vacuumChanged := applyBaseConfigFromToml(v, "maintenance.vacuum.", &vacuumConf.BaseConfig)
if k := "maintenance.vacuum.garbage_threshold"; v.IsSet(k) {
vacuumConf.GarbageThreshold = v.GetFloat64(k)
vacuumChanged = true
}
if k := "maintenance.vacuum.min_volume_age_seconds"; v.IsSet(k) {
vacuumConf.MinVolumeAgeSeconds = v.GetInt(k)
vacuumChanged = true
}
balanceConf := balance.LoadConfigFromPersistence(cp)
balanceChanged := applyBaseConfigFromToml(v, "maintenance.balance.", &balanceConf.BaseConfig)
if k := "maintenance.balance.imbalance_threshold"; v.IsSet(k) {
balanceConf.ImbalanceThreshold = v.GetFloat64(k)
balanceChanged = true
}
if k := "maintenance.balance.min_server_count"; v.IsSet(k) {
balanceConf.MinServerCount = v.GetInt(k)
balanceChanged = true
}
ecConf := erasure_coding.LoadConfigFromPersistence(cp)
ecChanged := applyBaseConfigFromToml(v, "maintenance.erasure_coding.", &ecConf.BaseConfig)
if k := "maintenance.erasure_coding.fullness_ratio"; v.IsSet(k) {
ecConf.FullnessRatio = v.GetFloat64(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.quiet_for_seconds"; v.IsSet(k) {
ecConf.QuietForSeconds = v.GetInt(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.min_size_mb"; v.IsSet(k) {
ecConf.MinSizeMB = v.GetInt(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.collection_filter"; v.IsSet(k) {
ecConf.CollectionFilter = v.GetString(k)
ecChanged = true
}
if k := "maintenance.erasure_coding.preferred_tags"; v.IsSet(k) {
// viper does not split comma-separated values from env vars
var tags []string
for _, tag := range v.GetStringSlice(k) {
tags = append(tags, strings.Split(tag, ",")...)
}
ecConf.PreferredTags = util.NormalizeTagList(tags)
ecChanged = true
}
if k := "maintenance.erasure_coding.replica_placement"; v.IsSet(k) {
ecConf.ReplicaPlacement = v.GetString(k)
ecChanged = true
}
if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged {
return nil
}
if !cp.IsConfigured() {
return fmt.Errorf("admin.toml maintenance settings require -dataDir to persist")
}
if maintenanceChanged {
if err := cp.SaveMaintenanceConfig(maintenanceConf); err != nil {
return fmt.Errorf("save maintenance config: %w", err)
}
glog.V(0).Infof("Applied [maintenance] settings from admin.toml (enabled: %v)", maintenanceConf.GetEnabled())
}
if vacuumChanged {
if err := cp.SaveVacuumTaskPolicy(vacuumConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save vacuum task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.vacuum] settings from admin.toml")
}
if balanceChanged {
if err := cp.SaveBalanceTaskPolicy(balanceConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save balance task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.balance] settings from admin.toml")
}
if ecChanged {
if err := cp.SaveErasureCodingTaskPolicy(ecConf.ToTaskPolicy()); err != nil {
return fmt.Errorf("save erasure coding task config: %w", err)
}
glog.V(0).Infof("Applied [maintenance.erasure_coding] settings from admin.toml")
}
return nil
}
func applyBaseConfigFromToml(v TomlConfig, prefix string, c *base.BaseConfig) bool {
changed := false
if k := prefix + "enabled"; v.IsSet(k) {
c.Enabled = v.GetBool(k)
changed = true
}
if k := prefix + "scan_interval_seconds"; v.IsSet(k) {
c.ScanIntervalSeconds = v.GetInt(k)
changed = true
}
if k := prefix + "max_concurrent"; v.IsSet(k) {
c.MaxConcurrent = v.GetInt(k)
changed = true
}
return changed
}
// ApplyPluginConfigFromToml overlays admin.toml [maintenance.*] settings onto
// existing plugin job type configs. Configs that do not exist yet get the
// overlay in applyPluginTomlDefaults when the plugin bootstraps them from a
// worker descriptor, so descriptor defaults are never lost.
func (s *AdminServer) ApplyPluginConfigFromToml(v TomlConfig) error {
plugin := s.GetPlugin()
if plugin == nil {
return nil
}
for _, section := range pluginConfigSections {
cfg, err := plugin.LoadJobTypeConfig(section.jobType)
if err != nil {
return fmt.Errorf("load %s plugin config: %w", section.jobType, err)
}
if cfg == nil {
continue
}
if !section.applyToml(v, cfg) {
continue
}
if err := plugin.SaveJobTypeConfig(cfg); err != nil {
return fmt.Errorf("save %s plugin config: %w", section.jobType, err)
}
glog.V(0).Infof("Applied [%s] settings from admin.toml to plugin config", section.prefix)
}
return nil
}
// applyPluginTomlDefaults overlays admin.toml onto a job type config the
// plugin is bootstrapping from descriptor defaults.
func applyPluginTomlDefaults(v TomlConfig, cfg *plugin_pb.PersistedJobTypeConfig) {
for _, section := range pluginConfigSections {
if section.jobType == cfg.JobType {
if section.applyToml(v, cfg) {
glog.V(0).Infof("Applied [%s] settings from admin.toml to plugin defaults", section.prefix)
}
return
}
}
}
type pluginConfigSection struct {
jobType string
prefix string
workerKeys map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue
adminKeys map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue
}
var pluginConfigSections = []pluginConfigSection{
{
jobType: "vacuum",
prefix: "maintenance.vacuum",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"garbage_threshold": doubleValue,
"min_volume_age_seconds": int64Value,
},
},
{
jobType: "volume_balance",
prefix: "maintenance.balance",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"imbalance_threshold": doubleValue,
"min_server_count": int64Value,
},
},
{
jobType: "erasure_coding",
prefix: "maintenance.erasure_coding",
workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"fullness_ratio": doubleValue,
"quiet_for_seconds": int64Value,
"min_size_mb": int64Value,
"preferred_tags": stringListValue,
"replica_placement": stringValue,
},
// workers read collection_filter from the admin values, not the worker values
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
"collection_filter": stringValue,
},
},
}
func doubleValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: v.GetFloat64(key)}}
}
func int64Value(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(v.GetInt(key))}}
}
func stringValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_StringValue{StringValue: v.GetString(key)}}
}
func stringListValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
// viper does not split comma-separated values from env vars
var items []string
for _, item := range v.GetStringSlice(key) {
items = append(items, strings.Split(item, ",")...)
}
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_StringList{StringList: &plugin_pb.StringList{Values: util.NormalizeTagList(items)}}}
}
func (section pluginConfigSection) applyToml(v TomlConfig, cfg *plugin_pb.PersistedJobTypeConfig) bool {
changed := false
if k := section.prefix + ".enabled"; v.IsSet(k) {
ensureAdminRuntime(cfg).Enabled = v.GetBool(k)
changed = true
}
if k := section.prefix + ".retry_limit"; v.IsSet(k) {
ensureAdminRuntime(cfg).RetryLimit = int32Setting(v, k)
changed = true
}
if k := section.prefix + ".retry_backoff_seconds"; v.IsSet(k) {
ensureAdminRuntime(cfg).RetryBackoffSeconds = int32Setting(v, k)
changed = true
}
for tomlKey, mapper := range section.workerKeys {
if k := section.prefix + "." + tomlKey; v.IsSet(k) {
if cfg.WorkerConfigValues == nil {
cfg.WorkerConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.WorkerConfigValues[tomlKey] = mapper(v, k)
changed = true
}
}
for tomlKey, mapper := range section.adminKeys {
if k := section.prefix + "." + tomlKey; v.IsSet(k) {
if cfg.AdminConfigValues == nil {
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.AdminConfigValues[tomlKey] = mapper(v, k)
changed = true
}
}
if changed {
cfg.UpdatedAt = timestamppb.Now()
cfg.UpdatedBy = "admin.toml"
}
return changed
}
// int32Setting clamps to [0, MaxInt32] so oversized toml values cannot wrap negative
func int32Setting(v TomlConfig, key string) int32 {
n := v.GetInt(key)
if n < 0 {
return 0
}
if n > math.MaxInt32 {
return math.MaxInt32
}
return int32(n)
}
func ensureAdminRuntime(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.AdminRuntimeConfig {
if cfg.AdminRuntime == nil {
// maintenance job types are enabled by default; a bare runtime must not disable them
cfg.AdminRuntime = &plugin_pb.AdminRuntimeConfig{Enabled: true}
}
return cfg.AdminRuntime
}

View File

@@ -0,0 +1,309 @@
package dash
import (
"math"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"github.com/spf13/viper"
)
func tomlConfig(t *testing.T, content string) *viper.Viper {
t.Helper()
v := viper.New()
v.SetConfigType("toml")
if err := v.ReadConfig(strings.NewReader(content)); err != nil {
t.Fatalf("read toml: %v", err)
}
return v
}
func TestApplyMaintenanceConfigFromToml(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
// simulate an earlier UI edit
saved := vacuum.NewDefaultConfig()
saved.MaxConcurrent = 5
saved.GarbageThreshold = 0.5
if err := cp.SaveVacuumTaskPolicy(saved.ToTaskPolicy()); err != nil {
t.Fatalf("save vacuum policy: %v", err)
}
balanceBefore := balance.LoadConfigFromPersistence(cp)
v := tomlConfig(t, `
[maintenance.vacuum]
garbage_threshold = 0.03
min_volume_age_seconds = 1800
[maintenance.erasure_coding]
enabled = false
fullness_ratio = 0.8
preferred_tags = "Fast, ssd"
`)
if err := cp.ApplyMaintenanceConfigFromToml(v); err != nil {
t.Fatalf("apply: %v", err)
}
vacuumConf := vacuum.LoadConfigFromPersistence(cp)
if vacuumConf.GarbageThreshold != 0.03 {
t.Errorf("garbage threshold = %v, want 0.03", vacuumConf.GarbageThreshold)
}
// sub-hour values round up to a whole hour instead of truncating to 0
if vacuumConf.MinVolumeAgeSeconds != 3600 {
t.Errorf("min volume age = %v, want 3600", vacuumConf.MinVolumeAgeSeconds)
}
if vacuumConf.MaxConcurrent != 5 {
t.Errorf("max concurrent = %v, want UI-saved 5 preserved", vacuumConf.MaxConcurrent)
}
ecConf := erasure_coding.LoadConfigFromPersistence(cp)
if ecConf.Enabled {
t.Errorf("ec enabled = true, want false")
}
if ecConf.FullnessRatio != 0.8 {
t.Errorf("fullness ratio = %v, want 0.8", ecConf.FullnessRatio)
}
if !reflect.DeepEqual(ecConf.PreferredTags, []string{"fast", "ssd"}) {
t.Errorf("preferred tags = %v, want [fast ssd]", ecConf.PreferredTags)
}
if ecConf.QuietForSeconds != 3600 {
t.Errorf("quiet for = %v, want default 3600 preserved", ecConf.QuietForSeconds)
}
// balance section absent: nothing written
if _, err := os.Stat(filepath.Join(dir, ConfigSubdir, BalanceTaskConfigFile)); !os.IsNotExist(err) {
t.Errorf("balance config written without [maintenance.balance] section")
}
if got := balance.LoadConfigFromPersistence(cp); !reflect.DeepEqual(got, balanceBefore) {
t.Errorf("balance config changed without [maintenance.balance] section")
}
}
func TestApplyMaintenanceConfigFromTomlEnabledToggle(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, "[maintenance]\nenabled = false\n")); err != nil {
t.Fatalf("apply: %v", err)
}
conf, err := cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if conf.GetEnabled() {
t.Errorf("maintenance still enabled after [maintenance] enabled = false")
}
if conf.ScanIntervalSeconds != 30*60 {
t.Errorf("scan interval = %d, want default 1800 kept alongside the toggle", conf.ScanIntervalSeconds)
}
if err := cp.ApplyMaintenanceConfigFromToml(tomlConfig(t, "[maintenance]\nenabled = true\n")); err != nil {
t.Fatalf("apply: %v", err)
}
conf, err = cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if !conf.GetEnabled() {
t.Errorf("maintenance still disabled after [maintenance] enabled = true")
}
}
func TestApplyMaintenanceConfigFromTomlNoKeys(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
if err := cp.ApplyMaintenanceConfigFromToml(viper.New()); err != nil {
t.Fatalf("apply: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, ConfigSubdir)); !os.IsNotExist(err) {
t.Errorf("conf dir created without any maintenance keys set")
}
}
func TestApplyMaintenanceConfigFromTomlRequiresDataDir(t *testing.T) {
cp := NewConfigPersistence("")
v := tomlConfig(t, `
[maintenance.vacuum]
garbage_threshold = 0.03
`)
if err := cp.ApplyMaintenanceConfigFromToml(v); err == nil {
t.Errorf("expected error when maintenance keys are set without a data dir")
}
}
func newPluginServer(t *testing.T) *AdminServer {
t.Helper()
p, err := adminplugin.New(adminplugin.Options{})
if err != nil {
t.Fatalf("new plugin: %v", err)
}
t.Cleanup(p.Shutdown)
return &AdminServer{plugin: p}
}
func TestApplyPluginConfigFromTomlUpdatesExistingConfig(t *testing.T) {
s := newPluginServer(t)
seed := &plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, RetryLimit: 1, RetryBackoffSeconds: 10},
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{
"garbage_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3}},
},
}
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
t.Fatalf("seed: %v", err)
}
v := tomlConfig(t, `
[maintenance.vacuum]
garbage_threshold = 0.1
retry_limit = 5
`)
if err := s.ApplyPluginConfigFromToml(v); err != nil {
t.Fatalf("apply: %v", err)
}
cfg, err := s.GetPlugin().LoadJobTypeConfig("vacuum")
if err != nil || cfg == nil {
t.Fatalf("load: %v %v", cfg, err)
}
if got := cfg.WorkerConfigValues["garbage_threshold"].GetDoubleValue(); got != 0.1 {
t.Errorf("garbage_threshold = %v, want 0.1", got)
}
if cfg.AdminRuntime.RetryLimit != 5 {
t.Errorf("retry_limit = %d, want 5", cfg.AdminRuntime.RetryLimit)
}
if !cfg.AdminRuntime.Enabled {
t.Errorf("enabled flipped off by unrelated toml keys")
}
if cfg.AdminRuntime.RetryBackoffSeconds != 10 {
t.Errorf("retry_backoff_seconds = %d, want 10 kept", cfg.AdminRuntime.RetryBackoffSeconds)
}
if cfg.UpdatedBy != "admin.toml" {
t.Errorf("updated_by = %q, want admin.toml", cfg.UpdatedBy)
}
}
func TestApplyPluginConfigFromTomlClampsRetryValues(t *testing.T) {
s := newPluginServer(t)
seed := &plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
}
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
t.Fatalf("seed: %v", err)
}
v := tomlConfig(t, `
[maintenance.vacuum]
retry_limit = 68719476736
retry_backoff_seconds = -5
`)
if err := s.ApplyPluginConfigFromToml(v); err != nil {
t.Fatalf("apply: %v", err)
}
cfg, err := s.GetPlugin().LoadJobTypeConfig("vacuum")
if err != nil || cfg == nil {
t.Fatalf("load: %v %v", cfg, err)
}
wantLimit := int32(math.MaxInt32)
if strconv.IntSize == 32 {
wantLimit = 0 // viper zeroes values that overflow int before the clamp sees them
}
if cfg.AdminRuntime.RetryLimit != wantLimit {
t.Errorf("retry_limit = %d, want clamped to %d", cfg.AdminRuntime.RetryLimit, wantLimit)
}
if cfg.AdminRuntime.RetryBackoffSeconds != 0 {
t.Errorf("retry_backoff_seconds = %d, want clamped to 0", cfg.AdminRuntime.RetryBackoffSeconds)
}
}
func TestApplyPluginConfigFromTomlLeavesMissingConfigToBootstrap(t *testing.T) {
s := newPluginServer(t)
v := tomlConfig(t, `
[maintenance.vacuum]
garbage_threshold = 0.1
`)
if err := s.ApplyPluginConfigFromToml(v); err != nil {
t.Fatalf("apply: %v", err)
}
if cfg, _ := s.GetPlugin().LoadJobTypeConfig("vacuum"); cfg != nil {
t.Errorf("bare config created; descriptor bootstrap would be skipped and disable the job type")
}
}
func TestApplyPluginTomlDefaultsOverlaysBootstrapConfig(t *testing.T) {
cfg := &plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, RetryLimit: 1},
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{
"garbage_threshold": {Kind: &plugin_pb.ConfigValue_DoubleValue{DoubleValue: 0.3}},
},
UpdatedBy: "plugin",
}
v := tomlConfig(t, `
[maintenance.vacuum]
garbage_threshold = 0.1
`)
applyPluginTomlDefaults(v, cfg)
if got := cfg.WorkerConfigValues["garbage_threshold"].GetDoubleValue(); got != 0.1 {
t.Errorf("garbage_threshold = %v, want 0.1", got)
}
if !cfg.AdminRuntime.Enabled || cfg.AdminRuntime.RetryLimit != 1 {
t.Errorf("descriptor defaults lost: %v", cfg.AdminRuntime)
}
if cfg.UpdatedBy != "admin.toml" {
t.Errorf("updated_by = %q, want admin.toml", cfg.UpdatedBy)
}
}
func TestApplyPluginConfigFromTomlErasureCodingKeyPlacement(t *testing.T) {
s := newPluginServer(t)
seed := &plugin_pb.PersistedJobTypeConfig{
JobType: "erasure_coding",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
}
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
t.Fatalf("seed: %v", err)
}
v := tomlConfig(t, `
[maintenance.erasure_coding]
collection_filter = "important"
replica_placement = "020"
preferred_tags = ["fast,SSD"]
`)
if err := s.ApplyPluginConfigFromToml(v); err != nil {
t.Fatalf("apply: %v", err)
}
cfg, err := s.GetPlugin().LoadJobTypeConfig("erasure_coding")
if err != nil || cfg == nil {
t.Fatalf("load: %v %v", cfg, err)
}
// collection_filter is read from the admin values by all workers
if got := cfg.AdminConfigValues["collection_filter"].GetStringValue(); got != "important" {
t.Errorf("admin collection_filter = %q, want important", got)
}
if _, ok := cfg.WorkerConfigValues["collection_filter"]; ok {
t.Errorf("collection_filter must not land in worker values")
}
if got := cfg.WorkerConfigValues["replica_placement"].GetStringValue(); got != "020" {
t.Errorf("replica_placement = %q, want 020", got)
}
if got := cfg.WorkerConfigValues["preferred_tags"].GetStringList().GetValues(); !reflect.DeepEqual(got, []string{"fast", "ssd"}) {
t.Errorf("preferred_tags = %v, want [fast ssd]", got)
}
}

View File

@@ -0,0 +1,91 @@
package dash
import (
"context"
"github.com/a-h/templ"
)
type contextKey string
const (
contextUsernameKey contextKey = "admin.username"
contextRoleKey contextKey = "admin.role"
contextCSRFKey contextKey = "admin.csrf"
contextURLPrefixKey contextKey = "admin.urlprefix"
)
// WithAuthContext stores auth metadata on the request context.
func WithAuthContext(ctx context.Context, username, role, csrfToken string) context.Context {
if username != "" {
ctx = context.WithValue(ctx, contextUsernameKey, username)
}
if role != "" {
ctx = context.WithValue(ctx, contextRoleKey, role)
}
if csrfToken != "" {
ctx = context.WithValue(ctx, contextCSRFKey, csrfToken)
}
return ctx
}
// UsernameFromContext retrieves the username from context.
func UsernameFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
if value, ok := ctx.Value(contextUsernameKey).(string); ok {
return value
}
return ""
}
// RoleFromContext retrieves the role from context.
func RoleFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
if value, ok := ctx.Value(contextRoleKey).(string); ok {
return value
}
return ""
}
// CSRFTokenFromContext retrieves the CSRF token from context.
func CSRFTokenFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
if value, ok := ctx.Value(contextCSRFKey).(string); ok {
return value
}
return ""
}
// WithURLPrefix stores the URL prefix on the context.
func WithURLPrefix(ctx context.Context, prefix string) context.Context {
return context.WithValue(ctx, contextURLPrefixKey, prefix)
}
// URLPrefixFromContext retrieves the URL prefix from context.
func URLPrefixFromContext(ctx context.Context) string {
if ctx == nil {
return ""
}
if value, ok := ctx.Value(contextURLPrefixKey).(string); ok {
return value
}
return ""
}
// P returns the URL prefix prepended to the given path.
// Use in Go handlers for redirect URLs.
func P(ctx context.Context, path string) string {
return URLPrefixFromContext(ctx) + path
}
// PUrl returns the URL prefix prepended to the given path as a templ.SafeURL.
// Use in templ templates for href attributes.
func PUrl(ctx context.Context, path string) templ.SafeURL {
return templ.SafeURL(URLPrefixFromContext(ctx) + path)
}

96
weed/admin/dash/csrf.go Normal file
View File

@@ -0,0 +1,96 @@
package dash
import (
"crypto/rand"
"crypto/subtle"
"encoding/hex"
"fmt"
"net/http"
"github.com/gorilla/sessions"
)
const sessionCSRFTokenKey = "csrf_token"
func generateCSRFToken() (string, error) {
tokenBytes := make([]byte, 32)
if _, err := rand.Read(tokenBytes); err != nil {
return "", err
}
return hex.EncodeToString(tokenBytes), nil
}
func getOrCreateSessionCSRFToken(session *sessions.Session, r *http.Request, w http.ResponseWriter) (string, error) {
if existing, ok := session.Values[sessionCSRFTokenKey].(string); ok && existing != "" {
return existing, nil
}
token, err := generateCSRFToken()
if err != nil {
return "", err
}
session.Values[sessionCSRFTokenKey] = token
if err := session.Save(r, w); err != nil {
return "", err
}
return token, nil
}
func requireSessionCSRFToken(w http.ResponseWriter, r *http.Request) bool {
expectedToken := CSRFTokenFromContext(r.Context())
username := UsernameFromContext(r.Context())
if expectedToken == "" {
// Admin UI can run without auth; in that mode CSRF token checks are not applicable.
if username == "" {
return true
}
writeJSONError(w, http.StatusForbidden, "missing CSRF session token")
return false
}
providedToken, err := getProvidedCSRFToken(r)
if err != nil {
writeJSONError(w, http.StatusBadRequest, "Failed to parse form: "+err.Error())
return false
}
if providedToken == "" || subtle.ConstantTimeCompare([]byte(expectedToken), []byte(providedToken)) != 1 {
writeJSONError(w, http.StatusForbidden, "invalid CSRF token")
return false
}
return true
}
func getProvidedCSRFToken(r *http.Request) (string, error) {
providedToken := r.Header.Get("X-CSRF-Token")
if providedToken != "" {
return providedToken, nil
}
if err := r.ParseForm(); err != nil {
return "", err
}
return r.FormValue("csrf_token"), nil
}
func EnsureSessionCSRFToken(session *sessions.Session, r *http.Request, w http.ResponseWriter) (string, error) {
if session == nil {
return "", fmt.Errorf("session is nil")
}
return getOrCreateSessionCSRFToken(session, r, w)
}
func ValidateSessionCSRFToken(session *sessions.Session, r *http.Request) error {
if session == nil {
return fmt.Errorf("session is nil")
}
expectedToken, _ := session.Values[sessionCSRFTokenKey].(string)
providedToken, err := getProvidedCSRFToken(r)
if err != nil {
return fmt.Errorf("failed to read CSRF token: %w", err)
}
if expectedToken == "" {
return fmt.Errorf("missing session CSRF token")
}
if providedToken == "" || subtle.ConstantTimeCompare([]byte(expectedToken), []byte(providedToken)) != 1 {
return fmt.Errorf("invalid CSRF token")
}
return nil
}

View File

@@ -0,0 +1,182 @@
package dash
import (
"fmt"
"strings"
"time"
)
// dashMaxSamples bounds the in-memory trend ring buffer. At the 15s sample
// cadence (piggy-backed on publishMaintenanceMetrics) this is ~15 minutes.
const dashMaxSamples = 60
// dashSample is one point-in-time snapshot of a few headline cluster numbers,
// derived from data the admin already holds (cluster topology + the in-process
// maintenance queue) — no Prometheus scrape required.
type dashSample struct {
t time.Time
volumes float64
ecShards float64
diskUsed float64
chunks float64
tasks float64 // pending/assigned/in-progress maintenance tasks
workers float64
}
// DashboardTrends carries inline-SVG sparklines of recent cluster history,
// keyed to the dashboard's existing summary cards so each card shows a value
// plus its trend (rather than a separate, duplicate row). Maintenance metrics
// that have no existing card carry their current value too, and fill the
// previously-empty columns of the EC row.
type DashboardTrends struct {
Samples int `json:"samples"`
// Sparklines (raw <svg>) for the existing summary cards.
Volumes string `json:"-"`
Chunks string `json:"-"`
DiskUsed string `json:"-"`
EcShards string `json:"-"`
// Maintenance cards: value + sparkline.
Tasks string `json:"-"`
TasksValue string `json:"tasks"`
Workers string `json:"-"`
WorkersValue string `json:"workers"`
}
// recordDashboardSample snapshots headline cluster numbers into the ring
// buffer. Cheap: topology is already cached, and the maintenance stats are
// in-memory. Called on the existing maintenance-metrics ticker.
func (s *AdminServer) recordDashboardSample() {
topology, err := s.GetClusterTopology()
if err != nil || topology == nil {
return
}
ecShards := 0
for _, vs := range topology.VolumeServers {
ecShards += vs.EcShards
}
sample := dashSample{
t: time.Now(),
volumes: float64(topology.TotalVolumes),
ecShards: float64(ecShards),
diskUsed: float64(topology.TotalSize),
chunks: float64(topology.TotalChunks),
}
if s.maintenanceManager != nil {
if stats := s.maintenanceManager.GetStats(); stats != nil {
active := 0
for status, n := range stats.TasksByStatus {
switch string(status) {
case "pending", "assigned", "in_progress":
active += n
}
}
sample.tasks = float64(active)
}
}
// Counted across both worker registries, same as the Prometheus gauge, so
// the card isn't stuck at 0 on clusters that only run plugin workers.
workers, _, _ := s.workerFleetTotals()
sample.workers = float64(workers)
s.dashSamplesMu.Lock()
s.dashSamples = append(s.dashSamples, sample)
if len(s.dashSamples) > dashMaxSamples {
s.dashSamples = s.dashSamples[len(s.dashSamples)-dashMaxSamples:]
}
s.dashSamplesMu.Unlock()
}
// GetDashboardTrends builds the trend cards from the current ring buffer.
func (s *AdminServer) GetDashboardTrends() DashboardTrends {
s.dashSamplesMu.Lock()
samples := make([]dashSample, len(s.dashSamples))
copy(samples, s.dashSamples)
s.dashSamplesMu.Unlock()
series := func(pick func(dashSample) float64) []float64 {
out := make([]float64, len(samples))
for i, smp := range samples {
out[i] = pick(smp)
}
return out
}
tasks := series(func(s dashSample) float64 { return s.tasks })
workers := series(func(s dashSample) float64 { return s.workers })
// Sparkline colors match the existing cards' border colors.
return DashboardTrends{
Samples: len(samples),
Volumes: sparklineSVG(series(func(s dashSample) float64 { return s.volumes }), "#1cc88a"), // success
Chunks: sparklineSVG(series(func(s dashSample) float64 { return s.chunks }), "#36b9cc"), // info
DiskUsed: sparklineSVG(series(func(s dashSample) float64 { return s.diskUsed }), "#f6c23e"), // warning
EcShards: sparklineSVG(series(func(s dashSample) float64 { return s.ecShards }), "#5a5c69"), // dark
Tasks: sparklineSVG(tasks, "#36b9cc"),
TasksValue: trendCount(last(tasks)),
Workers: sparklineSVG(workers, "#4e73df"),
WorkersValue: trendCount(last(workers)),
}
}
func last(v []float64) float64 {
if len(v) == 0 {
return 0
}
return v[len(v)-1]
}
// sparklineSVG renders a fixed-viewBox, width-responsive inline SVG line of the
// given points. Self-contained (no JS/chart lib); safe to inline in the page.
func sparklineSVG(pts []float64, color string) string {
const w, h = 240.0, 48.0
if len(pts) < 2 {
// Not enough history yet — draw a flat baseline so the card isn't empty.
return fmt.Sprintf(`<svg viewBox="0 0 %g %g" preserveAspectRatio="none" style="width:100%%;height:48px"><line x1="0" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="2" opacity="0.4"/></svg>`, w, h, h/2, w, h/2, color)
}
minV, maxV := pts[0], pts[0]
for _, v := range pts {
if v < minV {
minV = v
}
if v > maxV {
maxV = v
}
}
span := maxV - minV
if span == 0 {
span = 1
}
dx := w / float64(len(pts)-1)
var line strings.Builder
for i, v := range pts {
x := float64(i) * dx
y := h - 3 - (v-minV)/span*(h-6) // 3px padding top/bottom; SVG y grows down
if i == 0 {
fmt.Fprintf(&line, "M%.1f %.1f", x, y)
} else {
fmt.Fprintf(&line, " L%.1f %.1f", x, y)
}
}
// Area path closes back along the baseline for a subtle fill.
area := fmt.Sprintf("%s L%.1f %.1f L0 %.1f Z", line.String(), w, h, h)
return fmt.Sprintf(`<svg viewBox="0 0 %g %g" preserveAspectRatio="none" style="width:100%%;height:48px"><path d="%s" fill="%s" opacity="0.12"/><path d="%s" fill="none" stroke="%s" stroke-width="2"/></svg>`,
w, h, area, color, line.String(), color)
}
// trendCount formats a count with thousands separators.
func trendCount(v float64) string {
n := int64(v)
s := fmt.Sprintf("%d", n)
if n < 0 {
return s
}
var out []byte
for i, c := range []byte(s) {
if i > 0 && (len(s)-i)%3 == 0 {
out = append(out, ',')
}
out = append(out, c)
}
return string(out)
}

View File

@@ -0,0 +1,683 @@
package dash
import (
"context"
"fmt"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// matchesCollection checks if a volume/EC volume collection matches the filter collection.
// Handles the special case where empty collection ("") represents the "default" collection.
func matchesCollection(volumeCollection, filterCollection string) bool {
// Handle special case where "default" filter matches empty collection
if filterCollection == "default" && volumeCollection == "" {
return true
}
// Direct string match for named collections
return volumeCollection == filterCollection
}
// GetClusterEcShards retrieves cluster EC shards data with pagination, sorting, and filtering
func (s *AdminServer) GetClusterEcShards(page int, pageSize int, sortBy string, sortOrder string, collection string) (*ClusterEcShardsData, error) {
// Set defaults
if page < 1 {
page = 1
}
if pageSize < 1 || pageSize > 1000 {
pageSize = 100
}
if sortBy == "" {
sortBy = "volume_id"
}
if sortOrder == "" {
sortOrder = "asc"
}
var ecShards []EcShardWithInfo
volumeShardsMap := make(map[uint32]map[int]bool) // volumeId -> set of shards present
volumesWithAllShards := 0
volumesWithMissingShards := 0
// Get detailed EC shard information via gRPC
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
// Process EC shard information
for _, ecShardInfo := range diskInfo.EcShardInfos {
volumeId := ecShardInfo.Id
// Initialize volume shards map if needed
if volumeShardsMap[volumeId] == nil {
volumeShardsMap[volumeId] = make(map[int]bool)
}
// Create individual shard entries for each shard this server has
shardBits := ecShardInfo.EcIndexBits
shardsInfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
if (shardBits & (1 << uint(shardId))) != 0 {
// Mark this shard as present for this volume
volumeShardsMap[volumeId][shardId] = true
ecShard := EcShardWithInfo{
VolumeID: volumeId,
ShardID: uint32(shardId),
Collection: ecShardInfo.Collection,
Size: uint64(shardsInfo.Size(erasure_coding.ShardId(shardId))),
Server: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
DiskType: diskInfo.Type,
ModifiedTime: 0, // Not available in current API
EcIndexBits: ecShardInfo.EcIndexBits,
ShardCount: getShardCount(ecShardInfo.EcIndexBits),
}
ecShards = append(ecShards, ecShard)
}
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
// Calculate volume-level completeness (across all servers)
volumeCompleteness := make(map[uint32]bool)
volumeMissingShards := make(map[uint32][]int)
for volumeId, shardsPresent := range volumeShardsMap {
var missingShards []int
shardCount := len(shardsPresent)
// Find which shards are missing for this volume across ALL servers
// Uses default 10+4 (14 total shards)
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
if !shardsPresent[shardId] {
missingShards = append(missingShards, shardId)
}
}
isComplete := (shardCount == erasure_coding.TotalShardsCount)
volumeCompleteness[volumeId] = isComplete
volumeMissingShards[volumeId] = missingShards
if isComplete {
volumesWithAllShards++
} else {
volumesWithMissingShards++
}
}
// Update completeness info for each shard based on volume-level completeness
for i := range ecShards {
volumeId := ecShards[i].VolumeID
ecShards[i].IsComplete = volumeCompleteness[volumeId]
ecShards[i].MissingShards = volumeMissingShards[volumeId]
}
// Filter by collection if specified
if collection != "" {
var filteredShards []EcShardWithInfo
for _, shard := range ecShards {
if shard.Collection == collection {
filteredShards = append(filteredShards, shard)
}
}
ecShards = filteredShards
}
// Sort the results
sortEcShards(ecShards, sortBy, sortOrder)
// Calculate statistics for conditional display
dataCenters := make(map[string]bool)
racks := make(map[string]bool)
collections := make(map[string]bool)
for _, shard := range ecShards {
dataCenters[shard.DataCenter] = true
racks[shard.Rack] = true
if shard.Collection != "" {
collections[shard.Collection] = true
}
}
// Pagination
totalShards := len(ecShards)
totalPages := (totalShards + pageSize - 1) / pageSize
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if endIndex > totalShards {
endIndex = totalShards
}
if startIndex >= totalShards {
startIndex = 0
endIndex = 0
}
paginatedShards := ecShards[startIndex:endIndex]
// Build response
data := &ClusterEcShardsData{
EcShards: paginatedShards,
TotalShards: totalShards,
TotalVolumes: len(volumeShardsMap),
LastUpdated: time.Now(),
// Pagination
CurrentPage: page,
TotalPages: totalPages,
PageSize: pageSize,
// Sorting
SortBy: sortBy,
SortOrder: sortOrder,
// Statistics
DataCenterCount: len(dataCenters),
RackCount: len(racks),
CollectionCount: len(collections),
// Conditional display flags
ShowDataCenterColumn: len(dataCenters) > 1,
ShowRackColumn: len(racks) > 1,
ShowCollectionColumn: len(collections) > 1 || collection != "",
// Filtering
FilterCollection: collection,
// EC specific statistics
ShardsPerVolume: make(map[uint32]int), // This will be recalculated below
VolumesWithAllShards: volumesWithAllShards,
VolumesWithMissingShards: volumesWithMissingShards,
}
// Recalculate ShardsPerVolume for the response
for volumeId, shardsPresent := range volumeShardsMap {
data.ShardsPerVolume[volumeId] = len(shardsPresent)
}
// Set single values when only one exists
if len(dataCenters) == 1 {
for dc := range dataCenters {
data.SingleDataCenter = dc
break
}
}
if len(racks) == 1 {
for rack := range racks {
data.SingleRack = rack
break
}
}
if len(collections) == 1 {
for col := range collections {
data.SingleCollection = col
break
}
}
return data, nil
}
// GetClusterEcVolumes retrieves cluster EC volumes data grouped by volume ID with shard locations
func (s *AdminServer) GetClusterEcVolumes(page int, pageSize int, sortBy string, sortOrder string, collection string) (*ClusterEcVolumesData, error) {
// Set defaults
if page < 1 {
page = 1
}
if pageSize < 1 || pageSize > 1000 {
pageSize = 100
}
if sortBy == "" {
sortBy = "volume_id"
}
if sortOrder == "" {
sortOrder = "asc"
}
volumeData := make(map[uint32]*EcVolumeWithShards)
totalShards := 0
// Get detailed EC shard information via gRPC
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
// Process EC shard information
for _, ecShardInfo := range diskInfo.EcShardInfos {
volumeId := ecShardInfo.Id
// Initialize volume data if needed
if volumeData[volumeId] == nil {
volumeData[volumeId] = &EcVolumeWithShards{
VolumeID: volumeId,
Collection: ecShardInfo.Collection,
TotalShards: 0,
IsComplete: false,
MissingShards: []int{},
ShardLocations: make(map[int]string),
ShardSizes: make(map[int]int64),
DataCenters: []string{},
Servers: []string{},
Racks: []string{},
}
}
volume := volumeData[volumeId]
// Track data centers and servers
dcExists := false
for _, existingDc := range volume.DataCenters {
if existingDc == dc.Id {
dcExists = true
break
}
}
if !dcExists {
volume.DataCenters = append(volume.DataCenters, dc.Id)
}
serverExists := false
for _, existingServer := range volume.Servers {
if existingServer == node.Id {
serverExists = true
break
}
}
if !serverExists {
volume.Servers = append(volume.Servers, node.Id)
}
// Track racks
rackExists := false
for _, existingRack := range volume.Racks {
if existingRack == rack.Id {
rackExists = true
break
}
}
if !rackExists {
volume.Racks = append(volume.Racks, rack.Id)
}
// Process each shard this server has for this volume
shardBits := ecShardInfo.EcIndexBits
shardSizes := ecShardInfo.ShardSizes
sizeIndex := 0
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
if (shardBits & (1 << uint(shardId))) != 0 {
// Record shard location
volume.ShardLocations[shardId] = node.Id
if sizeIndex < len(shardSizes) {
size := shardSizes[sizeIndex]
if size >= 0 {
volume.ShardSizes[shardId] = size
}
}
sizeIndex++
totalShards++
}
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
// Calculate completeness for each volume
completeVolumes := 0
incompleteVolumes := 0
for _, volume := range volumeData {
volume.TotalShards = len(volume.ShardLocations)
// Find missing shards (default 10+4 = 14 total shards)
var missingShards []int
for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
if _, exists := volume.ShardLocations[shardId]; !exists {
missingShards = append(missingShards, shardId)
}
}
volume.MissingShards = missingShards
volume.IsComplete = (len(missingShards) == 0)
if volume.IsComplete {
completeVolumes++
} else {
incompleteVolumes++
}
}
// Convert map to slice
var ecVolumes []EcVolumeWithShards
for _, volume := range volumeData {
// Filter by collection if specified
if collection == "" || matchesCollection(volume.Collection, collection) {
ecVolumes = append(ecVolumes, *volume)
}
}
// Sort the results
sortEcVolumes(ecVolumes, sortBy, sortOrder)
// Calculate statistics for conditional display
dataCenters := make(map[string]bool)
collections := make(map[string]bool)
for _, volume := range ecVolumes {
for _, dc := range volume.DataCenters {
dataCenters[dc] = true
}
if volume.Collection != "" {
collections[volume.Collection] = true
}
}
// Pagination
totalVolumes := len(ecVolumes)
totalPages := (totalVolumes + pageSize - 1) / pageSize
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if endIndex > totalVolumes {
endIndex = totalVolumes
}
if startIndex >= totalVolumes {
startIndex = 0
endIndex = 0
}
paginatedVolumes := ecVolumes[startIndex:endIndex]
// Build response
data := &ClusterEcVolumesData{
EcVolumes: paginatedVolumes,
TotalVolumes: totalVolumes,
LastUpdated: time.Now(),
// Pagination
Page: page,
PageSize: pageSize,
TotalPages: totalPages,
// Sorting
SortBy: sortBy,
SortOrder: sortOrder,
// Filtering
Collection: collection,
// Conditional display flags
ShowDataCenterColumn: len(dataCenters) > 1,
ShowRackColumn: false, // We don't track racks in this view for simplicity
ShowCollectionColumn: len(collections) > 1 || collection != "",
// Statistics
CompleteVolumes: completeVolumes,
IncompleteVolumes: incompleteVolumes,
TotalShards: totalShards,
}
return data, nil
}
// sortEcVolumes sorts EC volumes based on the specified field and order
func sortEcVolumes(volumes []EcVolumeWithShards, sortBy string, sortOrder string) {
sort.Slice(volumes, func(i, j int) bool {
var less bool
switch sortBy {
case "volume_id":
less = volumes[i].VolumeID < volumes[j].VolumeID
case "collection":
if volumes[i].Collection == volumes[j].Collection {
less = volumes[i].VolumeID < volumes[j].VolumeID
} else {
less = volumes[i].Collection < volumes[j].Collection
}
case "total_shards":
if volumes[i].TotalShards == volumes[j].TotalShards {
less = volumes[i].VolumeID < volumes[j].VolumeID
} else {
less = volumes[i].TotalShards < volumes[j].TotalShards
}
case "completeness":
// Complete volumes first, then by volume ID
if volumes[i].IsComplete == volumes[j].IsComplete {
less = volumes[i].VolumeID < volumes[j].VolumeID
} else {
less = volumes[i].IsComplete && !volumes[j].IsComplete
}
default:
less = volumes[i].VolumeID < volumes[j].VolumeID
}
if sortOrder == "desc" {
return !less
}
return less
})
}
// getShardCount returns the number of shards represented by the bitmap
func getShardCount(ecIndexBits uint32) int {
count := 0
for i := 0; i < erasure_coding.MaxShardCount; i++ {
if (ecIndexBits & (1 << uint(i))) != 0 {
count++
}
}
return count
}
// sortEcShards sorts EC shards based on the specified field and order
func sortEcShards(shards []EcShardWithInfo, sortBy string, sortOrder string) {
sort.Slice(shards, func(i, j int) bool {
var less bool
switch sortBy {
case "shard_id":
less = shards[i].ShardID < shards[j].ShardID
case "server":
if shards[i].Server == shards[j].Server {
less = shards[i].ShardID < shards[j].ShardID // Secondary sort by shard ID
} else {
less = shards[i].Server < shards[j].Server
}
case "data_center":
if shards[i].DataCenter == shards[j].DataCenter {
less = shards[i].ShardID < shards[j].ShardID // Secondary sort by shard ID
} else {
less = shards[i].DataCenter < shards[j].DataCenter
}
case "rack":
if shards[i].Rack == shards[j].Rack {
less = shards[i].ShardID < shards[j].ShardID // Secondary sort by shard ID
} else {
less = shards[i].Rack < shards[j].Rack
}
default:
less = shards[i].ShardID < shards[j].ShardID
}
if sortOrder == "desc" {
return !less
}
return less
})
}
// GetEcVolumeDetails retrieves detailed information about a specific EC volume
func (s *AdminServer) GetEcVolumeDetails(volumeID uint32, sortBy string, sortOrder string) (*EcVolumeDetailsData, error) {
// Set defaults
if sortBy == "" {
sortBy = "shard_id"
}
if sortOrder == "" {
sortOrder = "asc"
}
var shards []EcShardWithInfo
var collection string
dataCenters := make(map[string]bool)
servers := make(map[string]bool)
// Get detailed EC shard information for the specific volume via gRPC
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
// Process EC shard information for this specific volume
for _, ecShardInfo := range diskInfo.EcShardInfos {
// An older master ignores the filter.
if ecShardInfo.Id == volumeID {
collection = ecShardInfo.Collection
dataCenters[dc.Id] = true
servers[node.Id] = true
// Create individual shard entries for each shard this server has
shardBits := ecShardInfo.EcIndexBits
shardSizes := ecShardInfo.ShardSizes
sizeIndex := 0
for shardId := 0; shardId < erasure_coding.MaxShardCount; shardId++ {
if (shardBits & (1 << uint(shardId))) != 0 {
var shardSize uint64
if sizeIndex < len(shardSizes) {
size := shardSizes[sizeIndex]
if size >= 0 {
shardSize = uint64(size)
}
}
sizeIndex++
ecShard := EcShardWithInfo{
VolumeID: ecShardInfo.Id,
ShardID: uint32(shardId),
Collection: ecShardInfo.Collection,
Size: shardSize,
Server: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
DiskType: diskInfo.Type,
ModifiedTime: 0, // Not available in current API
EcIndexBits: ecShardInfo.EcIndexBits,
ShardCount: getShardCount(ecShardInfo.EcIndexBits),
}
shards = append(shards, ecShard)
}
}
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
if len(shards) == 0 {
return nil, fmt.Errorf("EC volume %d not found", volumeID)
}
// Calculate completeness based on unique shard IDs
foundShards := make(map[int]bool)
for _, shard := range shards {
foundShards[int(shard.ShardID)] = true
}
totalUniqueShards := len(foundShards)
// Check completeness using default 10+4 (14 total shards)
isComplete := (totalUniqueShards == erasure_coding.TotalShardsCount)
// Calculate missing shards
var missingShards []int
for i := 0; i < erasure_coding.TotalShardsCount; i++ {
if !foundShards[i] {
missingShards = append(missingShards, i)
}
}
// Update completeness info for each shard
for i := range shards {
shards[i].IsComplete = isComplete
shards[i].MissingShards = missingShards
}
// Sort shards based on parameters
sortEcShards(shards, sortBy, sortOrder)
// Convert maps to slices
var dcList []string
for dc := range dataCenters {
dcList = append(dcList, dc)
}
var serverList []string
for server := range servers {
serverList = append(serverList, server)
}
data := &EcVolumeDetailsData{
VolumeID: volumeID,
Collection: collection,
Shards: shards,
TotalShards: totalUniqueShards,
IsComplete: isComplete,
MissingShards: missingShards,
DataCenters: dcList,
Servers: serverList,
LastUpdated: time.Now(),
SortBy: sortBy,
SortOrder: sortOrder,
}
return data, nil
}

View File

@@ -0,0 +1,334 @@
package dash
import (
"context"
"net/url"
"path"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
// FileEntry represents a file or directory entry in the file browser
type FileEntry struct {
Name string `json:"name"`
FullPath string `json:"full_path"`
IsDirectory bool `json:"is_directory"`
Size int64 `json:"size"`
ModTime time.Time `json:"mod_time"`
Mode string `json:"mode"`
Uid uint32 `json:"uid"`
Gid uint32 `json:"gid"`
Mime string `json:"mime"`
Replication string `json:"replication"`
Collection string `json:"collection"`
TtlSec int32 `json:"ttl_sec"`
}
// BreadcrumbItem represents a single breadcrumb in the navigation
type BreadcrumbItem struct {
Name string `json:"name"`
Path string `json:"path"`
}
// FileBrowserData contains all data needed for the file browser view
type FileBrowserData struct {
Username string `json:"username"`
CurrentPath string `json:"current_path"`
ParentPath string `json:"parent_path"`
Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"`
Entries []FileEntry `json:"entries"`
LastUpdated time.Time `json:"last_updated"`
IsBucketPath bool `json:"is_bucket_path"`
BucketName string `json:"bucket_name"`
IsTableBucketPath bool `json:"is_table_bucket_path"`
TableBucketName string `json:"table_bucket_name"`
S3Endpoint string `json:"s3_endpoint,omitempty"`
// Pagination fields
PageSize int `json:"page_size"`
HasNextPage bool `json:"has_next_page"`
LastFileName string `json:"last_file_name"` // Cursor for next page
CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes)
}
// GetFileBrowser retrieves file browser data for a given path with cursor-based
// pagination. A non-empty prefix limits the listing to entries whose name starts
// with it, so callers that only want part of a large directory don't have to page
// through all of it.
func (s *AdminServer) GetFileBrowser(dir string, prefix string, lastFileName string, pageSize int) (*FileBrowserData, error) {
if dir == "" {
dir = "/"
}
// Set defaults for pagination
if pageSize < 1 {
pageSize = 20 // Default page size
}
var entries []FileEntry
// Fetch entries using cursor-based pagination
// We fetch pageSize+1 to determine if there's a next page
fetchLimit := pageSize + 1
var fetchedCount int
var lastEntryName string
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Fetch entries starting from the cursor (lastFileName)
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: dir,
Prefix: prefix,
Limit: uint32(fetchLimit),
StartFromFileName: lastFileName,
InclusiveStartFrom: false, // Don't include the cursor file itself
})
if err != nil {
return err
}
for {
resp, err := stream.Recv()
if err != nil {
if err.Error() == "EOF" {
break
}
return err
}
entry := resp.Entry
if entry == nil {
continue
}
fetchedCount++
// Only add entries up to pageSize (the +1 is just to check for next page)
if fetchedCount <= pageSize {
fullPath := path.Join(dir, entry.Name)
var modTime time.Time
if entry.Attributes != nil && entry.Attributes.Mtime > 0 {
modTime = time.Unix(entry.Attributes.Mtime, 0)
}
var mode string
var uid, gid uint32
var size int64
var replication, collection string
var ttlSec int32
if entry.Attributes != nil {
mode = FormatFileMode(entry.Attributes.FileMode)
uid = entry.Attributes.Uid
gid = entry.Attributes.Gid
size = int64(entry.Attributes.FileSize)
ttlSec = entry.Attributes.TtlSec
}
// Get replication and collection from entry extended attributes
if entry.Extended != nil {
if repl, ok := entry.Extended["replication"]; ok {
replication = string(repl)
}
if coll, ok := entry.Extended["collection"]; ok {
collection = string(coll)
}
}
mime := ResolveEntryMime(entry)
fileEntry := FileEntry{
Name: entry.Name,
FullPath: fullPath,
IsDirectory: entry.IsDirectory,
Size: size,
ModTime: modTime,
Mode: mode,
Uid: uid,
Gid: gid,
Mime: mime,
Replication: replication,
Collection: collection,
TtlSec: ttlSec,
}
entries = append(entries, fileEntry)
lastEntryName = entry.Name
}
}
return nil
})
if err != nil {
return nil, err
}
// Determine if there's a next page
hasNextPage := fetchedCount > pageSize
// Generate breadcrumbs
breadcrumbs := s.generateBreadcrumbs(dir)
// Calculate parent path
parentPath := "/"
if dir != "/" {
parentPath = path.Dir(dir)
if parentPath == "." {
parentPath = "/"
}
}
// Check if this is a bucket path and determine if it's a table bucket
isBucketPath := false
bucketName := ""
isTableBucketPath := false
tableBucketName := ""
isRegularBucket := false
if strings.HasPrefix(dir, "/buckets/") {
isBucketPath = true
pathParts := strings.Split(strings.Trim(dir, "/"), "/")
if len(pathParts) >= 2 {
bucketName = pathParts[1]
// Check table bucket status early to avoid second WithFilerClient call
if err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err != nil {
return err
}
if s3tables.IsTableBucketEntry(resp.Entry) {
isTableBucketPath = true
tableBucketName = bucketName
} else {
isRegularBucket = true
}
return nil
}); err != nil {
glog.V(1).Infof("file browser table bucket lookup failed for %s: %v", bucketName, err)
}
}
}
s3Endpoint := ""
if isRegularBucket {
s3Endpoint = s.GetS3Endpoint()
}
return &FileBrowserData{
CurrentPath: dir,
ParentPath: parentPath,
Breadcrumbs: breadcrumbs,
Entries: entries,
LastUpdated: time.Now(),
IsBucketPath: isBucketPath,
BucketName: bucketName,
IsTableBucketPath: isTableBucketPath,
TableBucketName: tableBucketName,
S3Endpoint: s3Endpoint,
// Pagination metadata
PageSize: pageSize,
HasNextPage: hasNextPage,
LastFileName: lastEntryName, // Store for next page navigation
CurrentLastFileName: lastFileName, // Store input cursor for page size changes
}, nil
}
// generateBreadcrumbs creates breadcrumb navigation for the current path
func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem {
var breadcrumbs []BreadcrumbItem
// Always start with root
breadcrumbs = append(breadcrumbs, BreadcrumbItem{
Name: "Root",
Path: "/",
})
if dir == "/" {
return breadcrumbs
}
// Split path and build breadcrumbs
parts := strings.Split(strings.Trim(dir, "/"), "/")
currentPath := ""
for _, part := range parts {
if part == "" {
continue
}
currentPath += "/" + part
// Special handling for bucket paths
displayName := part
if len(breadcrumbs) == 1 && part == "buckets" {
displayName = "Object Store Buckets"
} else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/buckets/") {
displayName = "📦 " + part // Add bucket icon to bucket name
}
breadcrumbs = append(breadcrumbs, BreadcrumbItem{
Name: displayName,
Path: currentPath,
})
}
return breadcrumbs
}
// S3ObjectURL builds the path-style S3 URL for a filer path under /buckets/,
// percent-encoding each path segment. Returns "" for other paths.
func S3ObjectURL(endpoint, fullPath string) string {
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
if !ok || rel == "" || endpoint == "" {
return ""
}
segments := strings.Split(rel, "/")
for i, segment := range segments {
segments[i] = url.PathEscape(segment)
}
return strings.TrimRight(endpoint, "/") + "/" + strings.Join(segments, "/")
}
// GetS3ObjectURL returns the S3 URL an object under a regular bucket is served
// at, or "" when the path is not a bucket object, the bucket is an S3 Tables
// bucket, or no S3 endpoint is known.
func (s *AdminServer) GetS3ObjectURL(fullPath string) string {
rel, ok := strings.CutPrefix(fullPath, "/buckets/")
if !ok {
return ""
}
bucketName, key, found := strings.Cut(rel, "/")
if !found || key == "" {
return ""
}
endpoint := s.GetS3Endpoint()
if endpoint == "" {
return ""
}
if err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: "/buckets",
Name: bucketName,
})
if err != nil {
return err
}
if s3tables.IsTableBucketEntry(resp.Entry) {
endpoint = ""
}
return nil
}); err != nil {
glog.V(1).Infof("object url bucket lookup failed for %s: %v", bucketName, err)
return ""
}
return S3ObjectURL(endpoint, fullPath)
}

View File

@@ -0,0 +1,594 @@
package dash
import (
"path"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// TestGenerateBreadcrumbs tests the actual breadcrumb generation function
// from the production code with various path scenarios
func TestGenerateBreadcrumbs(t *testing.T) {
s := &AdminServer{}
tests := []struct {
name string
path string
expected []BreadcrumbItem
}{
{
name: "root path",
path: "/",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
},
},
{
name: "simple path",
path: "/folder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
},
},
{
name: "nested path",
path: "/folder/subfolder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
{Name: "subfolder", Path: "/folder/subfolder"},
},
},
{
name: "bucket path",
path: "/buckets/mybucket",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "Object Store Buckets", Path: "/buckets"},
{Name: "📦 mybucket", Path: "/buckets/mybucket"},
},
},
{
name: "bucket nested path",
path: "/buckets/mybucket/folder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "Object Store Buckets", Path: "/buckets"},
{Name: "📦 mybucket", Path: "/buckets/mybucket"},
{Name: "folder", Path: "/buckets/mybucket/folder"},
},
},
{
name: "path with trailing slash",
path: "/folder/",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Call the actual production function
result := s.generateBreadcrumbs(tt.path)
if len(result) != len(tt.expected) {
t.Errorf("expected %d breadcrumbs, got %d", len(tt.expected), len(result))
return
}
for i, crumb := range result {
if crumb.Name != tt.expected[i].Name {
t.Errorf("breadcrumb %d: expected name %q, got %q", i, tt.expected[i].Name, crumb.Name)
}
if crumb.Path != tt.expected[i].Path {
t.Errorf("breadcrumb %d: expected path %q, got %q", i, tt.expected[i].Path, crumb.Path)
}
}
})
}
}
// TestS3ObjectURL verifies path-style S3 URL construction with per-segment
// percent-encoding
func TestS3ObjectURL(t *testing.T) {
tests := []struct {
name string
endpoint string
fullPath string
expected string
}{
{
name: "simple object",
endpoint: "https://s3.example.com",
fullPath: "/buckets/public-images/homarr/example.png",
expected: "https://s3.example.com/public-images/homarr/example.png",
},
{
name: "space hash and question mark in key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/a b#c?d.txt",
expected: "https://s3.example.com/b/a%20b%23c%3Fd.txt",
},
{
name: "non-ascii key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/图片.png",
expected: "https://s3.example.com/b/%E5%9B%BE%E7%89%87.png",
},
{
name: "percent in key",
endpoint: "https://s3.example.com",
fullPath: "/buckets/b/100%.txt",
expected: "https://s3.example.com/b/100%25.txt",
},
{
name: "trailing slash on endpoint",
endpoint: "https://s3.example.com/",
fullPath: "/buckets/b/k",
expected: "https://s3.example.com/b/k",
},
{
name: "not a bucket path",
endpoint: "https://s3.example.com",
fullPath: "/topics/t/k",
expected: "",
},
{
name: "empty endpoint",
endpoint: "",
fullPath: "/buckets/b/k",
expected: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := S3ObjectURL(tt.endpoint, tt.fullPath); got != tt.expected {
t.Errorf("S3ObjectURL(%q, %q) = %q, expected %q", tt.endpoint, tt.fullPath, got, tt.expected)
}
})
}
}
// TestNormalizeS3PublicEndpoint verifies unusable configured endpoints are
// dropped rather than producing broken links
func TestNormalizeS3PublicEndpoint(t *testing.T) {
tests := []struct {
endpoint string
expected string
}{
{"https://s3.example.com", "https://s3.example.com"},
{"http://10.0.0.1:8333/", "http://10.0.0.1:8333"},
{"http://[::1]:8333", "http://[::1]:8333"},
{"https://proxy.example.com/s3", "https://proxy.example.com/s3"},
{"", ""},
{"/", ""},
{"s3.example.com", ""},
{"ftp://s3.example.com", ""},
{"https://", ""},
{"https://s3.example.com?x=1", ""},
{"https://s3.example.com/?", ""},
{"https://s3.example.com#frag", ""},
{"https://s3.example.com/#", ""},
{"http://user:pass@s3.example.com", ""},
}
for _, tt := range tests {
if got := normalizeS3PublicEndpoint(tt.endpoint); got != tt.expected {
t.Errorf("normalizeS3PublicEndpoint(%q) = %q, expected %q", tt.endpoint, got, tt.expected)
}
}
}
// TestPathHandlingWithForwardSlashes verifies that the production code
// correctly handles paths with forward slashes (not OS-specific backslashes)
func TestPathHandlingWithForwardSlashes(t *testing.T) {
tests := []struct {
name string
path string
hasSlash bool
}{
{
name: "root",
path: "/",
hasSlash: true,
},
{
name: "single level",
path: "/test",
hasSlash: true,
},
{
name: "multiple levels",
path: "/a/b/c",
hasSlash: true,
},
{
name: "bucket path",
path: "/buckets/mybucket/file",
hasSlash: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify no backslashes appear in paths (which would be wrong on Windows)
if strings.Contains(tt.path, "\\") {
t.Errorf("path contains backslash: %q", tt.path)
}
// Verify forward slashes are used
if tt.hasSlash && !strings.Contains(tt.path, "/") {
t.Errorf("path should contain forward slash: %q", tt.path)
}
})
}
}
// TestParentPathCalculationLogic verifies that parent path calculation
// uses path.Dir semantics (forward slashes), not filepath.Dir (OS-specific)
func TestParentPathCalculationLogic(t *testing.T) {
tests := []struct {
name string
currentDir string
expected string
}{
{
name: "root path",
currentDir: "/",
expected: "/",
},
{
name: "single level",
currentDir: "/folder",
expected: "/",
},
{
name: "two levels",
currentDir: "/folder/subfolder",
expected: "/folder",
},
{
name: "deep nesting",
currentDir: "/a/b/c/d",
expected: "/a/b/c",
},
{
name: "bucket root",
currentDir: "/buckets",
expected: "/",
},
{
name: "bucket directory",
currentDir: "/buckets/mybucket",
expected: "/buckets",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify using path.Dir (the correct approach for URLs)
// This demonstrates the expected behavior
parentPath := "/"
if tt.currentDir != "/" {
// path.Dir always uses forward slashes
parentPath = path.Dir(tt.currentDir)
if parentPath == "." {
parentPath = "/"
}
}
if parentPath != tt.expected {
t.Errorf("expected parent %q, got %q for %q", tt.expected, parentPath, tt.currentDir)
}
// Verify no backslashes in the result
if strings.Contains(parentPath, "\\") {
t.Errorf("parent path contains backslash: %q", parentPath)
}
})
}
}
// TestFileExtensionHandlingLogic verifies that file extensions are correctly
// identified using path semantics (always forward slashes)
func TestFileExtensionHandlingLogic(t *testing.T) {
tests := []struct {
filename string
expected string
}{
{"file.txt", ".txt"},
{"file.log", ".log"},
{"archive.tar.gz", ".gz"},
{"image.jpg", ".jpg"},
{"document.pdf", ".pdf"},
{"data.json", ".json"},
{"noextension", ""},
{".hidden", ".hidden"},
{"file.TXT", ".txt"},
{"file.JPG", ".jpg"},
}
for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) {
// Verify using path.Ext + strings.ToLower (the correct approach)
ext := strings.ToLower(path.Ext(tt.filename))
if ext != tt.expected {
t.Errorf("expected extension %q for %q, got %q", tt.expected, tt.filename, ext)
}
})
}
}
// TestBucketPathDetectionLogic verifies bucket path detection logic
func TestBucketPathDetectionLogic(t *testing.T) {
tests := []struct {
name string
path string
isBucket bool
expectedName string
}{
{
name: "root is not a bucket path",
path: "/",
isBucket: false,
expectedName: "",
},
{
name: "buckets root",
path: "/buckets/",
isBucket: true,
expectedName: "",
},
{
name: "single bucket",
path: "/buckets/mybucket",
isBucket: true,
expectedName: "mybucket",
},
{
name: "bucket with nested path",
path: "/buckets/mybucket/folder/file",
isBucket: true,
expectedName: "mybucket",
},
{
name: "non-bucket path",
path: "/data/folder",
isBucket: false,
expectedName: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify the bucket path detection logic
isBucketPath := false
bucketName := ""
if strings.HasPrefix(tt.path, "/buckets/") {
isBucketPath = true
pathParts := strings.Split(strings.Trim(tt.path, "/"), "/")
if len(pathParts) >= 2 {
bucketName = pathParts[1]
}
}
if isBucketPath != tt.isBucket {
t.Errorf("expected isBucketPath=%v, got %v for path %q", tt.isBucket, isBucketPath, tt.path)
}
if bucketName != tt.expectedName {
t.Errorf("expected bucket name %q, got %q for path %q", tt.expectedName, bucketName, tt.path)
}
})
}
}
// TestPathJoinHandlesEdgeCases verifies that path.Join handles edge cases
// properly for URL path construction (unlike filepath.Join which is OS-specific)
func TestPathJoinHandlesEdgeCases(t *testing.T) {
tests := []struct {
testName string
dir string
filename string
expected string
}{
{
testName: "root directory",
dir: "/",
filename: "file.txt",
expected: "/file.txt",
},
{
testName: "simple directory",
dir: "/folder",
filename: "file.txt",
expected: "/folder/file.txt",
},
{
testName: "nested directory",
dir: "/a/b/c",
filename: "file.txt",
expected: "/a/b/c/file.txt",
},
{
testName: "handles trailing slash",
dir: "/folder/",
filename: "file.txt",
expected: "/folder/file.txt",
},
{
testName: "handles empty name",
dir: "/folder",
filename: "",
expected: "/folder",
},
}
for _, tt := range tests {
t.Run(tt.testName, func(t *testing.T) {
// Verify path.Join behavior for URL paths
result := path.Join(tt.dir, tt.filename)
if result != tt.expected {
t.Errorf("path.Join(%q, %q) = %q, expected %q", tt.dir, tt.filename, result, tt.expected)
}
// Verify no backslashes in the result
if strings.Contains(result, "\\") {
t.Errorf("result contains backslash: %q", result)
}
})
}
}
// TestWindowsPathNormalizationBehavior validates that Windows-style paths
// are correctly converted to forward slashes for URL compatibility.
// This test verifies the actual util.CleanWindowsPath() function used in
// the ShowFileBrowser handler.
func TestWindowsPathNormalizationBehavior(t *testing.T) {
tests := []struct {
name string
windowsPath string
expectedNormPath string
}{
{
name: "backslash separator",
windowsPath: "\\folder\\subfolder",
expectedNormPath: "/folder/subfolder",
},
{
name: "mixed separators",
windowsPath: "/folder\\subfolder/file",
expectedNormPath: "/folder/subfolder/file",
},
{
name: "already normalized",
windowsPath: "/folder/file",
expectedNormPath: "/folder/file",
},
{
name: "simple backslash path",
windowsPath: "\\data",
expectedNormPath: "/data",
},
{
name: "deep nested path",
windowsPath: "\\a\\b\\c\\d",
expectedNormPath: "/a/b/c/d",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Test the actual production function
normalized := util.CleanWindowsPath(tt.windowsPath)
if normalized != tt.expectedNormPath {
t.Errorf("CleanWindowsPath(%q): expected %q, got %q",
tt.windowsPath, tt.expectedNormPath, normalized)
}
})
}
}
// TestBreadcrumbPathFormatting validates that breadcrumb paths always
// use forward slashes and maintain proper URL format
func TestBreadcrumbPathFormatting(t *testing.T) {
s := &AdminServer{}
testPaths := []string{
"/",
"/folder",
"/folder/subfolder",
"/buckets/mybucket",
"/buckets/mybucket/data",
}
for _, testPath := range testPaths {
t.Run("breadcrumbs_for_"+testPath, func(t *testing.T) {
breadcrumbs := s.generateBreadcrumbs(testPath)
// Verify all breadcrumb paths use forward slashes
for i, crumb := range breadcrumbs {
if strings.Contains(crumb.Path, "\\") {
t.Errorf("breadcrumb %d has backslash in path: %q", i, crumb.Path)
}
// Verify paths start with / (except when empty)
if crumb.Path != "" && !strings.HasPrefix(crumb.Path, "/") {
t.Errorf("breadcrumb %d path should start with /: %q", i, crumb.Path)
}
}
})
}
}
// TestDirectoryNavigation validates the complete navigation flow
// for various path scenarios
func TestDirectoryNavigation(t *testing.T) {
s := &AdminServer{}
tests := []struct {
name string
currentPath string
expectedParent string
expectedCrumbs int
}{
{
name: "navigate from root",
currentPath: "/",
expectedParent: "/",
expectedCrumbs: 1,
},
{
name: "navigate from single folder",
currentPath: "/documents",
expectedParent: "/",
expectedCrumbs: 2,
},
{
name: "navigate from nested folder",
currentPath: "/documents/projects/current",
expectedParent: "/documents/projects",
expectedCrumbs: 4,
},
{
name: "navigate bucket contents",
currentPath: "/buckets/data/files",
expectedParent: "/buckets/data",
expectedCrumbs: 4,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify breadcrumbs are generated correctly
breadcrumbs := s.generateBreadcrumbs(tt.currentPath)
if len(breadcrumbs) != tt.expectedCrumbs {
t.Errorf("expected %d breadcrumbs, got %d", tt.expectedCrumbs, len(breadcrumbs))
}
// Verify parent path calculation
expectedParent := "/"
if tt.currentPath != "/" {
expectedParent = path.Dir(tt.currentPath)
if expectedParent == "." {
expectedParent = "/"
}
}
if expectedParent != tt.expectedParent {
t.Errorf("expected parent %q, got %q", tt.expectedParent, expectedParent)
}
// Verify all paths use forward slashes
for i, crumb := range breadcrumbs {
if strings.Contains(crumb.Path, "\\") {
t.Errorf("breadcrumb %d contains backslash: %q", i, crumb.Path)
}
}
})
}
}

View File

@@ -0,0 +1,150 @@
package dash
import (
"mime"
"path"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func init() {
// Register text files
mime.AddExtensionType(".txt", "text/plain")
mime.AddExtensionType(".log", "text/plain")
mime.AddExtensionType(".cfg", "text/plain")
mime.AddExtensionType(".conf", "text/plain")
mime.AddExtensionType(".ini", "text/plain")
mime.AddExtensionType(".properties", "text/plain")
mime.AddExtensionType(".gitignore", "text/plain")
mime.AddExtensionType(".gitattributes", "text/plain")
mime.AddExtensionType(".env", "text/plain")
// Register markup and styling
mime.AddExtensionType(".md", "text/markdown")
mime.AddExtensionType(".markdown", "text/markdown")
mime.AddExtensionType(".html", "text/html")
mime.AddExtensionType(".htm", "text/html")
mime.AddExtensionType(".css", "text/css")
mime.AddExtensionType(".xml", "application/xml")
// Register code and scripting languages
mime.AddExtensionType(".js", "application/javascript")
mime.AddExtensionType(".mjs", "application/javascript")
mime.AddExtensionType(".ts", "text/typescript")
mime.AddExtensionType(".py", "text/x-python")
mime.AddExtensionType(".go", "text/x-go")
mime.AddExtensionType(".java", "text/x-java")
mime.AddExtensionType(".c", "text/x-c")
mime.AddExtensionType(".cpp", "text/x-c++")
mime.AddExtensionType(".cc", "text/x-c++")
mime.AddExtensionType(".cxx", "text/x-c++")
mime.AddExtensionType(".c++", "text/x-c++")
mime.AddExtensionType(".h", "text/x-c-header")
mime.AddExtensionType(".hpp", "text/x-c-header")
mime.AddExtensionType(".php", "text/x-php")
mime.AddExtensionType(".rb", "text/x-ruby")
mime.AddExtensionType(".pl", "text/x-perl")
mime.AddExtensionType(".rs", "text/x-rust")
mime.AddExtensionType(".swift", "text/x-swift")
mime.AddExtensionType(".kt", "text/x-kotlin")
mime.AddExtensionType(".scala", "text/x-scala")
mime.AddExtensionType(".sh", "text/x-shellscript")
mime.AddExtensionType(".bash", "text/x-shellscript")
mime.AddExtensionType(".zsh", "text/x-shellscript")
mime.AddExtensionType(".fish", "text/x-shellscript")
mime.AddExtensionType(".dockerfile", "text/x-dockerfile")
// Register data formats
mime.AddExtensionType(".json", "application/json")
mime.AddExtensionType(".yaml", "text/yaml")
mime.AddExtensionType(".yml", "text/yaml")
mime.AddExtensionType(".csv", "text/csv")
mime.AddExtensionType(".sql", "text/sql")
// Register image types
mime.AddExtensionType(".jpg", "image/jpeg")
mime.AddExtensionType(".jpeg", "image/jpeg")
mime.AddExtensionType(".png", "image/png")
mime.AddExtensionType(".gif", "image/gif")
mime.AddExtensionType(".bmp", "image/bmp")
mime.AddExtensionType(".webp", "image/webp")
mime.AddExtensionType(".svg", "image/svg+xml")
mime.AddExtensionType(".ico", "image/x-icon")
// Register document types
mime.AddExtensionType(".pdf", "application/pdf")
mime.AddExtensionType(".doc", "application/msword")
mime.AddExtensionType(".docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
mime.AddExtensionType(".xls", "application/vnd.ms-excel")
mime.AddExtensionType(".xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
mime.AddExtensionType(".ppt", "application/vnd.ms-powerpoint")
mime.AddExtensionType(".pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
// Register archive types
mime.AddExtensionType(".zip", "application/zip")
mime.AddExtensionType(".tar", "application/x-tar")
mime.AddExtensionType(".gz", "application/gzip")
mime.AddExtensionType(".bz2", "application/x-bzip2")
mime.AddExtensionType(".7z", "application/x-7z-compressed")
mime.AddExtensionType(".rar", "application/x-rar-compressed")
// Register video types
mime.AddExtensionType(".mp4", "video/mp4")
mime.AddExtensionType(".avi", "video/x-msvideo")
mime.AddExtensionType(".mov", "video/quicktime")
mime.AddExtensionType(".wmv", "video/x-ms-wmv")
mime.AddExtensionType(".flv", "video/x-flv")
mime.AddExtensionType(".webm", "video/webm")
// Register audio types
mime.AddExtensionType(".mp3", "audio/mpeg")
mime.AddExtensionType(".wav", "audio/wav")
mime.AddExtensionType(".flac", "audio/flac")
mime.AddExtensionType(".aac", "audio/aac")
mime.AddExtensionType(".ogg", "audio/ogg")
}
func ResolveEntryMime(entry *filer_pb.Entry) string {
if entry == nil {
return "application/octet-stream"
}
if entry.IsDirectory {
return "inode/directory"
}
if entry.Attributes != nil {
normalized := normalizeMimeType(entry.Attributes.Mime)
if normalized != "" {
return normalized
}
}
return inferMimeTypeFromName(entry.Name)
}
func normalizeMimeType(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
if mediaType, _, err := mime.ParseMediaType(value); err == nil && mediaType != "" {
return strings.ToLower(mediaType)
}
if idx := strings.Index(value, ";"); idx >= 0 {
value = value[:idx]
}
return strings.ToLower(strings.TrimSpace(value))
}
func inferMimeTypeFromName(filename string) string {
ext := path.Ext(filename)
if mimeType := mime.TypeByExtension(ext); mimeType != "" {
// mime.TypeByExtension can include parameters (e.g., charset), so we parse just the media type.
if mediaType, _, err := mime.ParseMediaType(mimeType); err == nil {
return mediaType
}
}
return "application/octet-stream"
}

View File

@@ -0,0 +1,68 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestResolveEntryMimePrefersStoredMime(t *testing.T) {
entry := &filer_pb.Entry{
Name: "report.txt",
Attributes: &filer_pb.FuseAttributes{
Mime: " application/pdf; charset=binary ",
},
}
if got := ResolveEntryMime(entry); got != "application/pdf" {
t.Fatalf("ResolveEntryMime() = %q, want %q", got, "application/pdf")
}
}
func TestResolveEntryMimePrefersStoredMimeMalformedParameter(t *testing.T) {
entry := &filer_pb.Entry{
Name: "report.txt",
Attributes: &filer_pb.FuseAttributes{
Mime: "APPLICATION/PDF; bad",
},
}
if got := ResolveEntryMime(entry); got != "application/pdf" {
t.Fatalf("ResolveEntryMime() = %q, want %q", got, "application/pdf")
}
}
func TestResolveEntryMimeFallsBackToFilename(t *testing.T) {
entry := &filer_pb.Entry{Name: "archive.zip"}
if got := ResolveEntryMime(entry); got != "application/zip" {
t.Fatalf("ResolveEntryMime() = %q, want %q", got, "application/zip")
}
}
func TestResolveEntryMimeReturnsDirectoryMime(t *testing.T) {
entry := &filer_pb.Entry{
Name: "folder",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{
Mime: "application/json",
},
}
if got := ResolveEntryMime(entry); got != "inode/directory" {
t.Fatalf("ResolveEntryMime() = %q, want %q", got, "inode/directory")
}
}
func TestResolveEntryMimeWhitespaceMimeFallsBackToFilename(t *testing.T) {
entry := &filer_pb.Entry{
Name: "archive.zip",
Attributes: &filer_pb.FuseAttributes{
Mime: " ",
},
}
if got := ResolveEntryMime(entry); got != "application/zip" {
t.Fatalf("ResolveEntryMime() = %q, want %q", got, "application/zip")
}
}

View File

@@ -0,0 +1,85 @@
package dash
// FormatFileMode converts file mode to Unix-style string representation (e.g., "drwxr-xr-x")
// Handles both Go's os.ModeDir format and standard Unix file type bits
func FormatFileMode(mode uint32) string {
var result []byte = make([]byte, 10)
// File type - handle Go's os.ModeDir first, then standard Unix file type bits
if mode&0x80000000 != 0 { // Go's os.ModeDir (0x80000000 = 2147483648)
result[0] = 'd'
} else {
switch mode & 0170000 { // S_IFMT mask
case 0040000: // S_IFDIR
result[0] = 'd'
case 0100000: // S_IFREG
result[0] = '-'
case 0120000: // S_IFLNK
result[0] = 'l'
case 0020000: // S_IFCHR
result[0] = 'c'
case 0060000: // S_IFBLK
result[0] = 'b'
case 0010000: // S_IFIFO
result[0] = 'p'
case 0140000: // S_IFSOCK
result[0] = 's'
default:
result[0] = '-' // S_IFREG is default
}
}
// Permission bits (always use the lower 12 bits regardless of file type format)
// Owner permissions
if mode&0400 != 0 { // S_IRUSR
result[1] = 'r'
} else {
result[1] = '-'
}
if mode&0200 != 0 { // S_IWUSR
result[2] = 'w'
} else {
result[2] = '-'
}
if mode&0100 != 0 { // S_IXUSR
result[3] = 'x'
} else {
result[3] = '-'
}
// Group permissions
if mode&0040 != 0 { // S_IRGRP
result[4] = 'r'
} else {
result[4] = '-'
}
if mode&0020 != 0 { // S_IWGRP
result[5] = 'w'
} else {
result[5] = '-'
}
if mode&0010 != 0 { // S_IXGRP
result[6] = 'x'
} else {
result[6] = '-'
}
// Other permissions
if mode&0004 != 0 { // S_IROTH
result[7] = 'r'
} else {
result[7] = '-'
}
if mode&0002 != 0 { // S_IWOTH
result[8] = 'w'
} else {
result[8] = '-'
}
if mode&0001 != 0 { // S_IXOTH
result[9] = 'x'
} else {
result[9] = '-'
}
return string(result)
}

View File

@@ -0,0 +1,250 @@
package dash
import (
"context"
"errors"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
// cloneGroup creates a deep copy of an iam_pb.Group to avoid mutating stored state.
func cloneGroup(g *iam_pb.Group) *iam_pb.Group {
clone := &iam_pb.Group{
Name: g.Name,
Disabled: g.Disabled,
}
if g.Members != nil {
clone.Members = make([]string, len(g.Members))
copy(clone.Members, g.Members)
}
if g.PolicyNames != nil {
clone.PolicyNames = make([]string, len(g.PolicyNames))
copy(clone.PolicyNames, g.PolicyNames)
}
return clone
}
func (s *AdminServer) GetGroups(ctx context.Context) ([]GroupData, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
groupNames, err := s.credentialManager.ListGroups(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list groups: %w", err)
}
var groups []GroupData
for _, name := range groupNames {
g, err := s.credentialManager.GetGroup(ctx, name)
if err != nil {
if errors.Is(err, credential.ErrGroupNotFound) {
glog.V(1).Infof("Group %s listed but not found, skipping", name)
continue
}
return nil, fmt.Errorf("failed to get group %s: %w", name, err)
}
status := "enabled"
if g.Disabled {
status = "disabled"
}
groups = append(groups, GroupData{
Name: g.Name,
MemberCount: len(g.Members),
PolicyCount: len(g.PolicyNames),
Status: status,
Members: g.Members,
PolicyNames: g.PolicyNames,
})
}
return groups, nil
}
func (s *AdminServer) GetGroupDetails(ctx context.Context, name string) (*GroupData, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, name)
if err != nil {
return nil, fmt.Errorf("failed to get group: %w", err)
}
status := "enabled"
if g.Disabled {
status = "disabled"
}
return &GroupData{
Name: g.Name,
MemberCount: len(g.Members),
PolicyCount: len(g.PolicyNames),
Status: status,
Members: g.Members,
PolicyNames: g.PolicyNames,
}, nil
}
func (s *AdminServer) CreateGroup(ctx context.Context, name string) (*GroupData, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
group := &iam_pb.Group{Name: name}
if err := s.credentialManager.CreateGroup(ctx, group); err != nil {
return nil, fmt.Errorf("failed to create group: %w", err)
}
glog.V(1).Infof("Created group %s", group.Name)
return &GroupData{
Name: group.Name,
Status: "enabled",
}, nil
}
func (s *AdminServer) DeleteGroup(ctx context.Context, name string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
// Check for members and attached policies before deleting (same guards as IAM handlers)
g, err := s.credentialManager.GetGroup(ctx, name)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
if len(g.Members) > 0 {
return fmt.Errorf("cannot delete group %s: group has %d member(s): %w", name, len(g.Members), credential.ErrGroupNotEmpty)
}
if len(g.PolicyNames) > 0 {
return fmt.Errorf("cannot delete group %s: group has %d attached policy(ies): %w", name, len(g.PolicyNames), credential.ErrGroupNotEmpty)
}
if err := s.credentialManager.DeleteGroup(ctx, name); err != nil {
return fmt.Errorf("failed to delete group: %w", err)
}
glog.V(1).Infof("Deleted group %s", name)
return nil
}
func (s *AdminServer) AddGroupMember(ctx context.Context, groupName, username string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, groupName)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
g = cloneGroup(g)
if _, err := s.credentialManager.GetUser(ctx, username); err != nil {
return fmt.Errorf("user %s not found: %w", username, err)
}
for _, m := range g.Members {
if m == username {
return nil // already a member
}
}
g.Members = append(g.Members, username)
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
return fmt.Errorf("failed to update group: %w", err)
}
glog.V(1).Infof("Added user %s to group %s", username, groupName)
return nil
}
func (s *AdminServer) RemoveGroupMember(ctx context.Context, groupName, username string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, groupName)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
g = cloneGroup(g)
found := false
var newMembers []string
for _, m := range g.Members {
if m == username {
found = true
} else {
newMembers = append(newMembers, m)
}
}
if !found {
return fmt.Errorf("user %s is not a member of group %s: %w", username, groupName, credential.ErrUserNotInGroup)
}
g.Members = newMembers
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
return fmt.Errorf("failed to update group: %w", err)
}
glog.V(1).Infof("Removed user %s from group %s", username, groupName)
return nil
}
func (s *AdminServer) AttachGroupPolicy(ctx context.Context, groupName, policyName string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, groupName)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
g = cloneGroup(g)
if _, err := s.credentialManager.GetPolicy(ctx, policyName); err != nil {
return fmt.Errorf("policy %s not found: %w", policyName, err)
}
for _, p := range g.PolicyNames {
if p == policyName {
return nil // already attached
}
}
g.PolicyNames = append(g.PolicyNames, policyName)
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
return fmt.Errorf("failed to update group: %w", err)
}
glog.V(1).Infof("Attached policy %s to group %s", policyName, groupName)
return nil
}
func (s *AdminServer) DetachGroupPolicy(ctx context.Context, groupName, policyName string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, groupName)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
g = cloneGroup(g)
found := false
var newPolicies []string
for _, p := range g.PolicyNames {
if p == policyName {
found = true
} else {
newPolicies = append(newPolicies, p)
}
}
if !found {
return fmt.Errorf("policy %s is not attached to group %s: %w", policyName, groupName, credential.ErrPolicyNotAttached)
}
g.PolicyNames = newPolicies
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
return fmt.Errorf("failed to update group: %w", err)
}
glog.V(1).Infof("Detached policy %s from group %s", policyName, groupName)
return nil
}
func (s *AdminServer) SetGroupStatus(ctx context.Context, groupName string, enabled bool) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
g, err := s.credentialManager.GetGroup(ctx, groupName)
if err != nil {
return fmt.Errorf("failed to get group: %w", err)
}
g = cloneGroup(g)
g.Disabled = !enabled
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
return fmt.Errorf("failed to update group: %w", err)
}
glog.V(1).Infof("Set group %s status to enabled=%v", groupName, enabled)
return nil
}

View File

@@ -0,0 +1,24 @@
package dash
import (
"io"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/admin/internal/httputil"
)
func writeJSON(w http.ResponseWriter, status int, payload interface{}) {
httputil.WriteJSON(w, status, payload)
}
func writeJSONError(w http.ResponseWriter, status int, message string) {
httputil.WriteJSONError(w, status, message)
}
func decodeJSONBody(r io.Reader, v interface{}) error {
return httputil.DecodeJSONBody(r, v)
}
func newJSONMaxReader(w http.ResponseWriter, r *http.Request) io.Reader {
return httputil.NewJSONMaxReader(w, r)
}

View File

@@ -0,0 +1,536 @@
package dash
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"path"
"slices"
"strings"
"time"
"unicode/utf8"
"github.com/apache/iceberg-go"
"github.com/parquet-go/parquet-go"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
const (
icebergPreviewDefaultRows = 50
icebergPreviewMaxRows = 200
icebergPreviewMaxFiles = 5
icebergPreviewMaxManifests = 100
icebergPreviewMaxListed = 500
icebergPreviewMaxCellChars = 200
icebergPreviewMaxMetaBytes = 64 << 20
// workerPreviewTimeout bounds one round trip to a worker. A page waiting on
// a worker that has gone quiet should say so rather than hang.
workerPreviewTimeout = 15 * time.Second
)
type IcebergDataFileInfo struct {
Path string `json:"path"`
Format string `json:"format"`
RecordCount int64 `json:"record_count"`
SizeBytes int64 `json:"size_bytes"`
}
// IcebergDataPreviewData backs the table data-preview page. PreviewError and
// PreviewNotes report per-snapshot problems without failing the whole page.
type IcebergDataPreviewData struct {
Username string `json:"username"`
CatalogName string `json:"catalog_name"`
NamespaceName string `json:"namespace_name"`
TableName string `json:"table_name"`
BucketARN string `json:"bucket_arn"`
SnapshotID int64 `json:"snapshot_id"`
SnapshotTime time.Time `json:"snapshot_time"`
CurrentSnapshotID int64 `json:"current_snapshot_id"`
Snapshots []IcebergSnapshotInfo `json:"snapshots"`
Columns []string `json:"columns"`
Rows [][]string `json:"rows"`
RowLimit int `json:"row_limit"`
ScannedFiles int `json:"scanned_files"`
SelectedFile string `json:"selected_file"`
DataFiles []IcebergDataFileInfo `json:"data_files"`
TotalDataFiles int `json:"total_data_files"`
TotalRecords int64 `json:"total_records"`
HasDeletes bool `json:"has_deletes"`
PreviewNotes []string `json:"preview_notes,omitempty"`
PreviewError string `json:"preview_error,omitempty"`
LastUpdated time.Time `json:"last_updated"`
// Format is what the catalog recorded. Anything but ICEBERG means the rows
// below, if any, came from a worker rather than from metadata this server read.
Format string `json:"format,omitempty"`
PreviewedBy string `json:"previewed_by,omitempty"`
}
// GetIcebergTableDataPreview walks the selected snapshot's manifests and reads
// sample rows from its Parquet data files. snapshotID 0 means the current
// snapshot; selectedFile restricts the preview to one manifest-listed file.
func (s *AdminServer) GetIcebergTableDataPreview(ctx context.Context, catalogName, bucketArn, namespace, tableName string, snapshotID int64, selectedFile string, rowLimit int) (IcebergDataPreviewData, error) {
if rowLimit < 1 {
rowLimit = icebergPreviewDefaultRows
}
if rowLimit > icebergPreviewMaxRows {
rowLimit = icebergPreviewMaxRows
}
data := IcebergDataPreviewData{
CatalogName: catalogName,
NamespaceName: namespace,
TableName: tableName,
BucketARN: bucketArn,
RowLimit: rowLimit,
SelectedFile: selectedFile,
LastUpdated: time.Now(),
}
namespaceParts, err := parseNamespaceInput(namespace)
if err != nil {
return data, err
}
var resp s3tables.GetTableResponse
req := &s3tables.GetTableRequest{TableBucketARN: bucketArn, Namespace: namespaceParts, Name: tableName}
if err := s.executeS3TablesOperation(ctx, "GetTable", req, &resp); err != nil {
return data, err
}
data.Format = resp.Format
if !strings.EqualFold(resp.Format, s3tables.FormatIceberg) && resp.Format != "" {
s.applyWorkerPreview(ctx, &data, bucketArn, namespaceParts, tableName, rowLimit)
return data, nil
}
if resp.Metadata == nil || len(resp.Metadata.FullMetadata) == 0 {
data.PreviewError = "Table has no Iceberg metadata."
return data, nil
}
var full icebergFullMetadata
if err := json.Unmarshal(resp.Metadata.FullMetadata, &full); err != nil {
data.PreviewError = fmt.Sprintf("Failed to parse Iceberg metadata: %v", err)
return data, nil
}
data.Snapshots = snapshotsFromFullMetadata(full.Snapshots)
data.CurrentSnapshotID = full.CurrentSnapshotID
snap := selectPreviewSnapshot(full, snapshotID)
if snap == nil {
if snapshotID != 0 {
data.PreviewError = fmt.Sprintf("Snapshot %d not found.", snapshotID)
} else {
data.PreviewError = "Table has no snapshots yet."
}
return data, nil
}
data.SnapshotID = snap.SnapshotID
if snap.TimestampMs > 0 {
data.SnapshotTime = time.Unix(0, snap.TimestampMs*int64(time.Millisecond))
}
if snap.ManifestList == "" {
data.PreviewError = "Snapshot has no manifest list."
return data, nil
}
dataFiles, hasDeletes, notes, err := s.listIcebergDataFiles(ctx, catalogName, namespace, tableName, snap.ManifestList)
if err != nil {
data.PreviewError = fmt.Sprintf("Failed to read snapshot manifests: %v", err)
return data, nil
}
data.HasDeletes = hasDeletes
data.PreviewNotes = notes
data.TotalDataFiles = len(dataFiles)
for _, f := range dataFiles {
data.TotalRecords += f.RecordCount
}
data.DataFiles = dataFiles
if len(dataFiles) > icebergPreviewMaxListed {
data.DataFiles = dataFiles[:icebergPreviewMaxListed]
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Listing first %d of %d data files.", icebergPreviewMaxListed, len(dataFiles)))
}
toScan := dataFiles
if selectedFile != "" {
toScan = nil
for _, f := range dataFiles {
if f.Path == selectedFile {
toScan = []IcebergDataFileInfo{f}
break
}
}
if toScan == nil {
data.PreviewError = "Requested data file is not part of this snapshot."
return data, nil
}
}
if len(toScan) == 0 {
data.PreviewNotes = append(data.PreviewNotes, "Snapshot contains no data files.")
return data, nil
}
for _, f := range toScan {
if len(data.Rows) >= rowLimit || data.ScannedFiles >= icebergPreviewMaxFiles {
break
}
if !strings.EqualFold(f.Format, string(iceberg.ParquetFile)) {
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Skipped %s: %s preview is not supported.", path.Base(f.Path), f.Format))
continue
}
filerPath, err := icebergLocationToFilerPath(f.Path, catalogName, namespace, tableName)
if err != nil {
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Skipped %s: %v", path.Base(f.Path), err))
continue
}
cols, rows, err := s.readParquetRows(ctx, filerPath, rowLimit-len(data.Rows))
if err != nil {
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Failed to read %s: %v", path.Base(f.Path), err))
data.ScannedFiles++
continue
}
if data.Columns == nil {
data.Columns = cols
} else if !slices.Equal(data.Columns, cols) {
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Skipped %s: column layout differs from the first file.", path.Base(f.Path)))
continue
}
data.Rows = append(data.Rows, rows...)
data.ScannedFiles++
}
if hasDeletes {
data.PreviewNotes = append(data.PreviewNotes, "Snapshot has row-level delete files; the preview shows raw data-file rows without applying deletes.")
}
return data, nil
}
func selectPreviewSnapshot(full icebergFullMetadata, snapshotID int64) *icebergSnapshot {
if snapshotID != 0 {
for i := range full.Snapshots {
if full.Snapshots[i].SnapshotID == snapshotID {
return &full.Snapshots[i]
}
}
return nil
}
return selectSnapshotForMetrics(full)
}
func (s *AdminServer) listIcebergDataFiles(ctx context.Context, bucketName, namespaceDir, tableName, manifestListLocation string) (files []IcebergDataFileInfo, hasDeletes bool, notes []string, err error) {
listPath, err := icebergLocationToFilerPath(manifestListLocation, bucketName, namespaceDir, tableName)
if err != nil {
return nil, false, nil, err
}
listBytes, err := s.readFilerFileContent(ctx, listPath)
if err != nil {
return nil, false, nil, fmt.Errorf("read manifest list %s: %w", manifestListLocation, err)
}
manifests, err := s3tables.ReadManifestList(listBytes)
if err != nil {
return nil, false, nil, fmt.Errorf("parse manifest list: %w", err)
}
scanned := 0
for _, mf := range manifests {
if mf.ManifestContent() != iceberg.ManifestContentData {
hasDeletes = true
continue
}
if scanned >= icebergPreviewMaxManifests {
notes = append(notes, fmt.Sprintf("Scanned first %d of %d manifests.", icebergPreviewMaxManifests, len(manifests)))
break
}
scanned++
mfPath, err := icebergLocationToFilerPath(mf.FilePath(), bucketName, namespaceDir, tableName)
if err != nil {
notes = append(notes, fmt.Sprintf("Skipped manifest %s: %v", path.Base(mf.FilePath()), err))
continue
}
mfBytes, err := s.readFilerFileContent(ctx, mfPath)
if err != nil {
notes = append(notes, fmt.Sprintf("Failed to read manifest %s: %v", path.Base(mf.FilePath()), err))
continue
}
entries, err := iceberg.ReadManifest(mf, bytes.NewReader(mfBytes), true)
if err != nil {
notes = append(notes, fmt.Sprintf("Failed to parse manifest %s: %v", path.Base(mf.FilePath()), err))
continue
}
for _, entry := range entries {
df := entry.DataFile()
files = append(files, IcebergDataFileInfo{
Path: df.FilePath(),
Format: string(df.FileFormat()),
RecordCount: df.Count(),
SizeBytes: df.FileSizeBytes(),
})
}
}
return files, hasDeletes, notes, nil
}
// icebergLocationToFilerPath maps an Iceberg file location (s3://bucket/...,
// an absolute filer path, or a table-relative path like data/x.parquet) to a
// filer path, which must stay under the table-buckets root.
func icebergLocationToFilerPath(location, bucketName, namespaceDir, tableName string) (string, error) {
p := location
if idx := strings.Index(p, "://"); idx >= 0 {
p = path.Join(s3tables.TablesPath, p[idx+3:])
} else if strings.HasPrefix(p, "/") {
p = path.Clean(p)
} else {
tableDir := path.Join(s3tables.TablesPath, bucketName, namespaceDir, tableName)
p = path.Join(tableDir, p)
if !strings.HasPrefix(p, tableDir+"/") {
return "", fmt.Errorf("location %q resolves outside the table directory", location)
}
}
p = path.Clean(p)
if !strings.HasPrefix(p, s3tables.TablesPath+"/") {
return "", fmt.Errorf("location %q resolves outside %s", location, s3tables.TablesPath)
}
return p, nil
}
func (s *AdminServer) lookupFilerEntry(ctx context.Context, fullPath string) (*filer_pb.Entry, error) {
dir, name := path.Dir(fullPath), path.Base(fullPath)
var entry *filer_pb.Entry
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
return err
}
entry = resp.Entry
return nil
})
if err != nil {
return nil, err
}
if entry == nil {
return nil, fmt.Errorf("not found: %s", fullPath)
}
return entry, nil
}
func (s *AdminServer) readFilerFileContent(ctx context.Context, fullPath string) ([]byte, error) {
entry, err := s.lookupFilerEntry(ctx, fullPath)
if err != nil {
return nil, err
}
if len(entry.Content) > 0 || len(entry.GetChunks()) == 0 {
return entry.Content, nil
}
size := int64(filer.FileSize(entry))
if size > icebergPreviewMaxMetaBytes {
return nil, fmt.Errorf("file too large to load (%d bytes)", size)
}
streamFn, err := filer.PrepareStreamContentWithThrottler(ctx, s.GetMasterClient(), VolumeServerReadJwt, entry.GetChunks(), 0, size, 0)
if err != nil {
return nil, err
}
var buf bytes.Buffer
if err := streamFn(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (s *AdminServer) readParquetRows(ctx context.Context, fullPath string, want int) ([]string, [][]string, error) {
entry, err := s.lookupFilerEntry(ctx, fullPath)
if err != nil {
return nil, nil, err
}
var readerAt io.ReaderAt
var size int64
if len(entry.Content) > 0 || len(entry.GetChunks()) == 0 {
readerAt = bytes.NewReader(entry.Content)
size = int64(len(entry.Content))
} else {
size = int64(filer.FileSize(entry))
readerAt = &filerChunkReaderAt{ctx: ctx, server: s, chunks: entry.GetChunks(), size: size}
}
pf, err := parquet.OpenFile(readerAt, size, parquet.SkipPageIndex(true), parquet.SkipBloomFilters(true))
if err != nil {
return nil, nil, fmt.Errorf("open parquet: %w", err)
}
leafColumns := pf.Schema().Columns()
cols := make([]string, len(leafColumns))
for i, c := range leafColumns {
cols[i] = strings.Join(c, ".")
}
reader := parquet.NewReader(pf)
defer reader.Close()
rowBuf := make([]parquet.Row, 32)
var out [][]string
for len(out) < want {
n, readErr := reader.ReadRows(rowBuf)
for i := 0; i < n && len(out) < want; i++ {
out = append(out, formatParquetRow(rowBuf[i], len(cols)))
}
if readErr != nil {
if readErr == io.EOF {
break
}
return nil, nil, readErr
}
if n == 0 {
break
}
}
return cols, out, nil
}
func formatParquetRow(row parquet.Row, columnCount int) []string {
cells := make([]string, columnCount)
seen := make([]bool, columnCount)
for _, v := range row {
ci := v.Column()
if ci < 0 || ci >= columnCount {
continue
}
cell := formatParquetValue(v)
if seen[ci] {
cells[ci] += ", " + cell
} else {
cells[ci] = cell
seen[ci] = true
}
}
return cells
}
func formatParquetValue(v parquet.Value) string {
if v.IsNull() {
return ""
}
switch v.Kind() {
case parquet.ByteArray, parquet.FixedLenByteArray:
b := v.ByteArray()
if utf8.Valid(b) {
return truncateCell(string(b))
}
return truncateCell(fmt.Sprintf("0x%x", b))
default:
return truncateCell(v.String())
}
}
func truncateCell(s string) string {
if len(s) <= icebergPreviewMaxCellChars {
return s
}
cut := icebergPreviewMaxCellChars
for cut > 0 && !utf8.RuneStart(s[cut]) {
cut--
}
return s[:cut] + "…"
}
// filerChunkReaderAt serves random-access reads over a chunked filer entry by
// issuing ranged volume-server reads, carrying the read JWT when configured.
type filerChunkReaderAt struct {
ctx context.Context
server *AdminServer
chunks []*filer_pb.FileChunk
size int64
}
func (r *filerChunkReaderAt) ReadAt(p []byte, off int64) (int, error) {
if off < 0 {
return 0, fmt.Errorf("negative offset: %d", off)
}
if off >= r.size {
return 0, io.EOF
}
want := int64(len(p))
if off+want > r.size {
want = r.size - off
}
streamFn, err := filer.PrepareStreamContentWithThrottler(r.ctx, r.server.GetMasterClient(), VolumeServerReadJwt, r.chunks, off, want, 0)
if err != nil {
return 0, err
}
// Write straight into p; a bytes.Buffer wrapping p would silently
// allocate a fresh backing array if it ever grew, dropping bytes the
// caller expects in p.
w := &sliceWriter{dst: p}
if err := streamFn(w); err != nil {
return w.n, err
}
if w.n < len(p) {
return w.n, io.EOF
}
return w.n, nil
}
// sliceWriter writes into a fixed destination slice, discarding any overflow.
type sliceWriter struct {
dst []byte
n int
}
func (w *sliceWriter) Write(p []byte) (int, error) {
c := copy(w.dst[w.n:], p)
w.n += c
return c, nil
}
// applyWorkerPreview asks the worker that last described this table for sample
// rows. Admin has no reader for a format it does not implement, so this is the
// only way the page shows anything but a location.
//
// The rows are fetched, never cached: they are the table's data rather than a
// description of it, and a stale copy sitting in admin would be worse than
// asking.
func (s *AdminServer) applyWorkerPreview(ctx context.Context, data *IcebergDataPreviewData, bucketArn string, namespaceParts []string, tableName string, rowLimit int) {
plugin := s.GetPlugin()
if plugin == nil {
data.PreviewError = fmt.Sprintf("Reading a %s table needs a plugin worker, and none is configured.", data.Format)
return
}
bucketName, err := s3tables.ParseBucketNameFromARN(bucketArn)
if err != nil {
data.PreviewError = err.Error()
return
}
objectID := append(append([]string{bucketName}, namespaceParts...), tableName)
requestCtx, cancel := context.WithTimeout(ctx, workerPreviewTimeout)
defer cancel()
response, err := plugin.RequestObjectPreview(requestCtx, objectID, data.Format, rowLimit)
if err != nil {
data.PreviewError = fmt.Sprintf("Could not read this %s table: %v", data.Format, err)
return
}
data.Columns = response.Columns
data.Rows = make([][]string, 0, len(response.Rows))
for _, row := range response.Rows {
cells := make([]string, len(row.Values))
for i, value := range row.Values {
cells[i] = truncateCell(value)
}
data.Rows = append(data.Rows, cells)
}
data.TotalRecords = response.TotalRows
data.PreviewedBy = pluginWorkerForObject(plugin, objectID, data.Format)
if int64(len(data.Rows)) < response.TotalRows {
data.PreviewNotes = append(data.PreviewNotes, fmt.Sprintf("Showing %d of %d rows.", len(data.Rows), response.TotalRows))
}
}
// pluginWorkerForObject names the worker that answered, for the page to show.
func pluginWorkerForObject(p *plugin.Plugin, objectID []string, format string) string {
if observed, ok := p.Observations().GetFormat(objectID, format); ok {
return observed.WorkerID
}
return ""
}

View File

@@ -0,0 +1,136 @@
package dash
import (
"bytes"
"io"
"strings"
"testing"
"github.com/parquet-go/parquet-go"
)
func TestIcebergLocationToFilerPath(t *testing.T) {
tests := []struct {
location string
want string
wantErr bool
}{
{location: "s3://warehouse/ns/tbl/data/f1.parquet", want: "/buckets/warehouse/ns/tbl/data/f1.parquet"},
{location: "s3a://warehouse/ns/tbl/metadata/snap-1.avro", want: "/buckets/warehouse/ns/tbl/metadata/snap-1.avro"},
{location: "/buckets/warehouse/ns/tbl/data/f1.parquet", want: "/buckets/warehouse/ns/tbl/data/f1.parquet"},
{location: "data/f1.parquet", want: "/buckets/warehouse/ns/tbl/data/f1.parquet"},
{location: "metadata/v3.metadata.json", want: "/buckets/warehouse/ns/tbl/metadata/v3.metadata.json"},
{location: "s3://warehouse/../../etc/passwd", wantErr: true},
{location: "/etc/passwd", wantErr: true},
{location: "../../../etc/passwd", wantErr: true},
}
for _, tc := range tests {
got, err := icebergLocationToFilerPath(tc.location, "warehouse", "ns", "tbl")
if tc.wantErr {
if err == nil {
t.Errorf("icebergLocationToFilerPath(%q) = %q, want error", tc.location, got)
}
continue
}
if err != nil {
t.Errorf("icebergLocationToFilerPath(%q): %v", tc.location, err)
continue
}
if got != tc.want {
t.Errorf("icebergLocationToFilerPath(%q) = %q, want %q", tc.location, got, tc.want)
}
}
}
func TestFormatParquetRow(t *testing.T) {
type record struct {
ID int64 `parquet:"id"`
Name string `parquet:"name"`
Score float64 `parquet:"score"`
Blob []byte `parquet:"blob"`
Note *string `parquet:"note,optional"`
}
var buf bytes.Buffer
w := parquet.NewGenericWriter[record](&buf)
note := "hello"
if _, err := w.Write([]record{
{ID: 1, Name: "alpha", Score: 1.5, Blob: []byte{0xff, 0xfe}, Note: &note},
{ID: 2, Name: "beta", Score: -2.25, Blob: []byte("text"), Note: nil},
}); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
pf, err := parquet.OpenFile(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
if err != nil {
t.Fatal(err)
}
leafColumns := pf.Schema().Columns()
cols := make([]string, len(leafColumns))
for i, c := range leafColumns {
cols[i] = strings.Join(c, ".")
}
reader := parquet.NewReader(pf)
defer reader.Close()
rows := make([]parquet.Row, 4)
n, err := reader.ReadRows(rows)
if err != nil && err != io.EOF {
t.Fatal(err)
}
if n != 2 {
t.Fatalf("got %d rows, want 2", n)
}
byName := func(cells []string, name string) string {
for i, col := range cols {
if col == name {
return cells[i]
}
}
t.Fatalf("column %q not found in %v", name, cols)
return ""
}
first := formatParquetRow(rows[0], len(cols))
if got := byName(first, "id"); got != "1" {
t.Errorf("id = %q, want 1", got)
}
if got := byName(first, "name"); got != "alpha" {
t.Errorf("name = %q, want alpha", got)
}
if got := byName(first, "score"); got != "1.5" {
t.Errorf("score = %q, want 1.5", got)
}
if got := byName(first, "blob"); got != "0xfffe" {
t.Errorf("blob = %q, want 0xfffe", got)
}
if got := byName(first, "note"); got != "hello" {
t.Errorf("note = %q, want hello", got)
}
second := formatParquetRow(rows[1], len(cols))
if got := byName(second, "blob"); got != "text" {
t.Errorf("blob = %q, want text", got)
}
if got := byName(second, "note"); got != "" {
t.Errorf("note = %q, want empty for null", got)
}
}
func TestTruncateCell(t *testing.T) {
long := strings.Repeat("界", 200)
got := truncateCell(long)
if len(got) > icebergPreviewMaxCellChars+len("…") {
t.Errorf("truncated cell too long: %d", len(got))
}
if !strings.HasSuffix(got, "…") {
t.Errorf("truncated cell should end with ellipsis")
}
if short := truncateCell("abc"); short != "abc" {
t.Errorf("short cell changed: %q", short)
}
}

View File

@@ -0,0 +1,73 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
)
// TestLoadMaintenanceConfigHonoursPersistedTaskConfigs guards against a regression: buildPolicyFromTaskConfigs used to call
// LoadConfigFromPersistence(nil), which can never satisfy the loaders' type assertion, so every
// task silently fell back to its compiled-in defaults (Enabled: true) and a task disabled in the
// admin UI kept being scheduled.
func TestLoadMaintenanceConfigHonoursPersistedTaskConfigs(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
// A maintenance.pb must exist, otherwise LoadMaintenanceConfig returns early with defaults.
if err := cp.SaveMaintenanceConfig(DefaultMaintenanceConfig()); err != nil {
t.Fatalf("save maintenance config: %v", err)
}
// Disable balance and vacuum the way the admin UI does, and change a value that is not a bool
// so a fallback to defaults cannot pass by coincidence.
disabledBalance := balance.NewDefaultConfig()
disabledBalance.Enabled = false
disabledBalance.MinServerCount = 7
if err := cp.SaveBalanceTaskPolicy(disabledBalance.ToTaskPolicy()); err != nil {
t.Fatalf("save balance policy: %v", err)
}
disabledVacuum := vacuum.NewDefaultConfig()
disabledVacuum.Enabled = false
if err := cp.SaveVacuumTaskPolicy(disabledVacuum.ToTaskPolicy()); err != nil {
t.Fatalf("save vacuum policy: %v", err)
}
config, err := cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if config.Policy == nil {
t.Fatal("policy is nil, want it populated from the persisted task configs")
}
balancePolicy := config.Policy.TaskPolicies["balance"]
if balancePolicy == nil {
t.Fatal("no balance task policy in the built maintenance policy")
}
if balancePolicy.Enabled {
t.Error("balance enabled = true, want false from the persisted config")
}
if got := balancePolicy.GetBalanceConfig().GetMinServerCount(); got != 7 {
t.Errorf("balance min server count = %d, want persisted 7", got)
}
vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
if vacuumPolicy == nil {
t.Fatal("no vacuum task policy in the built maintenance policy")
}
if vacuumPolicy.Enabled {
t.Error("vacuum enabled = true, want false from the persisted config")
}
// erasure_coding was never saved, so it keeps the loader's default of enabled.
ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
if ecPolicy == nil {
t.Fatal("no erasure_coding task policy in the built maintenance policy")
}
if !ecPolicy.Enabled {
t.Error("erasure_coding enabled = false, want the default true for a config never saved")
}
}

View File

@@ -0,0 +1,187 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
"google.golang.org/protobuf/proto"
)
// The task definitions these tests configure are process-global, so put them back the way a
// fresh process would have them. Passing no config store makes every task fall back to its
// own NewDefaultConfig, which is exactly the state package init left them in.
func restoreGlobalTaskState(t *testing.T) {
t.Helper()
t.Cleanup(func() {
tasks.GetGlobalConfigUpdateRegistry().UpdateAllConfigs(nil)
})
}
// TestDisabledTaskIsNotScannedAfterStartup walks the admin server's startup sequence over a
// data directory that has a disabled balance task saved in it, and checks the end state that
// actually matters: the balance detector reports disabled, so ScanWithTaskDetectors skips it.
//
// This is the whole reported bug in one test. The reporter disabled balance, and the
// scanner kept detecting balance tasks, cancelling them and re-detecting them. Two separate
// defects had to line up for the disabled flag to survive to here: the policy had to be built
// from the persisted configs rather than from a nil store, and the policy had to reach
// detector.IsEnabled() rather than dying in a failed type assertion.
func TestDisabledTaskIsNotScannedAfterStartup(t *testing.T) {
restoreGlobalTaskState(t)
dir := t.TempDir()
cp := NewConfigPersistence(dir)
// What the admin writes when a user turns balance off, and leaves vacuum on.
disabledBalance := balance.NewDefaultConfig()
disabledBalance.Enabled = false
if err := cp.SaveBalanceTaskPolicy(disabledBalance.ToTaskPolicy()); err != nil {
t.Fatalf("save balance policy: %v", err)
}
enabledVacuum := vacuum.NewDefaultConfig()
enabledVacuum.Enabled = true
if err := cp.SaveVacuumTaskPolicy(enabledVacuum.ToTaskPolicy()); err != nil {
t.Fatalf("save vacuum policy: %v", err)
}
// The admin server's startup sequence, in order:
// loadTaskConfigurationsFromPersistence, then InitMaintenanceManager.
tasks.GetGlobalConfigUpdateRegistry().UpdateAllConfigs(cp)
maintenanceConfig, err := cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
manager := maintenance.NewMaintenanceManager(nil, maintenanceConfig, cp)
if manager == nil {
t.Fatal("NewMaintenanceManager returned nil")
}
registry := tasks.GetGlobalTypesRegistry()
balanceDetector := registry.GetDetector(types.TaskTypeBalance)
if balanceDetector == nil {
t.Fatal("no balance detector registered")
}
if balanceDetector.IsEnabled() {
t.Error("balance detector reports enabled after startup over a data directory where " +
"balance is saved as disabled; the scanner will keep detecting and cancelling balance tasks")
}
vacuumDetector := registry.GetDetector(types.TaskTypeVacuum)
if vacuumDetector == nil {
t.Fatal("no vacuum detector registered")
}
if !vacuumDetector.IsEnabled() {
t.Error("vacuum detector reports disabled although vacuum is saved as enabled; " +
"the fix must not switch off tasks the user left on")
}
// Tasks the user never touched keep their compiled-in default of enabled rather than
// being switched off by a policy entry built from a config that was never saved.
for _, taskType := range []types.TaskType{types.TaskTypeErasureCoding, types.TaskTypeECBalance} {
detector := registry.GetDetector(taskType)
if detector == nil {
t.Fatalf("no %s detector registered", taskType)
}
if !detector.IsEnabled() {
t.Errorf("%s detector reports disabled although its config was never saved", taskType)
}
}
}
// TestLoadMaintenanceConfigKeepsPersistedEnabledFalse covers the top of issue-shaped startup:
// an operator persisted enabled=false, and the load path used to lose it twice over — once to
// ApplyDefaultsToProtobuf treating the bool zero value as unset, and once to an explicit
// force-enable "migration" block. The persisted flag must come back as saved, while fields the
// old file never carried still pick up their schema defaults.
func TestLoadMaintenanceConfigKeepsPersistedEnabledFalse(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
// A file that only knows about the enabled flag: everything else zero.
if err := cp.SaveMaintenanceConfig(&MaintenanceConfig{Enabled: proto.Bool(false)}); err != nil {
t.Fatalf("save maintenance config: %v", err)
}
loaded, err := cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if loaded.GetEnabled() {
t.Error("persisted enabled=false came back true; the maintenance system cannot be disabled")
}
if loaded.ScanIntervalSeconds != 30*60 {
t.Errorf("scan interval = %d, want schema default 1800 filled in", loaded.ScanIntervalSeconds)
}
// And with no file at all, the default is enabled.
fresh, err := NewConfigPersistence(t.TempDir()).LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if !fresh.GetEnabled() {
t.Error("maintenance not enabled by default when nothing is persisted")
}
}
// TestLoadMaintenanceConfigTreatsLegacyAbsentEnabledAsOn: a maintenance.pb written before
// enabled tracked presence carries no enabled field on the wire whether the old default left
// it false or an operator unchecked it — the two are indistinguishable. Such files must keep
// the enabled default rather than silently switching maintenance off on upgrade; only a file
// that explicitly persists the toggle may disable it.
func TestLoadMaintenanceConfigTreatsLegacyAbsentEnabledAsOn(t *testing.T) {
dir := t.TempDir()
cp := NewConfigPersistence(dir)
// What an old writer produced for enabled=false plus a tuned scan interval: the bool is
// simply absent from the wire.
if err := cp.SaveMaintenanceConfig(&MaintenanceConfig{ScanIntervalSeconds: 15 * 60}); err != nil {
t.Fatalf("save maintenance config: %v", err)
}
loaded, err := cp.LoadMaintenanceConfig()
if err != nil {
t.Fatalf("load maintenance config: %v", err)
}
if !loaded.GetEnabled() {
t.Error("legacy config without an enabled field loads as disabled; " +
"upgrading would silently switch the maintenance system off")
}
if loaded.ScanIntervalSeconds != 15*60 {
t.Errorf("scan interval = %d, want persisted 900 kept", loaded.ScanIntervalSeconds)
}
}
// TestPolicyMirrorsWhatTheDetectorsReport checks that the maintenance policy the queue and
// the scanner run on agrees with the detectors. A disagreement means one of the two paths
// into the task configs has gone stale again.
func TestPolicyMirrorsWhatTheDetectorsReport(t *testing.T) {
restoreGlobalTaskState(t)
dir := t.TempDir()
cp := NewConfigPersistence(dir)
disabledBalance := balance.NewDefaultConfig()
disabledBalance.Enabled = false
if err := cp.SaveBalanceTaskPolicy(disabledBalance.ToTaskPolicy()); err != nil {
t.Fatalf("save balance policy: %v", err)
}
tasks.GetGlobalConfigUpdateRegistry().UpdateAllConfigs(cp)
policy := cp.buildPolicyFromTaskConfigs()
for taskType, detector := range tasks.GetGlobalTypesRegistry().GetAllDetectors() {
policyEnabled := maintenance.IsTaskEnabled(policy, maintenance.MaintenanceTaskType(taskType))
if policyEnabled != detector.IsEnabled() {
t.Errorf("%s: policy says enabled=%v but the detector says enabled=%v",
taskType, policyEnabled, detector.IsEnabled())
}
}
}

View File

@@ -0,0 +1,133 @@
package dash
import (
"net/http"
"strings"
"github.com/gorilla/mux"
"github.com/gorilla/sessions"
)
const sessionName = "admin-session"
// SessionName returns the cookie session name used by the admin UI.
func SessionName() string {
return sessionName
}
type sessionValidationErrorKind int
const (
sessionValidationErrorKindUnauthenticated sessionValidationErrorKind = iota
sessionValidationErrorKindSessionInit
)
type sessionValidationError struct {
kind sessionValidationErrorKind
err error
}
func (e *sessionValidationError) Error() string {
if e.err != nil {
return e.err.Error()
}
return "session validation failed"
}
func (e *sessionValidationError) Unwrap() error {
return e.err
}
func validateSession(store sessions.Store, w http.ResponseWriter, r *http.Request) (string, string, string, error) {
session, err := store.Get(r, sessionName)
if err != nil {
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindSessionInit, err: err}
}
authenticated, _ := session.Values["authenticated"].(bool)
username, _ := session.Values["username"].(string)
role, _ := session.Values["role"].(string)
if !authenticated || username == "" {
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindUnauthenticated}
}
csrfToken, err := getOrCreateSessionCSRFToken(session, r, w)
if err != nil {
return "", "", "", &sessionValidationError{kind: sessionValidationErrorKindSessionInit, err: err}
}
return username, role, csrfToken, nil
}
// RequireAuth checks if user is authenticated.
func RequireAuth(store sessions.Store) mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
username, role, csrfToken, err := validateSession(store, w, r)
if err != nil {
prefix := URLPrefixFromContext(r.Context())
if verr, ok := err.(*sessionValidationError); ok && verr.kind == sessionValidationErrorKindUnauthenticated {
http.Redirect(w, r, prefix+"/login", http.StatusTemporaryRedirect)
} else {
http.Redirect(w, r, prefix+"/login?error=Unable to initialize session", http.StatusTemporaryRedirect)
}
return
}
ctx := WithAuthContext(r.Context(), username, role, csrfToken)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
}
// RequireAuthAPI checks if user is authenticated for API endpoints.
// Returns JSON error instead of redirecting to login page.
func RequireAuthAPI(store sessions.Store) mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
username, role, csrfToken, err := validateSession(store, w, r)
if err != nil {
if verr, ok := err.(*sessionValidationError); ok && verr.kind == sessionValidationErrorKindUnauthenticated {
writeJSON(w, http.StatusUnauthorized, map[string]string{
"error": "Authentication required",
"message": "Please log in to access this endpoint",
})
} else {
writeJSON(w, http.StatusInternalServerError, map[string]string{
"error": "Failed to initialize session",
"message": "Unable to initialize session",
})
}
return
}
ctx := WithAuthContext(r.Context(), username, role, csrfToken)
next.ServeHTTP(w, r.WithContext(ctx))
})
}
}
// RequireWriteAccess checks if user has admin role (write access).
// Returns JSON error for API endpoints, redirects for HTML endpoints.
func RequireWriteAccess() mux.MiddlewareFunc {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
role := RoleFromContext(r.Context())
if role != "admin" {
// Check if this is an API request (path starts with /api) or HTML request.
if strings.HasPrefix(r.URL.Path, "/api") {
writeJSON(w, http.StatusForbidden, map[string]string{
"error": "Insufficient permissions",
"message": "This operation requires admin access. Read-only users can only view data.",
})
} else {
http.Redirect(w, r, URLPrefixFromContext(r.Context())+"/admin?error=Insufficient permissions", http.StatusSeeOther)
}
return
}
next.ServeHTTP(w, r)
})
}
}

View File

@@ -0,0 +1,80 @@
package dash
import (
"context"
"sort"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// mountClientTypes are the metadata-subscriber client types that represent a
// FUSE/VFS mount: the Go `weed mount` ("mount") and the Rust VFS ("sw-vfs").
var mountClientTypes = []string{"mount", "sw-vfs"}
// mountClientsFilerTimeout bounds each per-filer ListMetadataSubscribers call so
// a slow or unreachable filer can't stall the admin dashboard.
const mountClientsFilerTimeout = 5 * time.Second
// GetMountClients queries every filer for its connected mount subscribers and
// aggregates them. Each filer only knows the clients connected to itself, so a
// cluster-wide view requires fanning out. The filers are queried concurrently,
// each under a short timeout, and an unreachable filer is skipped.
func (s *AdminServer) GetMountClients() (*MountClientsData, error) {
var (
mu sync.Mutex
clients []MountClient
wg sync.WaitGroup
)
for _, filerAddr := range s.GetAllFilers() {
wg.Add(1)
go func(filerAddr string) {
defer wg.Done()
ctx, cancel := context.WithTimeout(context.Background(), mountClientsFilerTimeout)
defer cancel()
err := pb.WithGrpcFilerClient(false, 0, pb.ServerAddress(filerAddr), s.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.ListMetadataSubscribers(ctx, &filer_pb.ListMetadataSubscribersRequest{
ClientTypes: mountClientTypes,
})
if err != nil {
return err
}
mu.Lock()
for _, sub := range resp.Subscribers {
clients = append(clients, MountClient{
ClientName: sub.ClientName,
ClientType: sub.ClientType,
Address: sub.Address,
PathPrefix: sub.PathPrefix,
ClientId: sub.ClientId,
ConnectedAt: time.Unix(0, sub.ConnectedAtNs),
FilerAddress: sub.FilerAddress,
})
}
mu.Unlock()
return nil
})
if err != nil {
glog.Warningf("list mount clients from filer %s: %v", filerAddr, err)
}
}(filerAddr)
}
wg.Wait()
sort.Slice(clients, func(i, j int) bool {
if clients[i].Address != clients[j].Address {
return clients[i].Address < clients[j].Address
}
return clients[i].PathPrefix < clients[j].PathPrefix
})
return &MountClientsData{
MountClients: clients,
TotalMountClients: len(clients),
LastUpdated: time.Now(),
}, nil
}

View File

@@ -0,0 +1,621 @@
package dash
import (
"context"
"fmt"
"io"
"path"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// GetTopics retrieves message queue topics data
func (s *AdminServer) GetTopics() (*TopicsData, error) {
var topics []TopicInfo
// Find broker leader and get topics
brokerLeader, err := s.findBrokerLeader()
if err != nil {
// If no broker leader found, return empty data
return &TopicsData{
Topics: topics,
TotalTopics: len(topics),
LastUpdated: time.Now(),
}, nil
}
// Connect to broker leader and list topics
err = s.withBrokerClient(brokerLeader, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp, err := client.ListTopics(ctx, &mq_pb.ListTopicsRequest{})
if err != nil {
return err
}
// Convert protobuf topics to TopicInfo - only include available data
for _, pbTopic := range resp.Topics {
topicInfo := TopicInfo{
Name: fmt.Sprintf("%s.%s", pbTopic.Namespace, pbTopic.Name),
Partitions: 0, // Will be populated by LookupTopicBrokers call
Retention: TopicRetentionInfo{
Enabled: false,
DisplayValue: 0,
DisplayUnit: "days",
},
}
// Get topic configuration to get partition count and retention info
lookupResp, err := client.LookupTopicBrokers(ctx, &mq_pb.LookupTopicBrokersRequest{
Topic: pbTopic,
})
if err == nil {
topicInfo.Partitions = len(lookupResp.BrokerPartitionAssignments)
}
// Get topic configuration for retention information
configResp, err := client.GetTopicConfiguration(ctx, &mq_pb.GetTopicConfigurationRequest{
Topic: pbTopic,
})
if err == nil && configResp.Retention != nil {
topicInfo.Retention = convertTopicRetention(configResp.Retention)
}
topics = append(topics, topicInfo)
}
return nil
})
if err != nil {
// If connection fails, return empty data
return &TopicsData{
Topics: topics,
TotalTopics: len(topics),
LastUpdated: time.Now(),
}, nil
}
return &TopicsData{
Topics: topics,
TotalTopics: len(topics),
LastUpdated: time.Now(),
// Don't include TotalMessages and TotalSize as they're not available
}, nil
}
// GetSubscribers retrieves message queue subscribers data
func (s *AdminServer) GetSubscribers() (*SubscribersData, error) {
var subscribers []SubscriberInfo
// Find broker leader and get subscriber info from broker stats
brokerLeader, err := s.findBrokerLeader()
if err != nil {
// If no broker leader found, return empty data
return &SubscribersData{
Subscribers: subscribers,
TotalSubscribers: len(subscribers),
ActiveSubscribers: 0,
LastUpdated: time.Now(),
}, nil
}
// Connect to broker leader and get subscriber information
// Note: SeaweedMQ doesn't have a direct API to list all subscribers
// We would need to collect this information from broker statistics
// For now, return empty data structure as subscriber info is not
// directly available through the current MQ API
err = s.withBrokerClient(brokerLeader, func(client mq_pb.SeaweedMessagingClient) error {
// TODO: Implement subscriber data collection from broker statistics
// This would require access to broker internal statistics about
// active subscribers, consumer groups, etc.
return nil
})
if err != nil {
// If connection fails, return empty data
return &SubscribersData{
Subscribers: subscribers,
TotalSubscribers: len(subscribers),
ActiveSubscribers: 0,
LastUpdated: time.Now(),
}, nil
}
activeCount := 0
for _, sub := range subscribers {
if sub.Status == "active" {
activeCount++
}
}
return &SubscribersData{
Subscribers: subscribers,
TotalSubscribers: len(subscribers),
ActiveSubscribers: activeCount,
LastUpdated: time.Now(),
}, nil
}
// GetTopicDetails retrieves detailed information about a specific topic
func (s *AdminServer) GetTopicDetails(namespace, topicName string) (*TopicDetailsData, error) {
// Find broker leader
brokerLeader, err := s.findBrokerLeader()
if err != nil {
return nil, fmt.Errorf("failed to find broker leader: %w", err)
}
var topicDetails *TopicDetailsData
// Connect to broker leader and get topic configuration
err = s.withBrokerClient(brokerLeader, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// Get topic configuration using the new API
configResp, err := client.GetTopicConfiguration(ctx, &mq_pb.GetTopicConfigurationRequest{
Topic: &schema_pb.Topic{
Namespace: namespace,
Name: topicName,
},
})
if err != nil {
return fmt.Errorf("failed to get topic configuration: %w", err)
}
// Initialize topic details
topicDetails = &TopicDetailsData{
TopicName: fmt.Sprintf("%s.%s", namespace, topicName),
Namespace: namespace,
Name: topicName,
Partitions: []PartitionInfo{},
Publishers: []PublisherInfo{},
Subscribers: []TopicSubscriberInfo{},
ConsumerGroupOffsets: []ConsumerGroupOffsetInfo{},
Retention: convertTopicRetention(configResp.Retention),
CreatedAt: time.Unix(0, configResp.CreatedAtNs),
LastUpdated: time.Unix(0, configResp.LastUpdatedNs),
}
// Set current time if timestamps are not available
if configResp.CreatedAtNs == 0 {
topicDetails.CreatedAt = time.Now()
}
if configResp.LastUpdatedNs == 0 {
topicDetails.LastUpdated = time.Now()
}
// Process partitions
for _, assignment := range configResp.BrokerPartitionAssignments {
if assignment.Partition != nil {
partitionInfo := PartitionInfo{
ID: assignment.Partition.RangeStart,
LeaderBroker: assignment.LeaderBroker,
FollowerBroker: assignment.FollowerBroker,
MessageCount: 0, // Will be enhanced later with actual stats
TotalSize: 0, // Will be enhanced later with actual stats
LastDataTime: time.Time{}, // Will be enhanced later
CreatedAt: time.Now(),
}
topicDetails.Partitions = append(topicDetails.Partitions, partitionInfo)
}
}
// Process flat schema format
if configResp.MessageRecordType != nil {
for _, field := range configResp.MessageRecordType.Fields {
isKey := false
for _, keyCol := range configResp.KeyColumns {
if field.Name == keyCol {
isKey = true
break
}
}
fieldType := "UNKNOWN"
if field.Type != nil && field.Type.Kind != nil {
fieldType = getFieldTypeName(field.Type)
}
schemaField := SchemaFieldInfo{
Name: field.Name,
Type: fieldType,
}
if isKey {
topicDetails.KeySchema = append(topicDetails.KeySchema, schemaField)
} else {
topicDetails.ValueSchema = append(topicDetails.ValueSchema, schemaField)
}
}
}
// Get publishers information
publishersResp, err := client.GetTopicPublishers(ctx, &mq_pb.GetTopicPublishersRequest{
Topic: &schema_pb.Topic{
Namespace: namespace,
Name: topicName,
},
})
if err != nil {
// Log error but don't fail the entire request
glog.V(0).Infof("failed to get topic publishers for %s.%s: %v", namespace, topicName, err)
} else {
glog.V(1).Infof("got %d publishers for topic %s.%s", len(publishersResp.Publishers), namespace, topicName)
topicDetails.Publishers = convertTopicPublishers(publishersResp.Publishers)
}
// Get subscribers information
subscribersResp, err := client.GetTopicSubscribers(ctx, &mq_pb.GetTopicSubscribersRequest{
Topic: &schema_pb.Topic{
Namespace: namespace,
Name: topicName,
},
})
if err != nil {
// Log error but don't fail the entire request
glog.V(0).Infof("failed to get topic subscribers for %s.%s: %v", namespace, topicName, err)
} else {
glog.V(1).Infof("got %d subscribers for topic %s.%s", len(subscribersResp.Subscribers), namespace, topicName)
topicDetails.Subscribers = convertTopicSubscribers(subscribersResp.Subscribers)
}
return nil
})
if err != nil {
return nil, err
}
// Get consumer group offsets from the filer
offsets, err := s.GetConsumerGroupOffsets(namespace, topicName)
if err != nil {
// Log error but don't fail the entire request
glog.V(0).Infof("failed to get consumer group offsets for %s.%s: %v", namespace, topicName, err)
} else {
glog.V(1).Infof("got %d consumer group offsets for topic %s.%s", len(offsets), namespace, topicName)
topicDetails.ConsumerGroupOffsets = offsets
}
return topicDetails, nil
}
// GetConsumerGroupOffsets retrieves consumer group offsets for a topic from the filer
func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]ConsumerGroupOffsetInfo, error) {
var offsets []ConsumerGroupOffsetInfo
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Get the topic directory: /topics/namespace/topicName
topicObj := topic.NewTopic(namespace, topicName)
topicDir := topicObj.Dir()
// List all version directories under the topic directory (e.g., v2025-07-10-05-44-34)
versionStream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: topicDir,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return fmt.Errorf("failed to list topic directory %s: %v", topicDir, err)
}
// Process each version directory
for {
versionResp, err := versionStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to receive version entries: %w", err)
}
// Only process directories that are versions (start with "v")
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
versionDir := path.Join(topicDir, versionResp.Entry.Name)
// List all partition directories under the version directory (e.g., 0315-0630)
partitionStream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: versionDir,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
glog.Warningf("Failed to list version directory %s: %v", versionDir, err)
continue
}
// Process each partition directory
for {
partitionResp, err := partitionStream.Recv()
if err != nil {
if err == io.EOF {
break
}
glog.Warningf("Failed to receive partition entries: %v", err)
break
}
// Only process directories that are partitions (format: NNNN-NNNN)
if partitionResp.Entry != nil && partitionResp.Entry.IsDirectory {
// Parse partition range to get partition start ID (e.g., "0315-0630" -> 315)
var partitionStart, partitionStop int32
if n, err := fmt.Sscanf(partitionResp.Entry.Name, "%04d-%04d", &partitionStart, &partitionStop); n != 2 || err != nil {
// Skip directories that don't match the partition format
continue
}
partitionDir := path.Join(versionDir, partitionResp.Entry.Name)
// List all .offset files in this partition directory
offsetStream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: partitionDir,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
glog.Warningf("Failed to list partition directory %s: %v", partitionDir, err)
continue
}
// Process each offset file
for {
offsetResp, err := offsetStream.Recv()
if err != nil {
if err == io.EOF {
break
}
glog.Warningf("Failed to receive offset entries: %v", err)
break
}
// Only process .offset files
if offsetResp.Entry != nil && !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
consumerGroup := strings.TrimSuffix(offsetResp.Entry.Name, ".offset")
// Read the offset value from the file
offsetData, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, offsetResp.Entry.Name)
if err != nil {
glog.Warningf("Failed to read offset file %s: %v", offsetResp.Entry.Name, err)
continue
}
if len(offsetData) == 8 {
offset := int64(util.BytesToUint64(offsetData))
// Get the file modification time
var lastUpdated time.Time
if offsetResp.Entry.Attributes != nil {
lastUpdated = time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
}
offsets = append(offsets, ConsumerGroupOffsetInfo{
ConsumerGroup: consumerGroup,
PartitionID: partitionStart, // Use partition start as the ID
Offset: offset,
LastUpdated: lastUpdated,
})
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to get consumer group offsets: %w", err)
}
return offsets, nil
}
// convertTopicPublishers converts protobuf TopicPublisher slice to PublisherInfo slice
func convertTopicPublishers(publishers []*mq_pb.TopicPublisher) []PublisherInfo {
publisherInfos := make([]PublisherInfo, 0, len(publishers))
for _, publisher := range publishers {
publisherInfo := PublisherInfo{
PublisherName: publisher.PublisherName,
ClientID: publisher.ClientId,
PartitionID: publisher.Partition.RangeStart,
Broker: publisher.Broker,
IsActive: publisher.IsActive,
LastPublishedOffset: publisher.LastPublishedOffset,
LastAckedOffset: publisher.LastAckedOffset,
}
// Convert timestamps
if publisher.ConnectTimeNs > 0 {
publisherInfo.ConnectTime = time.Unix(0, publisher.ConnectTimeNs)
}
if publisher.LastSeenTimeNs > 0 {
publisherInfo.LastSeenTime = time.Unix(0, publisher.LastSeenTimeNs)
}
publisherInfos = append(publisherInfos, publisherInfo)
}
return publisherInfos
}
// convertTopicSubscribers converts protobuf TopicSubscriber slice to TopicSubscriberInfo slice
func convertTopicSubscribers(subscribers []*mq_pb.TopicSubscriber) []TopicSubscriberInfo {
subscriberInfos := make([]TopicSubscriberInfo, 0, len(subscribers))
for _, subscriber := range subscribers {
subscriberInfo := TopicSubscriberInfo{
ConsumerGroup: subscriber.ConsumerGroup,
ConsumerID: subscriber.ConsumerId,
ClientID: subscriber.ClientId,
PartitionID: subscriber.Partition.RangeStart,
Broker: subscriber.Broker,
IsActive: subscriber.IsActive,
CurrentOffset: subscriber.CurrentOffset,
LastReceivedOffset: subscriber.LastReceivedOffset,
}
// Convert timestamps
if subscriber.ConnectTimeNs > 0 {
subscriberInfo.ConnectTime = time.Unix(0, subscriber.ConnectTimeNs)
}
if subscriber.LastSeenTimeNs > 0 {
subscriberInfo.LastSeenTime = time.Unix(0, subscriber.LastSeenTimeNs)
}
subscriberInfos = append(subscriberInfos, subscriberInfo)
}
return subscriberInfos
}
// findBrokerLeader finds the current broker leader
func (s *AdminServer) findBrokerLeader() (string, error) {
// First, try to find any broker from the cluster
var brokers []string
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), s.listClusterNodesRequest(cluster.BrokerType))
if err != nil {
return err
}
for _, node := range resp.ClusterNodes {
brokers = append(brokers, node.Address)
}
return nil
})
if err != nil {
return "", fmt.Errorf("failed to list brokers: %w", err)
}
if len(brokers) == 0 {
return "", fmt.Errorf("no brokers found in cluster")
}
// Try each broker to find the leader
for _, brokerAddr := range brokers {
err := s.withBrokerClient(brokerAddr, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
// Try to find broker leader
_, err := client.FindBrokerLeader(ctx, &mq_pb.FindBrokerLeaderRequest{
FilerGroup: s.filerGroup,
})
if err == nil {
return nil // This broker is the leader
}
return err
})
if err == nil {
return brokerAddr, nil
}
}
return "", fmt.Errorf("no broker leader found")
}
// withBrokerClient connects to a message queue broker and executes a function
func (s *AdminServer) withBrokerClient(brokerAddress string, fn func(client mq_pb.SeaweedMessagingClient) error) error {
return pb.WithBrokerGrpcClient(false, brokerAddress, s.grpcDialOption, fn)
}
// convertTopicRetention converts protobuf retention to TopicRetentionInfo
func convertTopicRetention(retention *mq_pb.TopicRetention) TopicRetentionInfo {
if retention == nil || !retention.Enabled {
return TopicRetentionInfo{
Enabled: false,
RetentionSeconds: 0,
DisplayValue: 0,
DisplayUnit: "days",
}
}
// Convert seconds to human-readable format
seconds := retention.RetentionSeconds
var displayValue int32
var displayUnit string
if seconds >= 86400 { // >= 1 day
displayValue = int32(seconds / 86400)
displayUnit = "days"
} else if seconds >= 3600 { // >= 1 hour
displayValue = int32(seconds / 3600)
displayUnit = "hours"
} else {
displayValue = int32(seconds)
displayUnit = "seconds"
}
return TopicRetentionInfo{
Enabled: retention.Enabled,
RetentionSeconds: seconds,
DisplayValue: displayValue,
DisplayUnit: displayUnit,
}
}
// getFieldTypeName converts a schema_pb.Type to a human-readable type name
func getFieldTypeName(fieldType *schema_pb.Type) string {
if fieldType.Kind == nil {
return "UNKNOWN"
}
switch kind := fieldType.Kind.(type) {
case *schema_pb.Type_ScalarType:
switch kind.ScalarType {
case schema_pb.ScalarType_BOOL:
return "BOOLEAN"
case schema_pb.ScalarType_INT32:
return "INT32"
case schema_pb.ScalarType_INT64:
return "INT64"
case schema_pb.ScalarType_FLOAT:
return "FLOAT"
case schema_pb.ScalarType_DOUBLE:
return "DOUBLE"
case schema_pb.ScalarType_BYTES:
return "BYTES"
case schema_pb.ScalarType_STRING:
return "STRING"
case schema_pb.ScalarType_TIMESTAMP:
return "TIMESTAMP"
case schema_pb.ScalarType_DATE:
return "DATE"
case schema_pb.ScalarType_TIME:
return "TIME"
case schema_pb.ScalarType_DECIMAL:
return "DECIMAL"
default:
return "SCALAR"
}
case *schema_pb.Type_ListType:
return "LIST"
case *schema_pb.Type_RecordType:
return "RECORD"
default:
return "UNKNOWN"
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,232 @@
package dash
import (
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestExpirePluginJobAPI(t *testing.T) {
makeRequest := func(adminServer *AdminServer, jobID string, body io.Reader) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/api/plugin/jobs/"+jobID+"/expire", body)
req = mux.SetURLVars(req, map[string]string{"jobId": jobID})
recorder := httptest.NewRecorder()
adminServer.ExpirePluginJobAPI(recorder, req)
return recorder
}
t.Run("empty job id", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "", nil)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("invalid json", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "job-id", strings.NewReader("{"))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("job not found", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, plugin.ErrJobNotFound
},
}
recorder := makeRequest(adminServer, "missing", strings.NewReader(`{"reason":"nope"}`))
if recorder.Code != http.StatusNotFound {
t.Fatalf("expected 404, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to unmarshal body: %v", err)
}
if payload["error"] == nil {
t.Fatalf("expected error payload, got %v", payload)
}
})
t.Run("successful expire", func(t *testing.T) {
expected := &plugin.TrackedJob{JobID: "foo", State: "assigned"}
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
if jobID != "foo" {
return nil, false, errors.New("unexpected")
}
return expected, true, nil
},
}
recorder := makeRequest(adminServer, "foo", strings.NewReader(`{"reason":"cleanup"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "foo" {
t.Fatalf("expected job_id foo, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || !expired {
t.Fatalf("expected expired=true, got %v", payload["expired"])
}
jobData, ok := payload["job"].(map[string]any)
if !ok || jobData["job_id"] != "foo" {
t.Fatalf("expected job info with job_id, got %v", payload["job"])
}
})
t.Run("non-active job", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, nil
},
}
recorder := makeRequest(adminServer, "bar", strings.NewReader(`{"reason":"ignore"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "bar" {
t.Fatalf("expected job_id bar, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || expired {
t.Fatalf("expected expired=false, got %v", payload["expired"])
}
if payload["message"] != "job is not active" {
t.Fatalf("expected message job is not active, got %v", payload["message"])
}
if _, exists := payload["job"]; exists {
t.Fatalf("expected no job payload for non-active job, got %v", payload["job"])
}
})
}
func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{},
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
DetectionIntervalMinutes: 60,
DetectionTimeoutSeconds: 300,
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script default to be backfilled")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok || scriptKind.StringValue == "" {
t.Fatalf("expected non-empty script default, got=%+v", script)
}
if config.AdminRuntime.DetectionIntervalMinutes != 60 {
t.Fatalf("expected runtime detection interval default to be backfilled")
}
}
func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: " "},
},
},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
},
},
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script config value")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok {
t.Fatalf("expected string script config value, got=%T", script.Kind)
}
if scriptKind.StringValue != "volume.fix.replication -apply" {
t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
}
}
func TestFilterTrackedJobsByLane(t *testing.T) {
t.Parallel()
jobs := []plugin.TrackedJob{
{JobID: "vacuum-1", JobType: "vacuum"},
{JobID: "iceberg-1", JobType: "iceberg_maintenance"},
{JobID: "lifecycle-1", JobType: "s3_lifecycle"},
}
filtered := filterTrackedJobsByLane(jobs, "iceberg")
if len(filtered) != 1 {
t.Fatalf("expected 1 iceberg job, got %d", len(filtered))
}
if filtered[0].JobID != "iceberg-1" {
t.Fatalf("expected iceberg job to be retained, got %+v", filtered[0])
}
}
func TestFilterActivitiesByLane(t *testing.T) {
t.Parallel()
activities := []plugin.JobActivity{
{JobID: "vacuum-1", JobType: "vacuum"},
{JobID: "iceberg-1", JobType: "iceberg_maintenance"},
{JobID: "lifecycle-1", JobType: "s3_lifecycle"},
}
filtered := filterActivitiesByLane(activities, "lifecycle")
if len(filtered) != 1 {
t.Fatalf("expected 1 lifecycle activity, got %d", len(filtered))
}
if filtered[0].JobID != "lifecycle-1" {
t.Fatalf("expected lifecycle activity to be retained, got %+v", filtered[0])
}
}

View File

@@ -0,0 +1,254 @@
package dash
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
// ErrPolicyStillAttached is returned when deleting a managed policy that is
// still attached to one or more users or groups.
var ErrPolicyStillAttached = errors.New("policy is still attached")
type IAMPolicy struct {
Name string `json:"name"`
Document policy_engine.PolicyDocument `json:"document"`
DocumentJSON string `json:"document_json"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type PoliciesCollection struct {
Policies map[string]policy_engine.PolicyDocument `json:"policies"`
}
type PoliciesData struct {
Username string `json:"username"`
Policies []IAMPolicy `json:"policies"`
TotalPolicies int `json:"total_policies"`
LastUpdated time.Time `json:"last_updated"`
}
// Policy management request structures
type CreatePolicyRequest struct {
Name string `json:"name" binding:"required"`
Document policy_engine.PolicyDocument `json:"document" binding:"required"`
DocumentJSON string `json:"document_json"`
}
type UpdatePolicyRequest struct {
Document policy_engine.PolicyDocument `json:"document" binding:"required"`
DocumentJSON string `json:"document_json"`
}
// PolicyManager interface is now in the credential package
// CredentialStorePolicyManager implements credential.PolicyManager by delegating to the credential store
type CredentialStorePolicyManager struct {
credentialManager *credential.CredentialManager
}
// NewCredentialStorePolicyManager creates a new CredentialStorePolicyManager
func NewCredentialStorePolicyManager(credentialManager *credential.CredentialManager) *CredentialStorePolicyManager {
return &CredentialStorePolicyManager{
credentialManager: credentialManager,
}
}
// GetPolicies retrieves all IAM policies via credential store
// Deprecated: This method delegates to the credential manager and will be moved/removed in a future release.
func (cspm *CredentialStorePolicyManager) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) {
return cspm.credentialManager.GetPolicies(ctx)
}
// CreatePolicy creates a new IAM policy via credential store
// Deprecated: This method delegates to the credential manager and will be moved/removed in a future release.
func (cspm *CredentialStorePolicyManager) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
return cspm.credentialManager.CreatePolicy(ctx, name, document)
}
// UpdatePolicy updates an existing IAM policy via credential store
// Deprecated: This method delegates to the credential manager and will be moved/removed in a future release.
func (cspm *CredentialStorePolicyManager) UpdatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
return cspm.credentialManager.UpdatePolicy(ctx, name, document)
}
// DeletePolicy deletes an IAM policy via credential store
// Deprecated: This method delegates to the credential manager and will be moved/removed in a future release.
func (cspm *CredentialStorePolicyManager) DeletePolicy(ctx context.Context, name string) error {
return cspm.credentialManager.DeletePolicy(ctx, name)
}
// GetPolicy retrieves a specific IAM policy via credential store
// Deprecated: This method delegates to the credential manager and will be moved/removed in a future release.
func (cspm *CredentialStorePolicyManager) GetPolicy(ctx context.Context, name string) (*policy_engine.PolicyDocument, error) {
return cspm.credentialManager.GetPolicy(ctx, name)
}
// AdminServer policy management methods using credential.PolicyManager
func (s *AdminServer) GetPolicyManager() credential.PolicyManager {
if s.credentialManager == nil {
glog.V(1).Infof("Credential manager is nil, policy management not available")
return nil
}
glog.V(1).Infof("Credential manager available, creating CredentialStorePolicyManager")
return NewCredentialStorePolicyManager(s.credentialManager)
}
// GetPolicies retrieves all IAM policies
func (s *AdminServer) GetPolicies() ([]IAMPolicy, error) {
policyManager := s.GetPolicyManager()
if policyManager == nil {
return nil, fmt.Errorf("policy manager not available")
}
ctx := context.Background()
policyMap, err := policyManager.GetPolicies(ctx)
if err != nil {
return nil, err
}
// Convert map[string]PolicyDocument to []IAMPolicy
var policies []IAMPolicy
for name, doc := range policyMap {
policy := IAMPolicy{
Name: name,
Document: doc,
DocumentJSON: "", // Will be populated if needed
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
policies = append(policies, policy)
}
return policies, nil
}
// CreatePolicy creates a new IAM policy
func (s *AdminServer) CreatePolicy(name string, document policy_engine.PolicyDocument) error {
policyManager := s.GetPolicyManager()
if policyManager == nil {
return fmt.Errorf("policy manager not available")
}
ctx := context.Background()
return policyManager.CreatePolicy(ctx, name, document)
}
// UpdatePolicy updates an existing IAM policy
func (s *AdminServer) UpdatePolicy(name string, document policy_engine.PolicyDocument) error {
policyManager := s.GetPolicyManager()
if policyManager == nil {
return fmt.Errorf("policy manager not available")
}
ctx := context.Background()
return policyManager.UpdatePolicy(ctx, name, document)
}
// DeletePolicy deletes an IAM policy. Deletion is rejected while the policy is
// still attached to any user or group, matching AWS IAM behavior and the IAM
// API handler, so a deleted policy name never lingers in an attached policy
// list.
func (s *AdminServer) DeletePolicy(name string) error {
policyManager := s.GetPolicyManager()
if policyManager == nil {
return fmt.Errorf("policy manager not available")
}
ctx := context.Background()
attached, err := s.IsPolicyAttached(ctx, name)
if err != nil {
return fmt.Errorf("failed to check policy attachments: %w", err)
}
if len(attached) > 0 {
return fmt.Errorf("policy %s is still attached to: %s: %w", name, strings.Join(attached, ", "), ErrPolicyStillAttached)
}
return policyManager.DeletePolicy(ctx, name)
}
// IsPolicyAttached returns the names of users and groups that still have the
// given managed policy attached. The returned entries are prefixed with
// "user:" or "group:". Returns nil when the policy is not attached anywhere.
func (s *AdminServer) IsPolicyAttached(ctx context.Context, policyName string) ([]string, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
var attached []string
usernames, err := s.credentialManager.ListUsers(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list users: %w", err)
}
for _, username := range usernames {
policies, err := s.credentialManager.ListAttachedUserPolicies(ctx, username)
if err != nil {
return nil, fmt.Errorf("failed to list policies for user %s: %w", username, err)
}
for _, p := range policies {
if p == policyName {
attached = append(attached, "user:"+username)
break
}
}
}
groupNames, err := s.credentialManager.ListGroups(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list groups: %w", err)
}
for _, groupName := range groupNames {
group, err := s.credentialManager.GetGroup(ctx, groupName)
if errors.Is(err, credential.ErrGroupNotFound) {
continue
}
if err != nil {
return nil, fmt.Errorf("failed to get group %s: %w", groupName, err)
}
for _, p := range group.PolicyNames {
if p == policyName {
attached = append(attached, "group:"+groupName)
break
}
}
}
return attached, nil
}
// GetPolicy retrieves a specific IAM policy
func (s *AdminServer) GetPolicy(name string) (*IAMPolicy, error) {
policyManager := s.GetPolicyManager()
if policyManager == nil {
return nil, fmt.Errorf("policy manager not available")
}
ctx := context.Background()
policyDoc, err := policyManager.GetPolicy(ctx, name)
if err != nil {
return nil, err
}
if policyDoc == nil {
return nil, nil
}
// Convert PolicyDocument to IAMPolicy
policy := &IAMPolicy{
Name: name,
Document: *policyDoc,
DocumentJSON: "", // Will be populated if needed
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
return policy, nil
}

View File

@@ -0,0 +1,160 @@
package dash
import (
"context"
"errors"
"testing"
"github.com/seaweedfs/seaweedfs/weed/credential"
_ "github.com/seaweedfs/seaweedfs/weed/credential/memory" // register memory store
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
func newAdminServerWithMemoryStore(t *testing.T) *AdminServer {
t.Helper()
cm, err := credential.NewCredentialManagerWithDefaults(credential.StoreTypeMemory)
if err != nil {
t.Fatalf("failed to create credential manager: %v", err)
}
return &AdminServer{credentialManager: cm}
}
func samplePolicyDocument() policy_engine.PolicyDocument {
return policy_engine.PolicyDocument{
Version: "2012-10-17",
Statement: []policy_engine.PolicyStatement{{
Effect: policy_engine.PolicyEffectAllow,
Action: policy_engine.NewStringOrStringSlice("s3:GetObject"),
Resource: policy_engine.NewStringOrStringSlicePtr("arn:aws:s3:::test/*"),
}},
}
}
func TestIsPolicyAttached(t *testing.T) {
server := newAdminServerWithMemoryStore(t)
ctx := context.Background()
const policyName = "policy_a"
if err := server.CreatePolicy(policyName, samplePolicyDocument()); err != nil {
t.Fatalf("CreatePolicy: %v", err)
}
if attached, err := server.IsPolicyAttached(ctx, policyName); err != nil {
t.Fatalf("IsPolicyAttached: %v", err)
} else if len(attached) != 0 {
t.Fatalf("expected no attachments, got %v", attached)
}
if err := server.credentialManager.CreateUser(ctx, &iam_pb.Identity{Name: "alice"}); err != nil {
t.Fatalf("CreateUser: %v", err)
}
if err := server.credentialManager.AttachUserPolicy(ctx, "alice", policyName); err != nil {
t.Fatalf("AttachUserPolicy: %v", err)
}
attached, err := server.IsPolicyAttached(ctx, policyName)
if err != nil {
t.Fatalf("IsPolicyAttached: %v", err)
}
if len(attached) != 1 || attached[0] != "user:alice" {
t.Fatalf("expected [user:alice], got %v", attached)
}
if err := server.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "devs", PolicyNames: []string{policyName}}); err != nil {
t.Fatalf("CreateGroup: %v", err)
}
attached, err = server.IsPolicyAttached(ctx, policyName)
if err != nil {
t.Fatalf("IsPolicyAttached: %v", err)
}
if len(attached) != 2 {
t.Fatalf("expected 2 attachments, got %v", attached)
}
}
func TestDeletePolicyRejectsWhenAttachedToUser(t *testing.T) {
server := newAdminServerWithMemoryStore(t)
ctx := context.Background()
const policyName = "policy_u"
if err := server.CreatePolicy(policyName, samplePolicyDocument()); err != nil {
t.Fatalf("CreatePolicy: %v", err)
}
if err := server.credentialManager.CreateUser(ctx, &iam_pb.Identity{Name: "bob"}); err != nil {
t.Fatalf("CreateUser: %v", err)
}
if err := server.credentialManager.AttachUserPolicy(ctx, "bob", policyName); err != nil {
t.Fatalf("AttachUserPolicy: %v", err)
}
if err := server.DeletePolicy(policyName); !errors.Is(err, ErrPolicyStillAttached) {
t.Fatalf("expected ErrPolicyStillAttached, got %v", err)
}
if p, err := server.GetPolicy(policyName); err != nil {
t.Fatalf("policy should still exist after rejected deletion: %v", err)
} else if p == nil {
t.Fatal("policy should still exist after rejected deletion, got nil")
}
attached, err := server.credentialManager.ListAttachedUserPolicies(ctx, "bob")
if err != nil {
t.Fatalf("ListAttachedUserPolicies: %v", err)
}
if len(attached) != 1 || attached[0] != policyName {
t.Fatalf("expected policy %q to remain attached, got %v", policyName, attached)
}
if err := server.credentialManager.DetachUserPolicy(ctx, "bob", policyName); err != nil {
t.Fatalf("DetachUserPolicy: %v", err)
}
if err := server.DeletePolicy(policyName); err != nil {
t.Fatalf("DeletePolicy after detach failed: %v", err)
}
if p, err := server.GetPolicy(policyName); err != nil {
t.Fatalf("GetPolicy after detach-delete: %v", err)
} else if p != nil {
t.Fatalf("policy should be gone after deletion, got %v", p)
}
}
func TestDeletePolicyRejectsWhenAttachedToGroup(t *testing.T) {
server := newAdminServerWithMemoryStore(t)
ctx := context.Background()
const policyName = "policy_g"
if err := server.CreatePolicy(policyName, samplePolicyDocument()); err != nil {
t.Fatalf("CreatePolicy: %v", err)
}
if err := server.credentialManager.CreateGroup(ctx, &iam_pb.Group{Name: "team_g", PolicyNames: []string{policyName}}); err != nil {
t.Fatalf("CreateGroup: %v", err)
}
if err := server.DeletePolicy(policyName); !errors.Is(err, ErrPolicyStillAttached) {
t.Fatalf("expected ErrPolicyStillAttached, got %v", err)
}
if p, err := server.GetPolicy(policyName); err != nil {
t.Fatalf("policy should still exist after rejected deletion: %v", err)
} else if p == nil {
t.Fatal("policy should still exist after rejected deletion, got nil")
}
}
func TestDeletePolicySucceedsWhenNotAttached(t *testing.T) {
server := newAdminServerWithMemoryStore(t)
const policyName = "policy_free"
if err := server.CreatePolicy(policyName, samplePolicyDocument()); err != nil {
t.Fatalf("CreatePolicy: %v", err)
}
if err := server.DeletePolicy(policyName); err != nil {
t.Fatalf("DeletePolicy for unattached policy failed: %v", err)
}
if p, err := server.GetPolicy(policyName); err != nil {
t.Fatalf("GetPolicy after delete: %v", err)
} else if p != nil {
t.Fatalf("policy should be gone after deletion, got %v", p)
}
}

View File

@@ -0,0 +1,57 @@
package dash
import (
"context"
weediam "github.com/seaweedfs/seaweedfs/weed/iam"
"github.com/seaweedfs/seaweedfs/weed/iam/integration"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// principalRoleArn builds the ARN SeaweedFS assigns a role by default (when
// its RoleDefinition.RoleArn isn't explicitly set) - see
// weed/iam/integration/iam_manager.go's CreateRole. ListRoles only returns
// role names, so this reconstructs the well-known default rather than
// fetching every role's stored definition just to populate a suggestion list.
func principalRoleArn(roleName string) string {
return "arn:aws:iam::role/" + roleName
}
// GetPrincipalSuggestions returns candidate ARNs for the policy editor's
// Principal/NotPrincipal autocomplete: one per S3 user, plus one per IAM
// role. Service accounts are deliberately not listed separately - a service
// account is just an additional credential for its parent user, so its ARN
// is identical to the one already suggested for that user.
//
// Role listing is best-effort: if the filer or role store is unavailable,
// the error is logged and suggestions fall back to users only, since an
// incomplete autocomplete list is far less disruptive than blocking policy
// editing over a suggestions-only feature.
func (s *AdminServer) GetPrincipalSuggestions(ctx context.Context) ([]string, error) {
var suggestions []string
users, err := s.GetObjectStoreUsers(ctx)
if err != nil {
return nil, err
}
for _, u := range users {
suggestions = append(suggestions, weediam.UserArn(u.Username))
}
roleStore, err := integration.NewFilerRoleStore(nil, func() string { return s.GetFilerAddress() })
if err != nil {
glog.Warningf("GetPrincipalSuggestions: failed to create role store: %v", err)
return suggestions, nil
}
roleNames, err := roleStore.ListRoles(ctx, s.GetFilerAddress())
if err != nil {
glog.Warningf("GetPrincipalSuggestions: failed to list roles: %v", err)
return suggestions, nil
}
for _, roleName := range roleNames {
suggestions = append(suggestions, principalRoleArn(roleName))
}
return suggestions, nil
}

View File

@@ -0,0 +1,11 @@
package dash
import "testing"
func TestPrincipalRoleArn(t *testing.T) {
got := principalRoleArn("S3ReadOnlyRole")
want := "arn:aws:iam::role/S3ReadOnlyRole"
if got != want {
t.Fatalf("principalRoleArn() = %q, want %q", got, want)
}
}

View File

@@ -0,0 +1,75 @@
package dash
import (
"testing"
)
// TestIsReadOnlyRole verifies that only the read-only session role is treated
// as view-only. The admin role and the empty role (used when auth is
// disabled) must not be treated as read-only, otherwise admins and no-auth
// deployments would lose legitimate access to credentials.
func TestIsReadOnlyRole(t *testing.T) {
if !IsReadOnlyRole(RoleReadOnly) {
t.Errorf("IsReadOnlyRole(%q) = false, want true", RoleReadOnly)
}
if !IsReadOnlyRole("readonly") {
t.Errorf("IsReadOnlyRole(\"readonly\") = false, want true")
}
if IsReadOnlyRole("admin") {
t.Errorf("IsReadOnlyRole(\"admin\") = true, want false")
}
if IsReadOnlyRole("") {
t.Errorf("IsReadOnlyRole(\"\") = true, want false; no-auth mode must not be redacted")
}
}
// TestObjectStoreUserRedactSecretKey verifies that redaction clears the
// reusable secret while preserving the public access key identifier.
func TestObjectStoreUserRedactSecretKey(t *testing.T) {
u := ObjectStoreUser{
Username: "victim",
AccessKey: "AKIAEXAMPLEKEY",
SecretKey: "super-secret-value",
}
u.RedactSecretKey()
if u.SecretKey != "" {
t.Errorf("SecretKey = %q, want empty after redaction", u.SecretKey)
}
if u.AccessKey == "" {
t.Errorf("AccessKey was emptied; only the secret should be redacted")
}
if u.Username == "" {
t.Errorf("Username was emptied; only the secret should be redacted")
}
}
// TestUserDetailsRedactSecretKeys verifies that redaction clears every
// access key's secret while preserving the access key identifiers.
func TestUserDetailsRedactSecretKeys(t *testing.T) {
d := &UserDetails{
Username: "victim",
AccessKeys: []AccessKeyInfo{
{AccessKey: "AKIAONE", SecretKey: "secret-one"},
{AccessKey: "AKIATWO", SecretKey: "secret-two"},
},
}
d.RedactSecretKeys()
for i, ak := range d.AccessKeys {
if ak.SecretKey != "" {
t.Errorf("AccessKeys[%d].SecretKey = %q, want empty after redaction", i, ak.SecretKey)
}
if ak.AccessKey == "" {
t.Errorf("AccessKeys[%d].AccessKey was emptied; only the secret should be redacted", i)
}
}
}
// TestUserDetailsRedactSecretKeys_Empty verifies redaction is a no-op
// (and does not panic) when there are no access keys.
func TestUserDetailsRedactSecretKeys_Empty(t *testing.T) {
d := &UserDetails{Username: "victim"}
d.RedactSecretKeys()
if len(d.AccessKeys) != 0 {
t.Errorf("expected no access keys, got %d", len(d.AccessKeys))
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,308 @@
package dash
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iam"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
var (
// ErrServiceAccountNotFound is returned when a service account is not found
ErrServiceAccountNotFound = errors.New("service account not found")
)
const (
serviceAccountPrefix = "sa:"
accessKeyPrefix = "ABIA" // Service account access keys use ABIA prefix
// Status constants
StatusActive = iam.AccessKeyStatusActive
StatusInactive = iam.AccessKeyStatusInactive
)
// GetServiceAccounts returns all service accounts, optionally filtered by parent user
func (s *AdminServer) GetServiceAccounts(ctx context.Context, parentUser string) ([]ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
pbAccounts, err := s.credentialManager.ListServiceAccounts(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list service accounts: %w", err)
}
var accounts []ServiceAccount
for _, sa := range pbAccounts {
if sa == nil {
continue
}
// Filter by parent user if specified
if parentUser != "" && sa.ParentUser != parentUser {
continue
}
status := StatusActive
if sa.Disabled {
status = StatusInactive
}
account := ServiceAccount{
ID: sa.Id,
ParentUser: sa.ParentUser,
Description: sa.Description,
Status: status,
CreateDate: time.Unix(sa.CreatedAt, 0),
}
if sa.Expiration > 0 {
account.Expiration = time.Unix(sa.Expiration, 0)
}
if sa.Credential != nil {
account.AccessKeyId = sa.Credential.AccessKey
}
accounts = append(accounts, account)
}
return accounts, nil
}
// GetServiceAccountDetails returns detailed information about a specific service account
func (s *AdminServer) GetServiceAccountDetails(ctx context.Context, id string) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
sa, err := s.credentialManager.GetServiceAccount(ctx, id)
if err != nil {
return nil, fmt.Errorf("failed to get service account: %w", err)
}
if sa == nil {
return nil, ErrServiceAccountNotFound
}
status := StatusActive
if sa.Disabled {
status = StatusInactive
}
account := &ServiceAccount{
ID: sa.Id,
ParentUser: sa.ParentUser,
Description: sa.Description,
Status: status,
CreateDate: time.Unix(sa.CreatedAt, 0),
}
if sa.Expiration > 0 {
account.Expiration = time.Unix(sa.Expiration, 0)
}
if sa.Credential != nil {
account.AccessKeyId = sa.Credential.AccessKey
}
return account, nil
}
// CreateServiceAccount creates a new service account for a parent user
func (s *AdminServer) CreateServiceAccount(ctx context.Context, req CreateServiceAccountRequest) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Validate parent user exists
if _, err := s.credentialManager.GetUser(ctx, req.ParentUser); err != nil {
return nil, fmt.Errorf("parent user lookup failed for %s: %w", req.ParentUser, err)
}
// Generate unique ID and credentials
uuid := generateAccountId()
// Maintain consistent ID format: sa:<parent>:<uuid>
saId := fmt.Sprintf("sa:%s:%s", req.ParentUser, uuid)
accessKey := accessKeyPrefix + generateAccessKey()[len(accessKeyPrefix):]
secretKey := generateSecretKey()
now := time.Now()
sa := &iam_pb.ServiceAccount{
Id: saId,
ParentUser: req.ParentUser,
Description: req.Description,
Credential: &iam_pb.Credential{
AccessKey: accessKey,
SecretKey: secretKey,
Status: StatusActive,
},
CreatedAt: now.Unix(),
Disabled: false,
}
if req.Expiration != "" {
exp, err := time.Parse(time.RFC3339, req.Expiration)
if err != nil {
return nil, fmt.Errorf("invalid expiration format: %w", err)
}
sa.Expiration = exp.Unix()
}
if err := s.credentialManager.CreateServiceAccount(ctx, sa); err != nil {
return nil, fmt.Errorf("failed to create service account: %w", err)
}
glog.V(1).Infof("Created service account %s for user %s", saId, req.ParentUser)
resp := &ServiceAccount{
ID: saId,
ParentUser: req.ParentUser,
Description: req.Description,
AccessKeyId: accessKey,
SecretAccessKey: secretKey, // Only returned on creation
Status: StatusActive,
CreateDate: now,
}
if sa.Expiration > 0 {
resp.Expiration = time.Unix(sa.Expiration, 0)
}
return resp, nil
}
// UpdateServiceAccount updates an existing service account
func (s *AdminServer) UpdateServiceAccount(ctx context.Context, id string, req UpdateServiceAccountRequest) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
sa, err := s.credentialManager.GetServiceAccount(ctx, id)
if err != nil {
return nil, fmt.Errorf("failed to get service account: %w", err)
}
if sa == nil {
return nil, ErrServiceAccountNotFound
}
if req.Description != "" {
sa.Description = req.Description
}
if req.Status != "" {
switch req.Status {
case StatusInactive:
sa.Disabled = true
case StatusActive:
sa.Disabled = false
default:
return nil, fmt.Errorf("invalid status value: %s (must be %s or %s)", req.Status, StatusActive, StatusInactive)
}
}
if req.Expiration != "" {
exp, err := time.Parse(time.RFC3339, req.Expiration)
if err != nil {
return nil, fmt.Errorf("invalid expiration format: %w", err)
}
sa.Expiration = exp.Unix()
}
if err := s.credentialManager.UpdateServiceAccount(ctx, id, sa); err != nil {
return nil, fmt.Errorf("failed to update service account: %w", err)
}
glog.V(1).Infof("Updated service account %s", id)
status := StatusActive
if sa.Disabled {
status = StatusInactive
}
accessKeyId := ""
if sa.Credential != nil {
accessKeyId = sa.Credential.AccessKey
}
resp := &ServiceAccount{
ID: sa.Id,
ParentUser: sa.ParentUser,
Description: sa.Description,
Status: status,
CreateDate: time.Unix(sa.CreatedAt, 0),
AccessKeyId: accessKeyId,
}
if sa.Expiration > 0 {
resp.Expiration = time.Unix(sa.Expiration, 0)
}
return resp, nil
}
// DeleteServiceAccount deletes a service account
func (s *AdminServer) DeleteServiceAccount(ctx context.Context, id string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
// Verify existence
sa, err := s.credentialManager.GetServiceAccount(ctx, id)
if err != nil {
return fmt.Errorf("failed to check service account: %w", err)
}
if sa == nil {
return ErrServiceAccountNotFound
}
if err := s.credentialManager.DeleteServiceAccount(ctx, id); err != nil {
return fmt.Errorf("failed to delete service account: %w", err)
}
glog.V(1).Infof("Deleted service account %s", id)
return nil
}
// GetServiceAccountByAccessKey finds a service account by its access key
func (s *AdminServer) GetServiceAccountByAccessKey(ctx context.Context, accessKey string) (*ServiceAccount, error) {
if !strings.HasPrefix(accessKey, accessKeyPrefix) {
return nil, fmt.Errorf("not a service account access key")
}
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Efficient lookup is now supported by the interface and optimized stores
sa, err := s.credentialManager.GetStore().GetServiceAccountByAccessKey(ctx, accessKey)
if err != nil {
return nil, fmt.Errorf("failed to find service account: %w", err)
}
status := StatusActive
if sa.Disabled {
status = StatusInactive
}
accessKeyId := ""
if sa.Credential != nil {
accessKeyId = sa.Credential.AccessKey
}
resp := &ServiceAccount{
ID: sa.Id,
ParentUser: sa.ParentUser,
Description: sa.Description,
AccessKeyId: accessKeyId,
Status: status,
CreateDate: time.Unix(sa.CreatedAt, 0),
}
if sa.Expiration > 0 {
resp.Expiration = time.Unix(sa.Expiration, 0)
}
return resp, nil
}

View File

@@ -0,0 +1,184 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/ec_balance"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/vacuum"
"google.golang.org/protobuf/proto"
)
// TestLoadTaskPolicyDefaultsMatchTaskDefaults pins the persistence layer's "nothing saved
// yet" defaults to each task's own NewDefaultConfig(). They used to be a second, hand-written
// copy and had drifted: with a data directory but no config file on disk, vacuum ran on a 24h
// scan interval instead of 2h, balance on 6h with a 0.1 imbalance threshold instead of 30m
// with 0.2, and erasure coding on 168h with a 0.90 fullness ratio and a 1024MB minimum volume
// size instead of 1h with 0.95 and 30MB - none of which is what the admin UI shows as the
// default for those fields.
func TestLoadTaskPolicyDefaultsMatchTaskDefaults(t *testing.T) {
cases := []struct {
name string
want *worker_pb.TaskPolicy
load func(cp *ConfigPersistence) (*worker_pb.TaskPolicy, error)
}{
{
name: "vacuum",
want: vacuum.NewDefaultConfig().ToTaskPolicy(),
load: func(cp *ConfigPersistence) (*worker_pb.TaskPolicy, error) { return cp.LoadVacuumTaskPolicy() },
},
{
name: "erasure_coding",
want: erasure_coding.NewDefaultConfig().ToTaskPolicy(),
load: func(cp *ConfigPersistence) (*worker_pb.TaskPolicy, error) {
return cp.LoadErasureCodingTaskPolicy()
},
},
{
name: "balance",
want: balance.NewDefaultConfig().ToTaskPolicy(),
load: func(cp *ConfigPersistence) (*worker_pb.TaskPolicy, error) { return cp.LoadBalanceTaskPolicy() },
},
{
name: "ec_balance",
want: ec_balance.NewDefaultConfig().ToTaskPolicy(),
load: func(cp *ConfigPersistence) (*worker_pb.TaskPolicy, error) { return cp.LoadEcBalanceTaskPolicy() },
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
// Both no-file branches have to agree with the task's own defaults: no data
// directory at all, and a data directory that has never been written to.
for _, cp := range []*ConfigPersistence{NewConfigPersistence(""), NewConfigPersistence(t.TempDir())} {
got, err := tc.load(cp)
if err != nil {
t.Fatalf("load %s policy: %v", tc.name, err)
}
if !proto.Equal(got, tc.want) {
t.Errorf("%s default policy (dataDir=%q) =\n %v\nwant NewDefaultConfig().ToTaskPolicy() =\n %v",
tc.name, cp.GetDataDir(), got, tc.want)
}
}
})
}
}
// TestLoadTaskConfigDefaultsMatchTaskDefaults covers the narrower Load*TaskConfig accessors,
// which carried a third copy of the same defaults.
func TestLoadTaskConfigDefaultsMatchTaskDefaults(t *testing.T) {
cp := NewConfigPersistence(t.TempDir())
vacuumConfig, err := cp.LoadVacuumTaskConfig()
if err != nil {
t.Fatalf("load vacuum config: %v", err)
}
if want := vacuum.NewDefaultConfig().ToTaskPolicy().GetVacuumConfig(); !proto.Equal(vacuumConfig, want) {
t.Errorf("vacuum default config = %v, want %v", vacuumConfig, want)
}
ecConfig, err := cp.LoadErasureCodingTaskConfig()
if err != nil {
t.Fatalf("load erasure coding config: %v", err)
}
if want := erasure_coding.NewDefaultConfig().ToTaskPolicy().GetErasureCodingConfig(); !proto.Equal(ecConfig, want) {
t.Errorf("erasure coding default config = %v, want %v", ecConfig, want)
}
balanceConfig, err := cp.LoadBalanceTaskConfig()
if err != nil {
t.Fatalf("load balance config: %v", err)
}
if want := balance.NewDefaultConfig().ToTaskPolicy().GetBalanceConfig(); !proto.Equal(balanceConfig, want) {
t.Errorf("balance default config = %v, want %v", balanceConfig, want)
}
}
// TestEcBalanceTaskPolicyRoundTrip checks the accessor ec_balance.LoadConfigFromPersistence
// asserts on. Before it existed, ec_balance was the one registered maintenance task whose
// configuration could not be persisted at all.
func TestEcBalanceTaskPolicyRoundTrip(t *testing.T) {
cp := NewConfigPersistence(t.TempDir())
saved := ec_balance.NewDefaultConfig()
saved.Enabled = false
saved.MinServerCount = 9
saved.ImbalanceThreshold = 0.42
saved.CollectionFilter = "pictures"
if err := cp.SaveEcBalanceTaskPolicy(saved.ToTaskPolicy()); err != nil {
t.Fatalf("save ec_balance policy: %v", err)
}
loaded := ec_balance.LoadConfigFromPersistence(cp)
if loaded == nil {
t.Fatal("ec_balance.LoadConfigFromPersistence returned nil")
}
if loaded.Enabled {
t.Error("ec_balance enabled = true, want the persisted false")
}
if loaded.MinServerCount != 9 {
t.Errorf("ec_balance min server count = %d, want the persisted 9", loaded.MinServerCount)
}
if loaded.ImbalanceThreshold != 0.42 {
t.Errorf("ec_balance imbalance threshold = %v, want the persisted 0.42", loaded.ImbalanceThreshold)
}
if loaded.CollectionFilter != "pictures" {
t.Errorf("ec_balance collection filter = %q, want the persisted %q", loaded.CollectionFilter, "pictures")
}
// The generic dispatcher the maintenance manager uses has to know the type too, and
// has to write the same file the dedicated loader reads.
dispatched := NewConfigPersistence(t.TempDir())
if err := dispatched.SaveTaskPolicy("ec_balance", saved.ToTaskPolicy()); err != nil {
t.Fatalf("SaveTaskPolicy(ec_balance): %v", err)
}
roundTripped := ec_balance.LoadConfigFromPersistence(dispatched)
if roundTripped == nil || roundTripped.MinServerCount != 9 {
t.Errorf("SaveTaskPolicy(ec_balance) did not round-trip through the ec_balance store: %+v", roundTripped)
}
}
// TestBuildPolicyKeepsTaskSpecificFields guards the fields the hand-written policy builder
// used to drop on the floor: the erasure coding preferred tags and replica placement, and
// the balance IO rate limit. Building each entry from the task's own ToTaskPolicy() keeps
// them, so a value set in admin.toml survives into the maintenance policy.
func TestBuildPolicyKeepsTaskSpecificFields(t *testing.T) {
cp := NewConfigPersistence(t.TempDir())
ecConfig := erasure_coding.NewDefaultConfig()
ecConfig.PreferredTags = []string{"ssd", "archive"}
ecConfig.ReplicaPlacement = "020"
if err := cp.SaveErasureCodingTaskPolicy(ecConfig.ToTaskPolicy()); err != nil {
t.Fatalf("save erasure coding policy: %v", err)
}
balanceConfig := balance.NewDefaultConfig()
balanceConfig.IoBytePerSecond = 5 << 20
if err := cp.SaveBalanceTaskPolicy(balanceConfig.ToTaskPolicy()); err != nil {
t.Fatalf("save balance policy: %v", err)
}
policy := cp.buildPolicyFromTaskConfigs()
ecPolicy := policy.TaskPolicies["erasure_coding"].GetErasureCodingConfig()
if ecPolicy == nil {
t.Fatal("no erasure coding config in the built policy")
}
if got := ecPolicy.GetReplicaPlacement(); got != "020" {
t.Errorf("erasure coding replica placement = %q, want the persisted %q", got, "020")
}
if got := ecPolicy.GetPreferredTags(); len(got) != 2 || got[0] != "ssd" || got[1] != "archive" {
t.Errorf("erasure coding preferred tags = %v, want the persisted [ssd archive]", got)
}
balancePolicy := policy.TaskPolicies["balance"].GetBalanceConfig()
if balancePolicy == nil {
t.Fatal("no balance config in the built policy")
}
if got := balancePolicy.GetIoBytePerSecond(); got != 5<<20 {
t.Errorf("balance IO limit = %d, want the persisted %d", got, 5<<20)
}
}

View File

@@ -0,0 +1,134 @@
package dash
import (
"sort"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// TierStats aggregates the volumes and EC shards that live on one storage
// tier: a local disk type ("hdd", "ssd", or a custom tag), or the remote
// storage a tiered volume was uploaded to. A remote-tiered volume reports
// the size of its cloud object, so its bytes belong to the remote tier,
// not to the local disk that holds only its index.
type TierStats struct {
Name string `json:"name"`
IsRemote bool `json:"is_remote"`
VolumeCount int `json:"volume_count"`
EcShardCount int `json:"ec_shard_count"`
DataSize int64 `json:"data_size"`
DiskUsed int64 `json:"disk_used"`
DiskCapacity int64 `json:"disk_capacity"`
MaxVolumes int64 `json:"max_volumes"`
}
// UsagePercent is the tier's local disk usage in percent, clamped to
// [0, 100]. Remote tiers have no capacity and return 0.
func (t TierStats) UsagePercent() int {
if t.IsRemote || t.DiskCapacity <= 0 {
return 0
}
percent := int(float64(t.DiskUsed) / float64(t.DiskCapacity) * 100)
if percent < 0 {
return 0
}
if percent > 100 {
return 100
}
return percent
}
// tierDiskType maps the empty disk type to its display name.
func tierDiskType(diskType string) string {
if diskType == "" {
return "hdd"
}
return diskType
}
// CollectTierStats walks the topology and groups capacity and usage by
// tier. DiskUsed/DiskCapacity come from the statfs numbers the volume
// servers report per disk type; a disk that predates disk_total_bytes
// falls back to the slot-based capacity estimate and to the logical
// bytes it holds, so mixed-version tiers don't underreport usage.
// Remote tiers have no local disk, so only VolumeCount and DataSize are
// meaningful there.
func CollectTierStats(topo *master_pb.TopologyInfo, volumeSizeLimitMb uint64) []TierStats {
if topo == nil {
return nil
}
tiers := make(map[string]*TierStats)
tier := func(name string, isRemote bool) *TierStats {
key := name
if isRemote {
key = "remote\x00" + name
}
t := tiers[key]
if t == nil {
t = &TierStats{Name: name, IsRemote: isRemote}
tiers[key] = t
}
return t
}
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
local := tier(tierDiskType(diskInfo.Type), false)
local.MaxVolumes += diskInfo.MaxVolumeCount
hasStatfs := diskInfo.DiskTotalBytes > 0
if hasStatfs {
local.DiskCapacity += int64(diskInfo.DiskTotalBytes)
if diskInfo.DiskTotalBytes > diskInfo.DiskFreeBytes {
local.DiskUsed += int64(diskInfo.DiskTotalBytes - diskInfo.DiskFreeBytes)
}
} else {
local.DiskCapacity += diskInfo.MaxVolumeCount * int64(volumeSizeLimitMb) * 1024 * 1024
}
var diskLocalBytes int64
for _, volInfo := range diskInfo.VolumeInfos {
if volInfo.RemoteStorageName != "" {
remote := tier(volInfo.RemoteStorageName, true)
remote.VolumeCount++
remote.DataSize += int64(volInfo.Size)
} else {
local.VolumeCount++
local.DataSize += int64(volInfo.Size)
diskLocalBytes += int64(volInfo.Size)
}
}
// ShardSizes is local to this node, so summing across
// nodes gives the tier's physical footprint.
for _, ecShardInfo := range diskInfo.EcShardInfos {
local.EcShardCount += erasure_coding.GetShardCount(ecShardInfo)
ecBytes := erasure_coding.EcShardsTotalSize(ecShardInfo)
local.DataSize += ecBytes
diskLocalBytes += ecBytes
}
// Without statfs numbers, approximate this disk's
// footprint with the logical bytes it holds.
if !hasStatfs {
local.DiskUsed += diskLocalBytes
}
}
}
}
}
result := make([]TierStats, 0, len(tiers))
for _, t := range tiers {
result = append(result, *t)
}
sort.Slice(result, func(i, j int) bool {
if result[i].IsRemote != result[j].IsRemote {
return !result[i].IsRemote
}
return result[i].Name < result[j].Name
})
return result
}

View File

@@ -0,0 +1,91 @@
package dash
import (
"reflect"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestCollectTierStats(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{{
Id: "rack1",
DataNodeInfos: []*master_pb.DataNodeInfo{
{
Id: "node1:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
Type: "",
MaxVolumeCount: 10,
DiskTotalBytes: 1_000_000,
DiskFreeBytes: 400_000,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Size: 1000},
{Id: 2, Size: 5000, RemoteStorageName: "s3.backup", RemoteStorageKey: "/2.dat"},
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 7, EcIndexBits: 0b111, ShardSizes: []int64{10, 20, 30}},
},
},
},
},
{
Id: "node2:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"ssd": {
Type: "ssd",
MaxVolumeCount: 5,
// No statfs numbers: capacity falls back to
// slots and usage to logical bytes.
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 3, Size: 2000, DiskType: "ssd"},
{Id: 4, Size: 7000, DiskType: "ssd", RemoteStorageName: "s3.backup", RemoteStorageKey: "/4.dat"},
{Id: 5, Size: 100, DiskType: "ssd", RemoteStorageName: "gcs.archive", RemoteStorageKey: "/5.dat"},
},
},
},
},
{
// An old server without statfs numbers sharing the
// hdd tier with node1: its logical bytes must still
// count toward the tier's DiskUsed.
Id: "node3:8080",
DiskInfos: map[string]*master_pb.DiskInfo{
"": {
Type: "",
MaxVolumeCount: 2,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 6, Size: 800},
},
},
},
},
},
}},
}},
}
got := CollectTierStats(topo, 30)
want := []TierStats{
{Name: "hdd", VolumeCount: 2, EcShardCount: 3, DataSize: 1860, DiskUsed: 600_800, DiskCapacity: 1_000_000 + 2*30*1024*1024, MaxVolumes: 12},
{Name: "ssd", VolumeCount: 1, DataSize: 2000, DiskUsed: 2000, DiskCapacity: 5 * 30 * 1024 * 1024, MaxVolumes: 5},
{Name: "gcs.archive", IsRemote: true, VolumeCount: 1, DataSize: 100},
{Name: "s3.backup", IsRemote: true, VolumeCount: 2, DataSize: 12000},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("CollectTierStats mismatch:\n got: %+v\nwant: %+v", got, want)
}
}
func TestCollectTierStatsEmpty(t *testing.T) {
if got := CollectTierStats(nil, 30); got != nil {
t.Errorf("expected nil for nil topology, got %+v", got)
}
if got := CollectTierStats(&master_pb.TopologyInfo{}, 30); len(got) != 0 {
t.Errorf("expected no tiers for empty topology, got %+v", got)
}
}

View File

@@ -0,0 +1,295 @@
package dash
import (
"context"
"fmt"
"io"
"path"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
)
// TopicRetentionPurger handles topic data purging based on retention policies
type TopicRetentionPurger struct {
adminServer *AdminServer
}
// NewTopicRetentionPurger creates a new topic retention purger
func NewTopicRetentionPurger(adminServer *AdminServer) *TopicRetentionPurger {
return &TopicRetentionPurger{
adminServer: adminServer,
}
}
// PurgeExpiredTopicData purges expired topic data based on retention policies
func (p *TopicRetentionPurger) PurgeExpiredTopicData() error {
glog.V(1).Infof("Starting topic data purge based on retention policies")
// Get all topics with retention enabled
topics, err := p.getTopicsWithRetention()
if err != nil {
return fmt.Errorf("failed to get topics with retention: %w", err)
}
glog.V(1).Infof("Found %d topics with retention enabled", len(topics))
// Process each topic
for _, topicRetention := range topics {
err := p.purgeTopicData(topicRetention)
if err != nil {
glog.Errorf("Failed to purge data for topic %s: %v", topicRetention.TopicName, err)
continue
}
}
glog.V(1).Infof("Completed topic data purge")
return nil
}
// TopicRetentionConfig represents a topic with its retention configuration
type TopicRetentionConfig struct {
TopicName string
Namespace string
Name string
RetentionSeconds int64
}
// getTopicsWithRetention retrieves all topics that have retention enabled
func (p *TopicRetentionPurger) getTopicsWithRetention() ([]TopicRetentionConfig, error) {
var topicsWithRetention []TopicRetentionConfig
// Find broker leader to get topics
brokerLeader, err := p.adminServer.findBrokerLeader()
if err != nil {
return nil, fmt.Errorf("failed to find broker leader: %w", err)
}
// Get all topics from the broker
err = p.adminServer.withBrokerClient(brokerLeader, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.ListTopics(ctx, &mq_pb.ListTopicsRequest{})
if err != nil {
return err
}
// Check each topic for retention configuration
for _, pbTopic := range resp.Topics {
configResp, err := client.GetTopicConfiguration(ctx, &mq_pb.GetTopicConfigurationRequest{
Topic: pbTopic,
})
if err != nil {
glog.Warningf("Failed to get configuration for topic %s.%s: %v", pbTopic.Namespace, pbTopic.Name, err)
continue
}
// Check if retention is enabled
if configResp.Retention != nil && configResp.Retention.Enabled && configResp.Retention.RetentionSeconds > 0 {
topicRetention := TopicRetentionConfig{
TopicName: fmt.Sprintf("%s.%s", pbTopic.Namespace, pbTopic.Name),
Namespace: pbTopic.Namespace,
Name: pbTopic.Name,
RetentionSeconds: configResp.Retention.RetentionSeconds,
}
topicsWithRetention = append(topicsWithRetention, topicRetention)
}
}
return nil
})
if err != nil {
return nil, err
}
return topicsWithRetention, nil
}
// purgeTopicData purges expired data for a specific topic
func (p *TopicRetentionPurger) purgeTopicData(topicRetention TopicRetentionConfig) error {
glog.V(1).Infof("Purging expired data for topic %s with retention %d seconds", topicRetention.TopicName, topicRetention.RetentionSeconds)
// Calculate cutoff time
cutoffTime := time.Now().Add(-time.Duration(topicRetention.RetentionSeconds) * time.Second)
// Get topic directory
topicObj := topic.NewTopic(topicRetention.Namespace, topicRetention.Name)
topicDir := topicObj.Dir()
var purgedDirs []string
err := p.adminServer.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// List all version directories under the topic directory
versionStream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: topicDir,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return fmt.Errorf("failed to list topic directory %s: %v", topicDir, err)
}
var versionDirs []VersionDirInfo
// Collect all version directories
for {
versionResp, err := versionStream.Recv()
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to receive version entries: %w", err)
}
// Only process directories that are versions (start with "v")
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
versionTime, err := p.parseVersionTime(versionResp.Entry.Name)
if err != nil {
glog.Warningf("Failed to parse version time from %s: %v", versionResp.Entry.Name, err)
continue
}
var modTime time.Time
if versionResp.Entry.Attributes != nil {
modTime = time.Unix(versionResp.Entry.Attributes.Mtime, 0)
}
versionDirs = append(versionDirs, VersionDirInfo{
Name: versionResp.Entry.Name,
VersionTime: versionTime,
ModTime: modTime,
})
}
}
// Sort version directories by time (oldest first)
sort.Slice(versionDirs, func(i, j int) bool {
return versionDirs[i].VersionTime.Before(versionDirs[j].VersionTime)
})
// Keep at least the most recent version directory, even if it's expired
if len(versionDirs) <= 1 {
glog.V(1).Infof("Topic %s has %d version directories, keeping all", topicRetention.TopicName, len(versionDirs))
return nil
}
// Purge expired directories (keep the most recent one)
for i := 0; i < len(versionDirs)-1; i++ {
versionDir := versionDirs[i]
// Check if this version directory is expired
if versionDir.VersionTime.Before(cutoffTime) {
dirPath := path.Join(topicDir, versionDir.Name)
// Delete the entire version directory
err := p.deleteDirectoryRecursively(client, dirPath)
if err != nil {
glog.Errorf("Failed to delete expired directory %s: %v", dirPath, err)
} else {
purgedDirs = append(purgedDirs, dirPath)
glog.V(1).Infof("Purged expired directory: %s (created: %s)", dirPath, versionDir.VersionTime.Format("2006-01-02 15:04:05"))
}
}
}
return nil
})
if err != nil {
return err
}
if len(purgedDirs) > 0 {
glog.V(0).Infof("Purged %d expired directories for topic %s", len(purgedDirs), topicRetention.TopicName)
}
return nil
}
// VersionDirInfo represents a version directory with its timestamp
type VersionDirInfo struct {
Name string
VersionTime time.Time
ModTime time.Time
}
// parseVersionTime parses the version directory name to extract the timestamp
// Version format: v2025-01-10-05-44-34
func (p *TopicRetentionPurger) parseVersionTime(versionName string) (time.Time, error) {
// Remove the 'v' prefix
if !strings.HasPrefix(versionName, "v") {
return time.Time{}, fmt.Errorf("invalid version format: %s", versionName)
}
timeStr := versionName[1:] // Remove 'v'
// Parse the time format: 2025-01-10-05-44-34
versionTime, err := time.Parse("2006-01-02-15-04-05", timeStr)
if err != nil {
return time.Time{}, fmt.Errorf("failed to parse version time %s: %v", timeStr, err)
}
return versionTime, nil
}
// deleteDirectoryRecursively deletes a directory and all its contents
func (p *TopicRetentionPurger) deleteDirectoryRecursively(client filer_pb.SeaweedFilerClient, dirPath string) error {
// List all entries in the directory
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: dirPath,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return fmt.Errorf("failed to list directory %s: %v", dirPath, err)
}
// Delete all entries
for {
resp, err := stream.Recv()
if err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to receive entries: %w", err)
}
if resp.Entry == nil {
continue
}
entryPath := path.Join(dirPath, resp.Entry.Name)
if resp.Entry.IsDirectory {
// Recursively delete subdirectory
err = p.deleteDirectoryRecursively(client, entryPath)
if err != nil {
return fmt.Errorf("failed to delete subdirectory %s: %v", entryPath, err)
}
} else {
// Delete file
if err := filer_pb.DoRemove(context.Background(), client, dirPath, resp.Entry.Name, false, false, false, false, nil); err != nil {
return fmt.Errorf("failed to delete file %s: %v", entryPath, err)
}
}
}
// Delete the directory itself
parentDir := path.Dir(dirPath)
dirName := path.Base(dirPath)
if err := filer_pb.DoRemove(context.Background(), client, parentDir, dirName, false, false, false, false, nil); err != nil {
return fmt.Errorf("failed to delete directory %s: %v", dirPath, err)
}
return nil
}

791
weed/admin/dash/types.go Normal file
View File

@@ -0,0 +1,791 @@
package dash
import (
"encoding/json"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
// Core cluster topology structures
type ClusterTopology struct {
Masters []MasterNode `json:"masters"`
DataCenters []DataCenter `json:"datacenters"`
VolumeServers []VolumeServer `json:"volume_servers"`
TotalVolumes int `json:"total_volumes"`
// TotalChunks counts chunks stored in volumes, not filer entries: a file
// is split into one or more chunks.
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
UpdatedAt time.Time `json:"updated_at"`
// TierStats breaks volumes and EC shards down by storage tier: local
// disk types plus one entry per remote storage holding tiered volumes.
TierStats []TierStats `json:"tier_stats"`
}
type MasterNode struct {
Address string `json:"address"`
IsLeader bool `json:"is_leader"`
}
type DataCenter struct {
ID string `json:"id"`
Racks []Rack `json:"racks"`
}
type Rack struct {
ID string `json:"id"`
Nodes []VolumeServer `json:"nodes"`
}
type VolumeServer struct {
ID string `json:"id"`
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
PublicURL string `json:"public_url"`
Volumes int `json:"volumes"`
MaxVolumes int `json:"max_volumes"`
DiskUsage int64 `json:"disk_usage"`
DiskCapacity int64 `json:"disk_capacity"`
LastHeartbeat time.Time `json:"last_heartbeat"`
// EC shard information
EcVolumes int `json:"ec_volumes"` // Number of EC volumes this server has shards for
EcShards int `json:"ec_shards"` // Total number of EC shards on this server
EcShardDetails []VolumeServerEcInfo `json:"ec_shard_details"` // Detailed EC shard information
// RemoteSize is the bytes this server's remote-tiered volumes hold in
// cloud storage. Those bytes are excluded from DiskUsage, which only
// counts what occupies local disks.
RemoteSize int64 `json:"remote_size"`
}
func (vs *VolumeServer) GetDisplayAddress() string {
if vs.PublicURL != "" {
return vs.PublicURL
}
return vs.Address
}
// VolumeServerEcInfo represents EC shard information for a specific volume on a server
type VolumeServerEcInfo struct {
VolumeID uint32 `json:"volume_id"`
Collection string `json:"collection"`
ShardCount int `json:"shard_count"` // Number of shards this server has for this volume
EcIndexBits uint32 `json:"ec_index_bits"` // Bitmap of which shards this server has
ShardNumbers []int `json:"shard_numbers"` // List of actual shard numbers this server has
ShardSizes map[int]int64 `json:"shard_sizes"` // Map from shard number to size in bytes
TotalSize int64 `json:"total_size"` // Total size of all shards on this server for this volume
}
// S3 Bucket management structures
type S3Bucket struct {
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
LogicalSize int64 `json:"logical_size"` // Actual data size (used space)
PhysicalSize int64 `json:"physical_size"` // Total allocated volume space
LastModified time.Time `json:"last_modified"`
Quota int64 `json:"quota"` // Quota in bytes, 0 means no quota
QuotaEnabled bool `json:"quota_enabled"` // Whether quota is enabled
ReadOnly bool `json:"read_only"` // Whether the bucket path is read-only in filer.conf (e.g. quota enforcement)
VersioningStatus string `json:"versioning_status"` // Versioning status: "" (never enabled), "Enabled", or "Suspended"
ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled
ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days
Owner string `json:"owner,omitempty"` // Bucket owner identity; empty means admin-only access
LifecycleRuleCount int `json:"lifecycle_rule_count"`
LifecycleEnabledCount int `json:"lifecycle_enabled_count"`
// PolicyStatementCount is the number of statements in the bucket policy,
// or 0 if the bucket has none. A policy document can't have zero
// statements (see policy_engine.ValidatePolicy), so >0 is a faithful
// "has a policy" flag.
PolicyStatementCount int `json:"policy_statement_count"`
}
type S3Object struct {
Key string `json:"key"`
Size int64 `json:"size"`
LastModified time.Time `json:"last_modified"`
ETag string `json:"etag"`
StorageClass string `json:"storage_class"`
}
type BucketDetails struct {
Bucket S3Bucket `json:"bucket"`
UpdatedAt time.Time `json:"updated_at"`
}
type BucketLifecycleRule struct {
ID string `json:"id,omitempty"`
Status string `json:"status"`
Prefix string `json:"prefix,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
SizeGreaterThan int64 `json:"size_greater_than,omitempty"`
SizeLessThan int64 `json:"size_less_than,omitempty"`
ExpirationDays int `json:"expiration_days,omitempty"`
ExpirationDate string `json:"expiration_date,omitempty"`
ExpiredObjectDeleteMarker bool `json:"expired_object_delete_marker,omitempty"`
NoncurrentVersionExpirationDays int `json:"noncurrent_version_expiration_days,omitempty"`
NewerNoncurrentVersions int `json:"newer_noncurrent_versions,omitempty"`
AbortMultipartDays int `json:"abort_multipart_days,omitempty"`
}
type BucketLifecycle struct {
Bucket string `json:"bucket"`
Rules []BucketLifecycleRule `json:"rules"`
XML string `json:"xml,omitempty"`
}
// ObjectStoreUser is defined in admin_data.go
// Volume management structures
type VolumeWithTopology struct {
*master_pb.VolumeInformationMessage
Server string `json:"server"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
}
type ClusterVolumesData struct {
Username string `json:"username"`
Volumes []VolumeWithTopology `json:"volumes"`
TotalVolumes int `json:"total_volumes"`
TotalSize int64 `json:"total_size"`
VolumeSizeLimit uint64 `json:"volume_size_limit"`
LastUpdated time.Time `json:"last_updated"`
// Pagination
CurrentPage int `json:"current_page"`
TotalPages int `json:"total_pages"`
PageSize int `json:"page_size"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
// Statistics
DataCenterCount int `json:"datacenter_count"`
RackCount int `json:"rack_count"`
DiskTypeCount int `json:"disk_type_count"`
CollectionCount int `json:"collection_count"`
VersionCount int `json:"version_count"`
// Conditional display flags
ShowDataCenterColumn bool `json:"show_datacenter_column"`
ShowRackColumn bool `json:"show_rack_column"`
ShowDiskTypeColumn bool `json:"show_disk_type_column"`
ShowCollectionColumn bool `json:"show_collection_column"`
ShowVersionColumn bool `json:"show_version_column"`
// Single values when only one exists
SingleDataCenter string `json:"single_datacenter"`
SingleRack string `json:"single_rack"`
SingleDiskType string `json:"single_disk_type"`
SingleCollection string `json:"single_collection"`
SingleVersion string `json:"single_version"`
// All versions when multiple exist
AllVersions []string `json:"all_versions"`
// All disk types when multiple exist
AllDiskTypes []string `json:"all_disk_types"`
// Filtering
FilterCollection string `json:"filter_collection"`
}
// ClusterEcShardsData represents the data for the cluster EC shards page
type ClusterEcShardsData struct {
Username string `json:"username"`
EcShards []EcShardWithInfo `json:"ec_shards"`
TotalShards int `json:"total_shards"`
TotalVolumes int `json:"total_volumes"`
LastUpdated time.Time `json:"last_updated"`
// Pagination
CurrentPage int `json:"current_page"`
TotalPages int `json:"total_pages"`
PageSize int `json:"page_size"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
// Statistics
DataCenterCount int `json:"datacenter_count"`
RackCount int `json:"rack_count"`
CollectionCount int `json:"collection_count"`
// Conditional display flags
ShowDataCenterColumn bool `json:"show_datacenter_column"`
ShowRackColumn bool `json:"show_rack_column"`
ShowCollectionColumn bool `json:"show_collection_column"`
// Single values when only one exists
SingleDataCenter string `json:"single_datacenter"`
SingleRack string `json:"single_rack"`
SingleCollection string `json:"single_collection"`
// Filtering
FilterCollection string `json:"filter_collection"`
// EC specific statistics
ShardsPerVolume map[uint32]int `json:"shards_per_volume"` // VolumeID -> shard count
VolumesWithAllShards int `json:"volumes_with_all_shards"` // Volumes with all 14 shards
VolumesWithMissingShards int `json:"volumes_with_missing_shards"` // Volumes missing shards
}
// EcShardWithInfo represents an EC shard with its topology information
type EcShardWithInfo struct {
VolumeID uint32 `json:"volume_id"`
ShardID uint32 `json:"shard_id"`
Collection string `json:"collection"`
Size uint64 `json:"size"`
Server string `json:"server"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
DiskType string `json:"disk_type"`
ModifiedTime int64 `json:"modified_time"`
// EC specific fields
EcIndexBits uint32 `json:"ec_index_bits"` // Bitmap of which shards this server has
ShardCount int `json:"shard_count"` // Number of shards this server has for this volume
IsComplete bool `json:"is_complete"` // True if this volume has all 14 shards
MissingShards []int `json:"missing_shards"` // List of missing shard IDs
}
// EcVolumeDetailsData represents the data for the EC volume details page
type EcVolumeDetailsData struct {
Username string `json:"username"`
VolumeID uint32 `json:"volume_id"`
Collection string `json:"collection"`
Shards []EcShardWithInfo `json:"shards"`
TotalShards int `json:"total_shards"`
IsComplete bool `json:"is_complete"`
MissingShards []int `json:"missing_shards"`
DataCenters []string `json:"datacenters"`
Servers []string `json:"servers"`
LastUpdated time.Time `json:"last_updated"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
}
type VolumeDetailsData struct {
Volume VolumeWithTopology `json:"volume"`
Replicas []VolumeWithTopology `json:"replicas"`
VolumeSizeLimit uint64 `json:"volume_size_limit"`
ReplicationCount int `json:"replication_count"`
LastUpdated time.Time `json:"last_updated"`
}
// Collection management structures
type CollectionInfo struct {
Name string `json:"name"`
DataCenter string `json:"datacenter"`
VolumeCount int `json:"volume_count"`
EcVolumeCount int `json:"ec_volume_count"`
ChunkCount int64 `json:"chunk_count"`
TotalSize int64 `json:"total_size"`
DiskTypes []string `json:"disk_types"`
}
type ClusterCollectionsData struct {
Username string `json:"username"`
Collections []CollectionInfo `json:"collections"`
TotalCollections int `json:"total_collections"`
TotalVolumes int `json:"total_volumes"`
TotalEcVolumes int `json:"total_ec_volumes"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
LastUpdated time.Time `json:"last_updated"`
}
// Master and Filer management structures
type MasterInfo struct {
Address string `json:"address"`
IsLeader bool `json:"is_leader"`
Suffrage string `json:"suffrage"`
}
type ClusterMastersData struct {
Username string `json:"username"`
Masters []MasterInfo `json:"masters"`
TotalMasters int `json:"total_masters"`
LeaderCount int `json:"leader_count"`
LastUpdated time.Time `json:"last_updated"`
}
type FilerInfo struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
}
type ClusterFilersData struct {
Username string `json:"username"`
Filers []FilerInfo `json:"filers"`
TotalFilers int `json:"total_filers"`
LastUpdated time.Time `json:"last_updated"`
}
// MountClient is one connected FUSE/VFS mount as reported by a filer's
// metadata-subscriber registry.
type MountClient struct {
ClientName string `json:"client_name"`
ClientType string `json:"client_type"` // "mount" (Go) or "sw-vfs" (Rust VFS)
Address string `json:"address"`
PathPrefix string `json:"path_prefix"`
ClientId int32 `json:"client_id"`
ConnectedAt time.Time `json:"connected_at"`
FilerAddress string `json:"filer_address"`
}
type MountClientsData struct {
Username string `json:"username"`
MountClients []MountClient `json:"mount_clients"`
TotalMountClients int `json:"total_mount_clients"`
LastUpdated time.Time `json:"last_updated"`
}
type MessageBrokerInfo struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Rack string `json:"rack"`
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
}
type ClusterBrokersData struct {
Username string `json:"username"`
Brokers []MessageBrokerInfo `json:"brokers"`
TotalBrokers int `json:"total_brokers"`
LastUpdated time.Time `json:"last_updated"`
}
type S3ServerInfo struct {
Address string `json:"address"`
DataCenter string `json:"datacenter"`
Version string `json:"version"`
CreatedAt time.Time `json:"created_at"`
}
type ClusterS3ServersData struct {
Username string `json:"username"`
S3Servers []S3ServerInfo `json:"s3_servers"`
TotalS3Servers int `json:"total_s3_servers"`
LastUpdated time.Time `json:"last_updated"`
}
type TopicInfo struct {
Name string `json:"name"`
Partitions int `json:"partitions"`
Subscribers int `json:"subscribers"`
MessageCount int64 `json:"message_count"`
TotalSize int64 `json:"total_size"`
LastMessage time.Time `json:"last_message"`
CreatedAt time.Time `json:"created_at"`
Retention TopicRetentionInfo `json:"retention"`
}
type TopicsData struct {
Username string `json:"username"`
Topics []TopicInfo `json:"topics"`
TotalTopics int `json:"total_topics"`
TotalMessages int64 `json:"total_messages"`
TotalSize int64 `json:"total_size"`
LastUpdated time.Time `json:"last_updated"`
}
type SubscriberInfo struct {
Name string `json:"name"`
Topic string `json:"topic"`
ConsumerGroup string `json:"consumer_group"`
Status string `json:"status"`
LastSeen time.Time `json:"last_seen"`
MessageCount int64 `json:"message_count"`
CreatedAt time.Time `json:"created_at"`
}
type SubscribersData struct {
Username string `json:"username"`
Subscribers []SubscriberInfo `json:"subscribers"`
TotalSubscribers int `json:"total_subscribers"`
ActiveSubscribers int `json:"active_subscribers"`
LastUpdated time.Time `json:"last_updated"`
}
// Topic Details structures
type PartitionInfo struct {
ID int32 `json:"id"`
LeaderBroker string `json:"leader_broker"`
FollowerBroker string `json:"follower_broker"`
MessageCount int64 `json:"message_count"`
TotalSize int64 `json:"total_size"`
LastDataTime time.Time `json:"last_data_time"`
CreatedAt time.Time `json:"created_at"`
}
type SchemaFieldInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Required bool `json:"required"`
}
type PublisherInfo struct {
PublisherName string `json:"publisher_name"`
ClientID string `json:"client_id"`
PartitionID int32 `json:"partition_id"`
Broker string `json:"broker"`
ConnectTime time.Time `json:"connect_time"`
LastSeenTime time.Time `json:"last_seen_time"`
IsActive bool `json:"is_active"`
LastPublishedOffset int64 `json:"last_published_offset"`
LastAckedOffset int64 `json:"last_acked_offset"`
}
type TopicSubscriberInfo struct {
ConsumerGroup string `json:"consumer_group"`
ConsumerID string `json:"consumer_id"`
ClientID string `json:"client_id"`
PartitionID int32 `json:"partition_id"`
Broker string `json:"broker"`
ConnectTime time.Time `json:"connect_time"`
LastSeenTime time.Time `json:"last_seen_time"`
IsActive bool `json:"is_active"`
CurrentOffset int64 `json:"current_offset"` // last acknowledged offset
LastReceivedOffset int64 `json:"last_received_offset"` // last received offset
}
type ConsumerGroupOffsetInfo struct {
ConsumerGroup string `json:"consumer_group"`
PartitionID int32 `json:"partition_id"`
Offset int64 `json:"offset"`
LastUpdated time.Time `json:"last_updated"`
}
type TopicRetentionInfo struct {
Enabled bool `json:"enabled"`
RetentionSeconds int64 `json:"retention_seconds"`
DisplayValue int32 `json:"display_value"` // for UI rendering
DisplayUnit string `json:"display_unit"` // for UI rendering
}
type TopicDetailsData struct {
Username string `json:"username"`
TopicName string `json:"topic_name"`
Namespace string `json:"namespace"`
Name string `json:"name"`
Partitions []PartitionInfo `json:"partitions"`
KeySchema []SchemaFieldInfo `json:"key_schema"` // Schema fields for keys
ValueSchema []SchemaFieldInfo `json:"value_schema"` // Schema fields for values
Publishers []PublisherInfo `json:"publishers"`
Subscribers []TopicSubscriberInfo `json:"subscribers"`
ConsumerGroupOffsets []ConsumerGroupOffsetInfo `json:"consumer_group_offsets"`
Retention TopicRetentionInfo `json:"retention"`
MessageCount int64 `json:"message_count"`
TotalSize int64 `json:"total_size"`
CreatedAt time.Time `json:"created_at"`
LastUpdated time.Time `json:"last_updated"`
}
// Volume server management structures
type ClusterVolumeServersData struct {
Username string `json:"username"`
VolumeServers []VolumeServer `json:"volume_servers"`
TotalVolumeServers int `json:"total_volume_servers"`
TotalVolumes int `json:"total_volumes"`
TotalCapacity int64 `json:"total_capacity"`
LastUpdated time.Time `json:"last_updated"`
}
// Type aliases for maintenance package types to support existing code
type MaintenanceTask = maintenance.MaintenanceTask
type MaintenanceTaskType = maintenance.MaintenanceTaskType
type MaintenanceTaskStatus = maintenance.MaintenanceTaskStatus
type MaintenanceTaskPriority = maintenance.MaintenanceTaskPriority
type MaintenanceWorker = maintenance.MaintenanceWorker
type MaintenanceConfig = maintenance.MaintenanceConfig
type MaintenanceStats = maintenance.MaintenanceStats
type MaintenanceConfigData = maintenance.MaintenanceConfigData
type MaintenanceQueueData = maintenance.MaintenanceQueueData
type QueueStats = maintenance.QueueStats
type WorkerDetailsData = maintenance.WorkerDetailsData
type WorkerPerformance = maintenance.WorkerPerformance
// Status constants (these are still static)
const (
TaskStatusPending = maintenance.TaskStatusPending
TaskStatusAssigned = maintenance.TaskStatusAssigned
TaskStatusInProgress = maintenance.TaskStatusInProgress
TaskStatusCompleted = maintenance.TaskStatusCompleted
TaskStatusFailed = maintenance.TaskStatusFailed
TaskStatusCancelled = maintenance.TaskStatusCancelled
PriorityLow = maintenance.PriorityLow
PriorityNormal = maintenance.PriorityNormal
PriorityHigh = maintenance.PriorityHigh
PriorityCritical = maintenance.PriorityCritical
)
// Helper functions from maintenance package
var DefaultMaintenanceConfig = maintenance.DefaultMaintenanceConfig
// MaintenanceWorkersData represents the data for the maintenance workers page
type MaintenanceWorkersData struct {
Workers []*WorkerDetailsData `json:"workers"`
ActiveWorkers int `json:"active_workers"`
BusyWorkers int `json:"busy_workers"`
TotalLoad int `json:"total_load"`
LastUpdated time.Time `json:"last_updated"`
}
// Maintenance system types are now in weed/admin/maintenance package
// EcVolumeWithShards represents an EC volume with its shard distribution
type EcVolumeWithShards struct {
VolumeID uint32 `json:"volume_id"`
Collection string `json:"collection"`
TotalShards int `json:"total_shards"`
IsComplete bool `json:"is_complete"`
MissingShards []int `json:"missing_shards"`
ShardLocations map[int]string `json:"shard_locations"` // shardId -> server
ShardSizes map[int]int64 `json:"shard_sizes"` // shardId -> size in bytes
DataCenters []string `json:"data_centers"`
Servers []string `json:"servers"`
Racks []string `json:"racks"`
ModifiedTime int64 `json:"modified_time"`
}
// ClusterEcVolumesData represents the response for clustered EC volumes view
type ClusterEcVolumesData struct {
EcVolumes []EcVolumeWithShards `json:"ec_volumes"`
TotalVolumes int `json:"total_volumes"`
LastUpdated time.Time `json:"last_updated"`
// Pagination
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalPages int `json:"total_pages"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
// Filtering
Collection string `json:"collection"`
// Conditional display flags
ShowDataCenterColumn bool `json:"show_datacenter_column"`
ShowRackColumn bool `json:"show_rack_column"`
ShowCollectionColumn bool `json:"show_collection_column"`
// Statistics
CompleteVolumes int `json:"complete_volumes"`
IncompleteVolumes int `json:"incomplete_volumes"`
TotalShards int `json:"total_shards"`
// User context
Username string `json:"username"`
}
// Collection detail page structures
type CollectionDetailsData struct {
Username string `json:"username"`
CollectionName string `json:"collection_name"`
RegularVolumes []VolumeWithTopology `json:"regular_volumes"`
EcVolumes []EcVolumeWithShards `json:"ec_volumes"`
TotalVolumes int `json:"total_volumes"`
TotalEcVolumes int `json:"total_ec_volumes"`
TotalChunks int64 `json:"total_chunks"`
TotalSize int64 `json:"total_size"`
DataCenters []string `json:"data_centers"`
DiskTypes []string `json:"disk_types"`
LastUpdated time.Time `json:"last_updated"`
// Pagination
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalPages int `json:"total_pages"`
// Sorting
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
}
// Service Account management structures
type ServiceAccount struct {
ID string `json:"id"`
ParentUser string `json:"parent_user"`
Description string `json:"description,omitempty"`
AccessKeyId string `json:"access_key_id,omitempty"`
SecretAccessKey string `json:"secret_access_key,omitempty"` // Only returned on creation
Status string `json:"status"`
CreateDate time.Time `json:"create_date"`
Expiration time.Time `json:"expiration,omitempty"`
// ServiceAccountId is used when returning a single ID in some API responses
ServiceAccountId string `json:"service_account_id,omitempty"`
// ServiceAccountIds is used when returning a list of IDs owned by a user
ServiceAccountIds []string `json:"service_account_ids,omitempty"`
}
type ServiceAccountsData struct {
Username string `json:"username"`
ServiceAccounts []ServiceAccount `json:"service_accounts"`
TotalAccounts int `json:"total_accounts"`
ActiveAccounts int `json:"active_accounts"`
AvailableUsers []string `json:"available_users"` // For parent user dropdown
LastUpdated time.Time `json:"last_updated"`
}
type CreateServiceAccountRequest struct {
ParentUser string `json:"parent_user"`
Description string `json:"description,omitempty"`
Expiration string `json:"expiration,omitempty"` // RFC3339 format
}
type UpdateServiceAccountRequest struct {
Status string `json:"status,omitempty"` // Active, Inactive
Description string `json:"description,omitempty"`
Expiration string `json:"expiration,omitempty"`
}
// Group management structures
type GroupData struct {
Name string `json:"name"`
MemberCount int `json:"member_count"`
PolicyCount int `json:"policy_count"`
Status string `json:"status"` // "enabled" or "disabled"
Members []string `json:"members"`
PolicyNames []string `json:"policy_names"`
}
type GroupsPageData struct {
Username string `json:"username"`
Groups []GroupData `json:"groups"`
TotalGroups int `json:"total_groups"`
ActiveGroups int `json:"active_groups"`
AvailableUsers []string `json:"available_users"`
AvailablePolicies []string `json:"available_policies"`
LastUpdated time.Time `json:"last_updated"`
}
type CreateGroupRequest struct {
Name string `json:"name"`
}
// STS Configuration display types
type STSConfigData struct {
Enabled bool `json:"enabled"`
Issuer string `json:"issuer,omitempty"`
TokenDuration string `json:"token_duration,omitempty"`
Providers []string `json:"providers,omitempty"`
LastUpdated time.Time `json:"last_updated"`
}
// Iceberg Catalog types
type IcebergCatalogInfo struct {
Name string `json:"name"`
ARN string `json:"arn"`
OwnerAccountID string `json:"owner_account_id"`
CreatedAt time.Time `json:"created_at"`
}
type IcebergCatalogData struct {
Username string `json:"username"`
Catalogs []IcebergCatalogInfo `json:"catalogs"`
TotalCatalogs int `json:"total_catalogs"`
IcebergPort int `json:"iceberg_port"`
LastUpdated time.Time `json:"last_updated"`
}
type IcebergNamespaceInfo struct {
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
}
type IcebergNamespacesData struct {
Username string `json:"username"`
CatalogName string `json:"catalog_name"`
BucketARN string `json:"bucket_arn"`
Namespaces []IcebergNamespaceInfo `json:"namespaces"`
TotalNamespaces int `json:"total_namespaces"`
LastUpdated time.Time `json:"last_updated"`
}
type IcebergTableInfo struct {
Name string `json:"name"`
CreatedAt time.Time `json:"created_at"`
}
type IcebergTablesData struct {
Username string `json:"username"`
CatalogName string `json:"catalog_name"`
NamespaceName string `json:"namespace_name"`
BucketARN string `json:"bucket_arn"`
Tables []IcebergTableInfo `json:"tables"`
TotalTables int `json:"total_tables"`
LastUpdated time.Time `json:"last_updated"`
}
type IcebergSchemaFieldInfo struct {
ID int `json:"id"`
Name string `json:"name"`
Type json.RawMessage `json:"type"`
Required bool `json:"required"`
}
type IcebergPartitionFieldInfo struct {
Name string `json:"name"`
Transform string `json:"transform"`
SourceID int `json:"source_id"`
FieldID int `json:"field_id"`
}
type IcebergPropertyInfo struct {
Key string `json:"key"`
Value string `json:"value"`
}
type IcebergSnapshotInfo struct {
SnapshotID int64 `json:"snapshot_id"`
Timestamp time.Time `json:"timestamp"`
Operation string `json:"operation"`
ManifestList string `json:"manifest_list"`
}
type IcebergTableDetailsData struct {
Username string `json:"username"`
CatalogName string `json:"catalog_name"`
NamespaceName string `json:"namespace_name"`
TableName string `json:"table_name"`
BucketARN string `json:"bucket_arn"`
TableARN string `json:"table_arn"`
Format string `json:"format"`
CreatedAt time.Time `json:"created_at"`
ModifiedAt time.Time `json:"modified_at"`
MetadataLocation string `json:"metadata_location"`
TableLocation string `json:"table_location"`
SchemaFields []IcebergSchemaFieldInfo `json:"schema_fields"`
PartitionFields []IcebergPartitionFieldInfo `json:"partition_fields"`
Properties []IcebergPropertyInfo `json:"properties"`
Snapshots []IcebergSnapshotInfo `json:"snapshots"`
SnapshotCount int `json:"snapshot_count"`
HasSnapshotCount bool `json:"has_snapshot_count"`
DataFileCount int64 `json:"data_file_count"`
HasDataFileCount bool `json:"has_data_file_count"`
TotalSizeBytes int64 `json:"total_size_bytes"`
HasTotalSize bool `json:"has_total_size"`
MetadataError string `json:"metadata_error,omitempty"`
// Set when the details came from a plugin worker rather than from metadata
// this server can read, so the page can say whose account it is and when.
ObservedBy string `json:"observed_by,omitempty"`
ObservedAt time.Time `json:"observed_at,omitempty"`
}

View File

@@ -0,0 +1,62 @@
package dash
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/credential"
_ "github.com/seaweedfs/seaweedfs/weed/credential/memory" // register the memory credential store
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
// TestGetObjectStoreUserDetailsIncludesInlinePolicies verifies that inline user
// policies — which are stored separately from the identity record and are
// authoritative at S3 enforcement time (they take precedence over the legacy
// Actions list) — are surfaced in the admin user details. They were previously
// invisible in the admin API/UI, so an operator could not tell what actually
// governed a user's access.
func TestGetObjectStoreUserDetailsIncludesInlinePolicies(t *testing.T) {
cm, err := credential.NewCredentialManagerWithDefaults(credential.StoreTypeMemory)
if err != nil {
t.Fatalf("failed to init credential manager: %v", err)
}
ctx := context.Background()
const username = "svc-app"
if err := cm.CreateUser(ctx, &iam_pb.Identity{Name: username}); err != nil {
t.Fatalf("failed to create user: %v", err)
}
const inlinePolicyName = "svc-app-policy"
doc := policy_engine.PolicyDocument{
Version: "2012-10-17",
Statement: []policy_engine.PolicyStatement{
{
Effect: "Allow",
Action: policy_engine.NewStringOrStringSlice("s3:GetObject"),
Resource: policy_engine.NewStringOrStringSlicePtr("arn:aws:s3:::ml-artifacts/*"),
},
},
}
if err := cm.PutUserInlinePolicy(ctx, username, inlinePolicyName, doc); err != nil {
t.Fatalf("failed to put inline policy: %v", err)
}
s := &AdminServer{credentialManager: cm}
details, err := s.GetObjectStoreUserDetails(username)
if err != nil {
t.Fatalf("GetObjectStoreUserDetails failed: %v", err)
}
found := false
for _, p := range details.PolicyNames {
if p == inlinePolicyName {
found = true
break
}
}
if !found {
t.Errorf("expected inline policy %q in details.PolicyNames, got %v", inlinePolicyName, details.PolicyNames)
}
}

View File

@@ -0,0 +1,479 @@
package dash
import (
"context"
"crypto/rand"
"errors"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iam"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
var (
ErrAccessKeyInUse = errors.New("access key already in use")
ErrUserNotFound = errors.New("user not found")
ErrInvalidInput = errors.New("invalid input")
)
// CreateObjectStoreUser creates a new user using the credential manager
func (s *AdminServer) CreateObjectStoreUser(req CreateUserRequest) (*ObjectStoreUser, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Create new identity
newIdentity := &iam_pb.Identity{
Name: req.Username,
Actions: req.Actions,
PolicyNames: req.PolicyNames,
}
// Add account if email is provided
if req.Email != "" {
newIdentity.Account = &iam_pb.Account{
Id: generateAccountId(),
DisplayName: req.Username,
EmailAddress: req.Email,
}
}
// Generate access key if requested
var accessKey, secretKey string
if req.GenerateKey {
accessKey = generateAccessKey()
secretKey = generateSecretKey()
newIdentity.Credentials = []*iam_pb.Credential{
{
AccessKey: accessKey,
SecretKey: secretKey,
},
}
}
// Create user using credential manager
err := s.credentialManager.CreateUser(ctx, newIdentity)
if err != nil {
if err == credential.ErrUserAlreadyExists {
return nil, fmt.Errorf("user %s already exists", req.Username)
}
return nil, fmt.Errorf("failed to create user: %w", err)
}
// Return created user
user := &ObjectStoreUser{
Username: req.Username,
Email: req.Email,
AccessKey: accessKey,
SecretKey: secretKey,
Permissions: req.Actions,
PolicyNames: req.PolicyNames,
}
return user, nil
}
// UpdateObjectStoreUser updates an existing user
func (s *AdminServer) UpdateObjectStoreUser(username string, req UpdateUserRequest) (*ObjectStoreUser, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Get existing user
identity, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return nil, fmt.Errorf("user %s not found", username)
}
return nil, fmt.Errorf("failed to get user: %w", err)
}
// Create updated identity
updatedIdentity := &iam_pb.Identity{
Name: identity.Name,
Account: identity.Account,
Credentials: identity.Credentials,
Actions: identity.Actions,
PolicyNames: identity.PolicyNames,
}
// Update actions if provided
if req.Actions != nil {
updatedIdentity.Actions = req.Actions
}
// Always update policy names when present in request (even if empty to allow clearing)
if req.PolicyNames != nil {
updatedIdentity.PolicyNames = req.PolicyNames
}
// Update email if provided
if req.Email != "" {
if updatedIdentity.Account == nil {
updatedIdentity.Account = &iam_pb.Account{
Id: generateAccountId(),
DisplayName: username,
}
}
updatedIdentity.Account.EmailAddress = req.Email
}
// Update user using credential manager
err = s.credentialManager.UpdateUser(ctx, username, updatedIdentity)
if err != nil {
return nil, fmt.Errorf("failed to update user: %w", err)
}
// Return updated user
user := &ObjectStoreUser{
Username: username,
Email: req.Email,
Permissions: updatedIdentity.Actions,
PolicyNames: updatedIdentity.PolicyNames,
}
// Get first access key for display
if len(updatedIdentity.Credentials) > 0 {
user.AccessKey = updatedIdentity.Credentials[0].AccessKey
user.SecretKey = updatedIdentity.Credentials[0].SecretKey
}
return user, nil
}
// DeleteObjectStoreUser deletes a user using the credential manager
func (s *AdminServer) DeleteObjectStoreUser(username string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Delete user using credential manager
err := s.credentialManager.DeleteUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return fmt.Errorf("user %s not found", username)
}
return fmt.Errorf("failed to delete user: %w", err)
}
return nil
}
// GetObjectStoreUserDetails returns detailed information about a user
func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Get user using credential manager (resolves static users via filer gRPC)
identity, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return nil, fmt.Errorf("user %s not found", username)
}
return nil, fmt.Errorf("failed to get user: %w", err)
}
details := &UserDetails{
Username: username,
Actions: identity.Actions,
PolicyNames: identity.PolicyNames,
}
// Inline user policies are stored separately from the identity record and
// are authoritative at enforcement time (they take precedence over the
// legacy Actions list). Surface their names alongside attached policies so
// the admin reflects the user's actual permissions rather than only the
// (potentially overridden) Actions.
if inlinePolicyNames, err := s.credentialManager.ListUserInlinePolicies(ctx, username); err != nil {
glog.Warningf("failed to list inline policies for user %s: %v", username, err)
} else if len(inlinePolicyNames) > 0 {
details.PolicyNames = append(details.PolicyNames, inlinePolicyNames...)
}
// Set email from account if available
if identity.Account != nil {
details.Email = identity.Account.EmailAddress
}
// Look up groups the user belongs to
groupNames, err := s.credentialManager.ListGroups(ctx)
if err != nil {
return nil, fmt.Errorf("failed to list groups: %w", err)
}
for _, gName := range groupNames {
g, err := s.credentialManager.GetGroup(ctx, gName)
if err != nil {
return nil, fmt.Errorf("failed to get group %s: %w", gName, err)
}
for _, member := range g.Members {
if member == username {
details.Groups = append(details.Groups, gName)
break
}
}
}
// Convert credentials to access key info
for _, cred := range identity.Credentials {
details.AccessKeys = append(details.AccessKeys, AccessKeyInfo{
AccessKey: cred.AccessKey,
SecretKey: cred.SecretKey,
Status: cred.Status,
CreatedAt: time.Now().AddDate(0, -1, 0), // Mock creation date
})
}
return details, nil
}
// CreateAccessKey creates a new access key for a user
func (s *AdminServer) CreateAccessKey(username string, req *CreateAccessKeyRequest) (*AccessKeyInfo, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Check if user exists
_, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return nil, fmt.Errorf("user %s: %w", username, ErrUserNotFound)
}
return nil, fmt.Errorf("failed to get user: %w", err)
}
if req == nil {
req = &CreateAccessKeyRequest{}
}
// Validate provided keys (shared with the IAM API and embedded IAM paths).
if req.AccessKey != "" {
if err := iam.ValidateCallerSuppliedAccessKeyId(req.AccessKey); err != nil {
return nil, fmt.Errorf("%s: %w", err.Error(), ErrInvalidInput)
}
}
if req.SecretKey != "" {
if err := iam.ValidateCallerSuppliedSecretAccessKey(req.SecretKey); err != nil {
return nil, fmt.Errorf("%s: %w", err.Error(), ErrInvalidInput)
}
}
// Enforce the both-or-none rule to match the IAM API and embedded IAM
// paths — silently generating the missing half lets a caller end up
// with a credential they did not fully choose.
if (req.AccessKey != "") != (req.SecretKey != "") {
return nil, fmt.Errorf("access key and secret key must be supplied together: %w", ErrInvalidInput)
}
// Use provided keys or generate new ones
accessKey := req.AccessKey
secretKey := req.SecretKey
if accessKey == "" {
accessKey = generateAccessKey()
secretKey = generateSecretKey()
}
// Verify access key is globally unique
existingUser, err := s.credentialManager.GetUserByAccessKey(ctx, accessKey)
if existingUser != nil {
return nil, ErrAccessKeyInUse
}
if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !isNotFoundError(err) {
return nil, fmt.Errorf("failed to check access key uniqueness: %w", err)
}
credential := &iam_pb.Credential{
AccessKey: accessKey,
SecretKey: secretKey,
Status: AccessKeyStatusActive,
}
// Create access key using credential manager
err = s.credentialManager.CreateAccessKey(ctx, username, credential)
if err != nil {
return nil, fmt.Errorf("failed to create access key: %w", err)
}
return &AccessKeyInfo{
AccessKey: accessKey,
SecretKey: secretKey,
Status: AccessKeyStatusActive,
CreatedAt: time.Now(),
}, nil
}
// DeleteAccessKey deletes an access key for a user
func (s *AdminServer) DeleteAccessKey(username, accessKeyId string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Delete access key using credential manager
err := s.credentialManager.DeleteAccessKey(ctx, username, accessKeyId)
if err != nil {
if err == credential.ErrUserNotFound {
return fmt.Errorf("user %s not found", username)
}
if err == credential.ErrAccessKeyNotFound {
return fmt.Errorf("access key %s not found for user %s", accessKeyId, username)
}
return fmt.Errorf("failed to delete access key: %w", err)
}
return nil
}
// UpdateAccessKeyStatus updates the status of an access key for a user
func (s *AdminServer) UpdateAccessKeyStatus(username, accessKeyId, status string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
// Validate status against allowed values
if status != AccessKeyStatusActive && status != AccessKeyStatusInactive {
return fmt.Errorf("invalid status '%s': must be '%s' or '%s'", status, AccessKeyStatusActive, AccessKeyStatusInactive)
}
ctx := context.Background()
// Get user using credential manager
identity, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return fmt.Errorf("user %s not found", username)
}
return fmt.Errorf("failed to get user: %w", err)
}
// Find and update the access key status
found := false
for _, cred := range identity.Credentials {
if cred.AccessKey == accessKeyId {
cred.Status = status
found = true
break
}
}
if !found {
return fmt.Errorf("access key %s not found for user %s", accessKeyId, username)
}
// Update user using credential manager
err = s.credentialManager.UpdateUser(ctx, username, identity)
if err != nil {
return fmt.Errorf("failed to update user access key status: %w", err)
}
return nil
}
// GetUserPolicies returns the policies for a user (actions)
func (s *AdminServer) GetUserPolicies(username string) ([]string, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Get user using credential manager
identity, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return nil, fmt.Errorf("user %s not found", username)
}
return nil, fmt.Errorf("failed to get user: %w", err)
}
return identity.Actions, nil
}
// UpdateUserPolicies updates the policies (actions) for a user
func (s *AdminServer) UpdateUserPolicies(username string, actions []string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
ctx := context.Background()
// Get existing user
identity, err := s.credentialManager.GetUser(ctx, username)
if err != nil {
if err == credential.ErrUserNotFound {
return fmt.Errorf("user %s not found", username)
}
return fmt.Errorf("failed to get user: %w", err)
}
// Create updated identity with new actions
updatedIdentity := &iam_pb.Identity{
Name: identity.Name,
Account: identity.Account,
Credentials: identity.Credentials,
Actions: actions,
PolicyNames: identity.PolicyNames,
}
// Update user using credential manager
err = s.credentialManager.UpdateUser(ctx, username, updatedIdentity)
if err != nil {
return fmt.Errorf("failed to update user policies: %w", err)
}
return nil
}
// isNotFoundError checks for "not found" in the error message as a fallback
// for stores (e.g. gRPC) that don't return the credential.ErrAccessKeyNotFound sentinel.
func isNotFoundError(err error) bool {
return err != nil && strings.Contains(strings.ToLower(err.Error()), "not found")
}
// Helper functions for generating keys and IDs
func generateAccessKey() string {
// Generate 20-character access key (AWS standard)
const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
b := make([]byte, 20)
for i := range b {
b[i] = charset[randomInt(len(charset))]
}
return string(b)
}
func generateSecretKey() string {
// Use the IAM helper to generate URL-safe secret keys (no +, / characters)
// that won't break S3 signature authentication
key, err := iam.GenerateSecretAccessKey()
if err != nil {
panic(fmt.Sprintf("failed to generate secret key: %v", err))
}
return key
}
func generateAccountId() string {
// Generate 12-digit account ID
b := make([]byte, 4)
rand.Read(b)
val := (uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]))
return fmt.Sprintf("%012d", val)
}
func randomInt(max int) int {
b := make([]byte, 1)
rand.Read(b)
return int(b[0]) % max
}

View File

@@ -0,0 +1,116 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/credential"
_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc" // Import to register filer_etc store
"github.com/seaweedfs/seaweedfs/weed/pb"
"google.golang.org/grpc"
)
// TestFilerAddressFunctionInterface tests that the filer_etc store
// implements the correct SetFilerAddressFunc interface (issue #7575)
func TestFilerAddressFunctionInterface(t *testing.T) {
// Create credential manager with filer_etc store
credentialManager, err := credential.NewCredentialManagerWithDefaults("")
if err != nil {
t.Fatalf("Failed to initialize credential manager: %v", err)
}
store := credentialManager.GetStore()
if store == nil {
t.Fatal("Credential store is nil")
}
// Check if store is filer_etc type
if store.GetName() != credential.StoreTypeFilerEtc {
t.Skipf("Skipping test - store is not filer_etc (got: %s)", store.GetName())
}
// Check if store implements SetFilerAddressFunc interface
// This is the critical check for bug #7575
filerFuncSetter, ok := store.(interface {
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
})
if !ok {
t.Fatal("FilerEtcStore does not implement SetFilerAddressFunc interface - bug #7575")
}
// Verify we can call the method without panic
mockFilerAddress := pb.ServerAddress("localhost:8888")
filerFuncSetter.SetFilerAddressFunc(func() pb.ServerAddress {
return mockFilerAddress
}, grpc.WithInsecure())
t.Log("FilerEtcStore correctly implements SetFilerAddressFunc interface")
}
// TestGenerateAccessKey tests the access key generation function
func TestGenerateAccessKey(t *testing.T) {
key1 := generateAccessKey()
key2 := generateAccessKey()
// Check length
if len(key1) != 20 {
t.Errorf("Expected access key length 20, got %d", len(key1))
}
// Check uniqueness
if key1 == key2 {
t.Error("Generated access keys should be unique")
}
// Check character set
for _, c := range key1 {
if !((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
t.Errorf("Access key contains invalid character: %c", c)
}
}
}
// TestGenerateSecretKey tests the secret key generation function
func TestGenerateSecretKey(t *testing.T) {
key1 := generateSecretKey()
key2 := generateSecretKey()
// Check length (IAM standard secret key length)
if len(key1) != 42 {
t.Errorf("Expected secret key length 42, got %d", len(key1))
}
// Check that key contains only URL-safe characters (no +, /)
for _, c := range key1 {
if c == '+' || c == '/' || c == '=' {
t.Errorf("Secret key contains non-URL-safe character: %c", c)
}
}
// Check uniqueness
if key1 == key2 {
t.Error("Generated secret keys should be unique")
}
}
// TestGenerateAccountId tests the account ID generation function
func TestGenerateAccountId(t *testing.T) {
id1 := generateAccountId()
id2 := generateAccountId()
// Check length
if len(id1) != 12 {
t.Errorf("Expected account ID length 12, got %d", len(id1))
}
// Check that it's a number
for _, c := range id1 {
if c < '0' || c > '9' {
t.Errorf("Account ID contains non-digit character: %c", c)
}
}
// Check uniqueness (they should usually be different)
if id1 == id2 {
t.Log("Warning: Generated account IDs are the same (rare but possible)")
}
}

View File

@@ -0,0 +1,116 @@
package dash
import (
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
)
// TestVacuumMonitorStateTransitions verifies the state transition logic that
// drives the vacuum monitor goroutine: the plugin registry correctly reports
// whether a vacuum-capable worker is present or absent.
func TestVacuumMonitorStateTransitions(t *testing.T) {
t.Parallel()
opts := adminplugin.Options{
DataDir: "",
}
p, err := adminplugin.New(opts)
if err != nil {
t.Fatalf("failed to create plugin: %v", err)
}
defer p.Shutdown()
// Initially no workers => no capable worker.
if p.HasCapableWorker("vacuum") {
t.Fatalf("expected no capable worker initially")
}
// Simulate state transition false -> true: worker connects.
p.WorkerConnectForTest(&plugin_pb.WorkerHello{
WorkerId: "vacuum-worker-1",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanDetect: true, CanExecute: true},
},
})
if !p.HasCapableWorker("vacuum") {
t.Fatalf("expected capable worker after connect")
}
// Simulate state transition true -> false: worker disconnects.
p.WorkerDisconnectForTest("vacuum-worker-1")
// Give registry removal a moment (it's synchronous, but be safe).
time.Sleep(10 * time.Millisecond)
if p.HasCapableWorker("vacuum") {
t.Fatalf("expected no capable worker after disconnect")
}
}
// fakeToggler records disable/enable calls for testing.
type fakeToggler struct {
disableCalls int
enableCalls int
}
func (f *fakeToggler) disableVacuum() error {
f.disableCalls++
return nil
}
func (f *fakeToggler) enableVacuum() error {
f.enableCalls++
return nil
}
func TestSyncVacuumState(t *testing.T) {
t.Parallel()
t.Run("no change when state matches", func(t *testing.T) {
tog := &fakeToggler{}
result, failed := syncVacuumState(false, false, tog, false)
if result != false || failed {
t.Error("expected false, no failure")
}
result, failed = syncVacuumState(true, true, tog, false)
if result != true || failed {
t.Error("expected true, no failure")
}
if tog.disableCalls != 0 || tog.enableCalls != 0 {
t.Errorf("expected no calls, got disable=%d enable=%d", tog.disableCalls, tog.enableCalls)
}
})
t.Run("worker connects triggers disable", func(t *testing.T) {
tog := &fakeToggler{}
result, failed := syncVacuumState(true, false, tog, false)
if result != true || failed {
t.Error("expected true after disable, no failure")
}
if tog.disableCalls != 1 {
t.Errorf("expected 1 disable call, got %d", tog.disableCalls)
}
if tog.enableCalls != 0 {
t.Errorf("expected 0 enable calls, got %d", tog.enableCalls)
}
})
t.Run("worker disconnects triggers enable", func(t *testing.T) {
tog := &fakeToggler{}
result, failed := syncVacuumState(false, true, tog, false)
if result != false || failed {
t.Error("expected false after enable, no failure")
}
if tog.enableCalls != 1 {
t.Errorf("expected 1 enable call, got %d", tog.enableCalls)
}
if tog.disableCalls != 0 {
t.Errorf("expected 0 disable calls, got %d", tog.disableCalls)
}
})
}

View File

@@ -0,0 +1,339 @@
package dash
import (
"context"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// VolumeListExport is a full-cluster volume report, downloadable from the admin
// UI as JSON. It mirrors the topology the `volume.list` shell command walks
// (data center -> rack -> node -> disk -> volumes / EC shards) and adds derived
// fields the shell command does not print: garbage and fullness ratios, a
// human-readable modified time, remote-tiering keys, per-disk capacity counts,
// and the cluster-wide duplicate-volume-id scan.
type VolumeListExport struct {
GeneratedAt time.Time `json:"generated_at"`
VolumeSizeLimitMB uint64 `json:"volume_size_limit_mb"`
FilterCollection string `json:"filter_collection,omitempty"`
Totals VolumeExportTotals `json:"totals"`
DuplicateVolumeIds []DuplicateVolumeId `json:"duplicate_volume_ids"`
DataCenters []*ExportDataCenter `json:"data_centers"`
}
// VolumeExportTotals aggregates the included records. TotalSize sums normal
// volume sizes and EC shard sizes (matching volume.list's statistics); the
// file/delete totals cover normal volumes only.
type VolumeExportTotals struct {
VolumeCount int `json:"volume_count"`
EcShardCount int `json:"ec_shard_count"`
TotalSize uint64 `json:"total_size"`
FileCount uint64 `json:"file_count"`
DeletedFileCount uint64 `json:"deleted_file_count"`
DeletedBytes uint64 `json:"deleted_bytes"`
}
// DuplicateVolumeId flags a volume id that appears under more than one
// collection, where collection.delete or a bare-id operation is dangerous.
type DuplicateVolumeId struct {
VolumeId uint32 `json:"volume_id"`
Collections []string `json:"collections"`
}
type ExportDataCenter struct {
Id string `json:"id"`
Racks []*ExportRack `json:"racks"`
}
type ExportRack struct {
Id string `json:"id"`
Nodes []*ExportNode `json:"nodes"`
}
type ExportNode struct {
Id string `json:"id"`
GrpcPort uint32 `json:"grpc_port,omitempty"`
Disks []*ExportDisk `json:"disks"`
}
// ExportDisk capacity counts describe the whole physical disk; the Volumes /
// EcShards lists are filtered when a collection filter is active.
type ExportDisk struct {
DiskType string `json:"disk_type"`
DiskId uint32 `json:"disk_id"`
VolumeCount int64 `json:"volume_count"`
MaxVolumeCount int64 `json:"max_volume_count"`
ActiveVolumeCount int64 `json:"active_volume_count"`
FreeVolumeCount int64 `json:"free_volume_count"`
RemoteVolumeCount int64 `json:"remote_volume_count"`
Volumes []*ExportVolume `json:"volumes,omitempty"`
EcShards []*ExportEcShard `json:"ec_shards,omitempty"`
}
type ExportVolume struct {
Id uint32 `json:"id"`
Collection string `json:"collection"`
Size uint64 `json:"size"`
FileCount uint64 `json:"file_count"`
DeleteCount uint64 `json:"delete_count"`
DeletedByteCount uint64 `json:"deleted_byte_count"`
GarbageRatio float64 `json:"garbage_ratio"`
FullnessRatio float64 `json:"fullness_ratio"`
ReplicaPlacement string `json:"replica_placement"`
Version uint32 `json:"version"`
Ttl string `json:"ttl"`
ReadOnly bool `json:"read_only"`
CompactRevision uint32 `json:"compact_revision"`
ModifiedAtSecond int64 `json:"modified_at_second"`
ModifiedAt string `json:"modified_at,omitempty"`
DiskType string `json:"disk_type"`
DiskId uint32 `json:"disk_id"`
RemoteStorageName string `json:"remote_storage_name,omitempty"`
}
type ExportEcShard struct {
Id uint32 `json:"id"`
Collection string `json:"collection"`
ShardIds []uint32 `json:"shard_ids"`
ShardSizes []int64 `json:"shard_sizes"`
TotalSize int64 `json:"total_size"`
FileCount uint64 `json:"file_count"`
DeleteCount uint64 `json:"delete_count"`
DiskType string `json:"disk_type"`
DiskId uint32 `json:"disk_id"`
ExpireAtSec uint64 `json:"expire_at_sec,omitempty"`
}
// ExportClusterVolumeList builds a full-cluster volume report from the master
// topology. A non-empty collection limits the listed volumes and EC shards to
// that collection; the duplicate-id scan always covers the whole cluster.
// generatedAt is passed in so the report is deterministic and testable.
func (s *AdminServer) ExportClusterVolumeList(ctx context.Context, collection string, generatedAt time.Time) (*VolumeListExport, error) {
export := &VolumeListExport{
GeneratedAt: generatedAt,
FilterCollection: collection,
DuplicateVolumeIds: []DuplicateVolumeId{},
DataCenters: []*ExportDataCenter{},
}
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(ctx, client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
export.VolumeSizeLimitMB = resp.VolumeSizeLimitMb
topo := resp.TopologyInfo
if topo == nil {
return nil
}
volumeSizeLimit := resp.VolumeSizeLimitMb * 1024 * 1024
export.DuplicateVolumeIds = findDuplicateVolumeIdsForExport(topo)
dcs := append([]*master_pb.DataCenterInfo(nil), topo.DataCenterInfos...)
sort.Slice(dcs, func(i, j int) bool { return dcs[i].Id < dcs[j].Id })
for _, dc := range dcs {
exportDc := &ExportDataCenter{Id: dc.Id, Racks: []*ExportRack{}}
racks := append([]*master_pb.RackInfo(nil), dc.RackInfos...)
sort.Slice(racks, func(i, j int) bool { return racks[i].Id < racks[j].Id })
for _, rack := range racks {
exportRack := &ExportRack{Id: rack.Id, Nodes: []*ExportNode{}}
nodes := append([]*master_pb.DataNodeInfo(nil), rack.DataNodeInfos...)
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Id < nodes[j].Id })
for _, node := range nodes {
exportNode := &ExportNode{Id: node.Id, GrpcPort: node.GrpcPort, Disks: []*ExportDisk{}}
for _, diskType := range sortedDiskTypes(node.DiskInfos) {
// Split per physical disk like volume.list so DiskId is meaningful.
for _, diskInfo := range node.DiskInfos[diskType].SplitByPhysicalDisk() {
if disk := buildExportDisk(diskInfo, collection, volumeSizeLimit, &export.Totals); disk != nil {
exportNode.Disks = append(exportNode.Disks, disk)
}
}
}
if len(exportNode.Disks) > 0 {
exportRack.Nodes = append(exportRack.Nodes, exportNode)
}
}
if len(exportRack.Nodes) > 0 {
exportDc.Racks = append(exportDc.Racks, exportRack)
}
}
if len(exportDc.Racks) > 0 {
export.DataCenters = append(export.DataCenters, exportDc)
}
}
return nil
})
if err != nil {
return nil, err
}
return export, nil
}
func sortedDiskTypes(diskInfos map[string]*master_pb.DiskInfo) []string {
types := make([]string, 0, len(diskInfos))
for t := range diskInfos {
types = append(types, t)
}
sort.Strings(types)
return types
}
// buildExportDisk returns nil when the disk holds nothing matching the filter.
func buildExportDisk(diskInfo *master_pb.DiskInfo, collection string, volumeSizeLimit uint64, totals *VolumeExportTotals) *ExportDisk {
diskType := diskInfo.Type
if diskType == "" {
diskType = "hdd"
}
vols := append([]*master_pb.VolumeInformationMessage(nil), diskInfo.VolumeInfos...)
sort.Slice(vols, func(i, j int) bool { return vols[i].Id < vols[j].Id })
var volumes []*ExportVolume
for _, vi := range vols {
if collection != "" && !matchesCollection(vi.Collection, collection) {
continue
}
volumes = append(volumes, buildExportVolume(vi, volumeSizeLimit))
totals.VolumeCount++
totals.TotalSize += vi.Size
totals.FileCount += vi.FileCount
totals.DeletedFileCount += vi.DeleteCount
totals.DeletedBytes += vi.DeletedByteCount
}
shards := append([]*master_pb.VolumeEcShardInformationMessage(nil), diskInfo.EcShardInfos...)
sort.Slice(shards, func(i, j int) bool { return shards[i].Id < shards[j].Id })
var ecShards []*ExportEcShard
for _, eci := range shards {
if collection != "" && !matchesCollection(eci.Collection, collection) {
continue
}
shard := buildExportEcShard(eci)
ecShards = append(ecShards, shard)
totals.EcShardCount++
totals.TotalSize += uint64(shard.TotalSize)
}
if len(volumes) == 0 && len(ecShards) == 0 {
return nil
}
return &ExportDisk{
DiskType: diskType,
DiskId: diskInfo.DiskId,
VolumeCount: diskInfo.VolumeCount,
MaxVolumeCount: diskInfo.MaxVolumeCount,
ActiveVolumeCount: diskInfo.ActiveVolumeCount,
FreeVolumeCount: diskInfo.FreeVolumeCount,
RemoteVolumeCount: diskInfo.RemoteVolumeCount,
Volumes: volumes,
EcShards: ecShards,
}
}
func buildExportVolume(m *master_pb.VolumeInformationMessage, volumeSizeLimit uint64) *ExportVolume {
v := &ExportVolume{
Id: m.Id,
Collection: m.Collection,
Size: m.Size,
FileCount: m.FileCount,
DeleteCount: m.DeleteCount,
DeletedByteCount: m.DeletedByteCount,
Version: m.Version,
ReadOnly: m.ReadOnly,
CompactRevision: m.CompactRevision,
ModifiedAtSecond: m.ModifiedAtSecond,
DiskType: m.DiskType,
DiskId: m.DiskId,
RemoteStorageName: m.RemoteStorageName,
}
// Decode replica placement and TTL the way volume.list does.
if vi, err := storage.NewVolumeInfo(m); err == nil {
v.ReplicaPlacement = vi.ReplicaPlacement.String()
v.Ttl = vi.Ttl.String()
}
if m.Size > 0 {
v.GarbageRatio = float64(m.DeletedByteCount) / float64(m.Size)
}
if volumeSizeLimit > 0 {
v.FullnessRatio = float64(m.Size) / float64(volumeSizeLimit)
}
if m.ModifiedAtSecond > 0 {
v.ModifiedAt = time.Unix(m.ModifiedAtSecond, 0).UTC().Format(time.RFC3339)
}
return v
}
func buildExportEcShard(m *master_pb.VolumeEcShardInformationMessage) *ExportEcShard {
si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(m)
return &ExportEcShard{
Id: m.Id,
Collection: m.Collection,
ShardIds: si.IdsUint32(),
ShardSizes: si.SizesInt64(),
TotalSize: int64(si.TotalSize()),
FileCount: m.FileCount,
DeleteCount: m.DeleteCount,
DiskType: m.DiskType,
DiskId: m.DiskId,
ExpireAtSec: m.ExpireAtSec,
}
}
// findDuplicateVolumeIdsForExport reports volume ids living under more than one
// collection. Volume ids are meant to be globally unique; a duplicate is a
// cluster-health hazard (collection.delete destroys one collection's copy,
// bare-id lookups are ambiguous). This mirrors volume.list's findDuplicateVolumeIds
// rather than reaching into the shell package, per the admin-renderer design.
// First collection per id is tracked alone, and a set is allocated only on the
// first clash, keeping allocations O(duplicates) on million-volume clusters.
func findDuplicateVolumeIdsForExport(topo *master_pb.TopologyInfo) []DuplicateVolumeId {
firstCollectionByVid := make(map[uint32]string)
collectionsByVid := make(map[uint32]map[string]struct{})
note := func(vid uint32, collection string) {
if collections := collectionsByVid[vid]; collections != nil {
collections[collection] = struct{}{}
return
}
first, seen := firstCollectionByVid[vid]
if !seen {
firstCollectionByVid[vid] = collection
return
}
if first == collection {
return
}
collectionsByVid[vid] = map[string]struct{}{first: {}, collection: {}}
}
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, disk := range node.DiskInfos {
for _, vi := range disk.VolumeInfos {
note(vi.Id, vi.Collection)
}
for _, ec := range disk.EcShardInfos {
note(ec.Id, ec.Collection)
}
}
}
}
}
duplicates := make([]DuplicateVolumeId, 0, len(collectionsByVid))
for vid, set := range collectionsByVid {
names := make([]string, 0, len(set))
for name := range set {
names = append(names, name)
}
sort.Strings(names)
duplicates = append(duplicates, DuplicateVolumeId{VolumeId: vid, Collections: names})
}
sort.Slice(duplicates, func(i, j int) bool { return duplicates[i].VolumeId < duplicates[j].VolumeId })
return duplicates
}

View File

@@ -0,0 +1,129 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
func TestBuildExportVolumeDerivedFields(t *testing.T) {
v := buildExportVolume(&master_pb.VolumeInformationMessage{
Id: 7,
Collection: "photos",
Size: 100,
DeletedByteCount: 25,
ReplicaPlacement: 0,
ModifiedAtSecond: 1700000000,
}, 1000)
if v.GarbageRatio != 0.25 {
t.Errorf("garbage ratio = %v, want 0.25", v.GarbageRatio)
}
if v.FullnessRatio != 0.1 {
t.Errorf("fullness ratio = %v, want 0.1", v.FullnessRatio)
}
if v.ReplicaPlacement != "000" {
t.Errorf("replica placement = %q, want 000", v.ReplicaPlacement)
}
if v.ModifiedAt == "" {
t.Error("modified_at should be set when modified_at_second > 0")
}
}
func TestBuildExportDiskFilterAndTotals(t *testing.T) {
disk := &master_pb.DiskInfo{
Type: "hdd",
VolumeCount: 2,
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 1, Collection: "keep", Size: 10, FileCount: 3, DeleteCount: 1, DeletedByteCount: 2},
{Id: 2, Collection: "drop", Size: 99, FileCount: 9},
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 3, Collection: "keep", EcIndexBits: 0b111, ShardSizes: []int64{4, 4, 4}, FileCount: 5},
{Id: 4, Collection: "drop", EcIndexBits: 0b1, ShardSizes: []int64{7}},
},
}
var totals VolumeExportTotals
got := buildExportDisk(disk, "keep", 1000, &totals)
if got == nil {
t.Fatal("expected disk with matching records, got nil")
}
if len(got.Volumes) != 1 || got.Volumes[0].Id != 1 {
t.Errorf("expected only volume 1 after filter, got %+v", got.Volumes)
}
if len(got.EcShards) != 1 || got.EcShards[0].Id != 3 {
t.Errorf("expected only ec shard 3 after filter, got %+v", got.EcShards)
}
if got.VolumeCount != 2 {
t.Errorf("disk VolumeCount should stay physical (2), got %d", got.VolumeCount)
}
if got.EcShards[0].TotalSize != 12 {
t.Errorf("ec shard total size = %d, want 12", got.EcShards[0].TotalSize)
}
if totals.VolumeCount != 1 || totals.EcShardCount != 1 {
t.Errorf("totals counts = vol %d ec %d, want 1/1", totals.VolumeCount, totals.EcShardCount)
}
// 10 (volume size) + 12 (kept ec shard sizes).
if totals.TotalSize != 22 {
t.Errorf("totals TotalSize = %d, want 22", totals.TotalSize)
}
if totals.FileCount != 3 || totals.DeletedFileCount != 1 || totals.DeletedBytes != 2 {
t.Errorf("totals files = %d del %d delBytes %d, want 3/1/2", totals.FileCount, totals.DeletedFileCount, totals.DeletedBytes)
}
}
func TestBuildExportDiskPrunesWhenNoMatch(t *testing.T) {
disk := &master_pb.DiskInfo{
Type: "hdd",
VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 1, Collection: "other"}},
}
var totals VolumeExportTotals
if got := buildExportDisk(disk, "keep", 0, &totals); got != nil {
t.Errorf("expected nil for disk with no matching records, got %+v", got)
}
}
func TestFindDuplicateVolumeIdsForExport(t *testing.T) {
topo := &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{{
RackInfos: []*master_pb.RackInfo{{
DataNodeInfos: []*master_pb.DataNodeInfo{
{DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 5, Collection: "a"},
{Id: 7, Collection: "shared"}, // replica below, same collection
},
}}},
{DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {
VolumeInfos: []*master_pb.VolumeInformationMessage{
{Id: 5, Collection: "b"}, // 5 now spans a + b -> duplicate
{Id: 7, Collection: "shared"}, // replica, not a duplicate
},
EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
{Id: 5, Collection: "c"}, // 5 also as EC in c
},
}}},
},
}},
}},
}
dups := findDuplicateVolumeIdsForExport(topo)
if len(dups) != 1 {
t.Fatalf("expected exactly 1 duplicate, got %+v", dups)
}
if dups[0].VolumeId != 5 {
t.Errorf("duplicate volume id = %d, want 5", dups[0].VolumeId)
}
want := []string{"a", "b", "c"}
if len(dups[0].Collections) != len(want) {
t.Fatalf("collections = %v, want %v", dups[0].Collections, want)
}
for i := range want {
if dups[0].Collections[i] != want[i] {
t.Errorf("collections[%d] = %q, want %q", i, dups[0].Collections[i], want[i])
}
}
}

View File

@@ -0,0 +1,651 @@
package dash
import (
"context"
"fmt"
"math"
"sort"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
)
// GetClusterVolumes retrieves cluster volumes data with pagination, sorting, and filtering
func (s *AdminServer) GetClusterVolumes(page int, pageSize int, sortBy string, sortOrder string, collection string) (*ClusterVolumesData, error) {
// Set defaults
if page < 1 {
page = 1
}
if pageSize < 1 || pageSize > 1000 {
pageSize = 100
}
if sortBy == "" {
sortBy = "id"
}
if sortOrder == "" {
sortOrder = "asc"
}
var volumes []VolumeWithTopology
var totalSize int64
var cachedTopologyInfo *master_pb.TopologyInfo
// Get detailed volume information via gRPC
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
// Cache the topology info for reuse
cachedTopologyInfo = resp.TopologyInfo
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
// Process regular volumes
for _, volInfo := range diskInfo.VolumeInfos {
volume := VolumeWithTopology{
VolumeInformationMessage: volInfo,
Server: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
}
volumes = append(volumes, volume)
totalSize += int64(volInfo.Size)
}
// Process EC shards in the same loop
for _, ecShardInfo := range diskInfo.EcShardInfos {
// Add all shard sizes for this EC volume
for _, shardSize := range ecShardInfo.ShardSizes {
totalSize += shardSize
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
// Filter by collection if specified
if collection != "" {
var filteredVolumes []VolumeWithTopology
var filteredTotalSize int64
var filteredEcTotalSize int64
for _, volume := range volumes {
if matchesCollection(volume.Collection, collection) {
filteredVolumes = append(filteredVolumes, volume)
filteredTotalSize += int64(volume.Size)
}
}
// Filter EC shard sizes by collection using already processed data
// This reuses the topology traversal done above (lines 43-71) to avoid a second pass
if cachedTopologyInfo != nil {
for _, dc := range cachedTopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
for _, ecShardInfo := range diskInfo.EcShardInfos {
if matchesCollection(ecShardInfo.Collection, collection) {
// Add all shard sizes for this EC volume
for _, shardSize := range ecShardInfo.ShardSizes {
filteredEcTotalSize += shardSize
}
}
}
}
}
}
}
}
volumes = filteredVolumes
totalSize = filteredTotalSize + filteredEcTotalSize
}
// Calculate unique data center, rack, disk type, collection, and version counts from filtered volumes
dataCenterMap := make(map[string]bool)
rackMap := make(map[string]bool)
diskTypeMap := make(map[string]bool)
collectionMap := make(map[string]bool)
versionMap := make(map[string]bool)
hasRemoteVolumes := false
for _, volume := range volumes {
if volume.RemoteStorageName != "" {
hasRemoteVolumes = true
}
if volume.DataCenter != "" {
dataCenterMap[volume.DataCenter] = true
}
if volume.Rack != "" {
rackMap[volume.Rack] = true
}
diskType := volume.DiskType
if diskType == "" {
diskType = "hdd" // Default to hdd if not specified
}
diskTypeMap[diskType] = true
// Handle collection for display purposes
collectionName := volume.Collection
if collectionName == "" {
collectionName = "default"
}
collectionMap[collectionName] = true
versionMap[fmt.Sprintf("%d", volume.Version)] = true
}
dataCenterCount := len(dataCenterMap)
rackCount := len(rackMap)
diskTypeCount := len(diskTypeMap)
collectionCount := len(collectionMap)
versionCount := len(versionMap)
// Sort volumes
s.sortVolumes(volumes, sortBy, sortOrder)
// Get volume size limit from master
var volumeSizeLimit uint64
err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
volumeSizeLimit = uint64(resp.VolumeSizeLimitMB) * 1024 * 1024 // Convert MB to bytes
return nil
})
if err != nil {
// If we can't get the limit, set a default
volumeSizeLimit = 30 * 1024 * 1024 * 1024 // 30GB default
}
// Calculate pagination
totalVolumes := len(volumes)
totalPages := (totalVolumes + pageSize - 1) / pageSize
if totalPages == 0 {
totalPages = 1
}
// Apply pagination
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if startIndex >= totalVolumes {
volumes = []VolumeWithTopology{}
} else {
if endIndex > totalVolumes {
endIndex = totalVolumes
}
volumes = volumes[startIndex:endIndex]
}
// Determine conditional display flags and extract single values
showDataCenterColumn := dataCenterCount > 1
showRackColumn := rackCount > 1
// Remote-tiered volumes surface in the disk type column, so show it
// whenever any volume lives on a remote tier.
showDiskTypeColumn := diskTypeCount > 1 || hasRemoteVolumes
showCollectionColumn := collectionCount > 1 && collection == "" // Hide column when filtering by collection
showVersionColumn := versionCount > 1
var singleDataCenter, singleRack, singleDiskType, singleCollection, singleVersion string
var allVersions, allDiskTypes []string
if dataCenterCount == 1 {
for dc := range dataCenterMap {
singleDataCenter = dc
break
}
}
if rackCount == 1 {
for rack := range rackMap {
singleRack = rack
break
}
}
if diskTypeCount == 1 {
for diskType := range diskTypeMap {
singleDiskType = diskType
break
}
} else {
// Collect all disk types and sort them
for diskType := range diskTypeMap {
allDiskTypes = append(allDiskTypes, diskType)
}
sort.Strings(allDiskTypes)
}
if collectionCount == 1 {
for collection := range collectionMap {
singleCollection = collection
break
}
}
if versionCount == 1 {
for version := range versionMap {
singleVersion = "v" + version
break
}
} else {
// Collect all versions and sort them
for version := range versionMap {
allVersions = append(allVersions, "v"+version)
}
sort.Strings(allVersions)
}
return &ClusterVolumesData{
Volumes: volumes,
TotalVolumes: totalVolumes,
TotalSize: totalSize,
VolumeSizeLimit: volumeSizeLimit,
LastUpdated: time.Now(),
CurrentPage: page,
TotalPages: totalPages,
PageSize: pageSize,
SortBy: sortBy,
SortOrder: sortOrder,
DataCenterCount: dataCenterCount,
RackCount: rackCount,
DiskTypeCount: diskTypeCount,
CollectionCount: collectionCount,
VersionCount: versionCount,
ShowDataCenterColumn: showDataCenterColumn,
ShowRackColumn: showRackColumn,
ShowDiskTypeColumn: showDiskTypeColumn,
ShowCollectionColumn: showCollectionColumn,
ShowVersionColumn: showVersionColumn,
SingleDataCenter: singleDataCenter,
SingleRack: singleRack,
SingleDiskType: singleDiskType,
SingleCollection: singleCollection,
SingleVersion: singleVersion,
AllVersions: allVersions,
AllDiskTypes: allDiskTypes,
FilterCollection: collection,
}, nil
}
// sortVolumes sorts the volumes slice based on the specified field and order
func (s *AdminServer) sortVolumes(volumes []VolumeWithTopology, sortBy string, sortOrder string) {
sort.Slice(volumes, func(i, j int) bool {
var less bool
switch sortBy {
case "id":
less = volumes[i].Id < volumes[j].Id
case "server":
less = volumes[i].Server < volumes[j].Server
case "datacenter":
less = volumes[i].DataCenter < volumes[j].DataCenter
case "rack":
less = volumes[i].Rack < volumes[j].Rack
case "collection":
less = volumes[i].Collection < volumes[j].Collection
case "size":
less = volumes[i].Size < volumes[j].Size
case "filecount":
less = volumes[i].FileCount < volumes[j].FileCount
case "replication":
less = volumes[i].ReplicaPlacement < volumes[j].ReplicaPlacement
case "disktype":
less = volumes[i].DiskType < volumes[j].DiskType
case "version":
less = volumes[i].Version < volumes[j].Version
default:
less = volumes[i].Id < volumes[j].Id
}
if sortOrder == "desc" {
return !less
}
return less
})
}
// GetVolumeDetails retrieves detailed information about a specific volume
func (s *AdminServer) GetVolumeDetails(volumeID uint32, server string) (*VolumeDetailsData, error) {
var primaryVolume VolumeWithTopology
var replicas []VolumeWithTopology
var volumeSizeLimit uint64
var found bool
// Find the volume and all its replicas in the cluster
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(context.Background(), &master_pb.VolumeListRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
return err
}
if resp.TopologyInfo != nil {
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
for _, diskInfo := range node.DiskInfos {
for _, volInfo := range diskInfo.VolumeInfos {
// An older master ignores the filter.
if volInfo.Id == volumeID {
diskType := volInfo.DiskType
if diskType == "" {
diskType = "hdd"
}
volume := VolumeWithTopology{
VolumeInformationMessage: volInfo,
Server: node.Id,
DataCenter: dc.Id,
Rack: rack.Id,
}
// If this is the requested server, it's the primary volume
if node.Id == server {
primaryVolume = volume
found = true
} else {
// This is a replica on another server
replicas = append(replicas, volume)
}
}
}
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("volume %d not found on server %s", volumeID, server)
}
// Get volume size limit from master
err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
volumeSizeLimit = uint64(resp.VolumeSizeLimitMB) * 1024 * 1024 // Convert MB to bytes
return nil
})
if err != nil {
// If we can't get the limit, set a default
volumeSizeLimit = 30 * 1024 * 1024 * 1024 // 30GB default
}
return &VolumeDetailsData{
Volume: primaryVolume,
Replicas: replicas,
VolumeSizeLimit: volumeSizeLimit,
ReplicationCount: len(replicas) + 1, // Include the primary volume
LastUpdated: time.Now(),
}, nil
}
// SetVolumeReadOnly changes the access mode of a single volume replica.
func (s *AdminServer) SetVolumeReadOnly(ctx context.Context, volumeID uint32, server string, readOnly bool) error {
var address pb.ServerAddress
err := s.masterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(ctx, &master_pb.VolumeListRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
return err
}
address, err = volumeReplicaAddress(resp.GetTopologyInfo(), volumeID, server)
return err
})
if err != nil {
return err
}
return s.WithVolumeServerClient(address, func(client volume_server_pb.VolumeServerClient) error {
if readOnly {
_, err := client.VolumeMarkReadonly(ctx, &volume_server_pb.VolumeMarkReadonlyRequest{
VolumeId: volumeID,
Persist: true,
})
return err
}
_, err := client.VolumeMarkWritable(ctx, &volume_server_pb.VolumeMarkWritableRequest{VolumeId: volumeID})
return err
})
}
// Resolve the selected replica through the master rather than dialing a
// user-supplied address. Node IDs can differ from their network addresses.
func volumeReplicaAddress(topology *master_pb.TopologyInfo, volumeID uint32, server string) (pb.ServerAddress, error) {
for _, dc := range topology.GetDataCenterInfos() {
for _, rack := range dc.GetRackInfos() {
for _, node := range rack.GetDataNodeInfos() {
if node.GetId() != server {
continue
}
for _, disk := range node.GetDiskInfos() {
for _, volume := range disk.GetVolumeInfos() {
// Older masters may ignore the volume ID filter.
if volume.GetId() == volumeID {
return pb.NewServerAddressFromDataNode(node), nil
}
}
}
}
}
}
return "", fmt.Errorf("volume %d not found on server %s", volumeID, server)
}
// VacuumVolume performs a vacuum operation on a specific volume
func (s *AdminServer) VacuumVolume(volumeID int, server string) error {
// Validate volumeID range before converting to uint32
if volumeID < 0 || uint64(volumeID) > math.MaxUint32 {
return fmt.Errorf("volume ID out of range: %d", volumeID)
}
return s.WithMasterClient(func(client master_pb.SeaweedClient) error {
_, err := client.VacuumVolume(context.Background(), &master_pb.VacuumVolumeRequest{
// lgtm[go/incorrect-integer-conversion]
// Safe conversion: volumeID has been validated to be in range [0, 0xFFFFFFFF] above
VolumeId: uint32(volumeID),
GarbageThreshold: 0.0001, // A very low threshold to ensure all garbage is collected
Collection: "", // Empty for all collections
})
return err
})
}
// GetClusterVolumeServers retrieves cluster volume servers data including EC shard information
func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, error) {
var volumeServerMap map[string]*VolumeServer
// Fetch public URL mapping from master HTTP API
publicUrls := s.fetchPublicUrlMap()
// Make only ONE VolumeList call and use it for both topology building AND EC shard processing
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
resp, err := pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
// Get volume size limit from response, default to 30GB if not set
volumeSizeLimitMB := resp.VolumeSizeLimitMb
if volumeSizeLimitMB == 0 {
volumeSizeLimitMB = 30000 // default to 30000MB (30GB)
}
// Build basic topology from the VolumeList response (replaces GetClusterTopology call)
volumeServerMap = make(map[string]*VolumeServer)
if resp.TopologyInfo != nil {
// Process topology to build basic volume server info (similar to cluster_topology.go logic)
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, node := range rack.DataNodeInfos {
// Initialize volume server if not exists
if volumeServerMap[node.Id] == nil {
// Look up PublicUrl from master HTTP API
nodeAddr := node.Address
if nodeAddr == "" {
nodeAddr = node.Id
}
publicUrl := publicUrls[nodeAddr]
if publicUrl == "" {
publicUrl = nodeAddr
}
volumeServerMap[node.Id] = &VolumeServer{
Address: node.Id,
PublicURL: publicUrl,
DataCenter: dc.Id,
Rack: rack.Id,
Volumes: 0,
DiskUsage: 0,
DiskCapacity: 0,
EcVolumes: 0,
EcShards: 0,
EcShardDetails: []VolumeServerEcInfo{},
}
}
vs := volumeServerMap[node.Id]
// Process EC shard information for this server at volume server level (not per-disk)
ecVolumeMap := make(map[uint32]*VolumeServerEcInfo)
// Temporary map to accumulate shard info across disks
ecShardAccumulator := make(map[uint32][]*master_pb.VolumeEcShardInformationMessage)
// Process disk information
for _, diskInfo := range node.DiskInfos {
vs.MaxVolumes += int(diskInfo.MaxVolumeCount)
// Prefer the real physical disk capacity the volume server
// reports; the slot-based estimate overstates capacity when
// maxVolumeCount is configured higher than the disk holds.
if diskInfo.DiskTotalBytes > 0 {
vs.DiskCapacity += int64(diskInfo.DiskTotalBytes)
} else {
vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024
}
// Count regular volumes and calculate disk usage.
// A remote-tiered volume reports its cloud object's
// size; keep those bytes out of the local disk usage
// that is compared against DiskCapacity.
for _, volInfo := range diskInfo.VolumeInfos {
vs.Volumes++
if volInfo.RemoteStorageName != "" {
vs.RemoteSize += int64(volInfo.Size)
} else {
vs.DiskUsage += int64(volInfo.Size)
}
}
// Accumulate EC shard information across all disks for this volume server
for _, ecShardInfo := range diskInfo.EcShardInfos {
volumeId := ecShardInfo.Id
ecShardAccumulator[volumeId] = append(ecShardAccumulator[volumeId], ecShardInfo)
}
}
// Process accumulated EC shard information per volume
for volumeId, ecShardInfos := range ecShardAccumulator {
if len(ecShardInfos) == 0 {
continue
}
// Initialize EC volume info
ecInfo := &VolumeServerEcInfo{
VolumeID: volumeId,
Collection: ecShardInfos[0].Collection,
ShardCount: 0,
EcIndexBits: 0,
ShardNumbers: []int{},
ShardSizes: make(map[int]int64),
TotalSize: 0,
}
// Merge EcIndexBits from all disks and collect shard sizes
allShardSizes := make(map[erasure_coding.ShardId]int64)
for _, ecShardInfo := range ecShardInfos {
si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)
ecInfo.EcIndexBits |= si.Bitmap()
// Collect shard sizes from this disk
for _, id := range si.Ids() {
allShardSizes[id] += int64(si.Size(id))
}
}
// Process final merged shard information
for shardId := range allShardSizes {
ecInfo.ShardCount++
ecInfo.ShardNumbers = append(ecInfo.ShardNumbers, int(shardId))
vs.EcShards++
// Add shard size if available
if shardSize, exists := allShardSizes[shardId]; exists {
ecInfo.ShardSizes[int(shardId)] = shardSize
ecInfo.TotalSize += shardSize
vs.DiskUsage += shardSize // Add EC shard size to total disk usage
}
}
ecVolumeMap[volumeId] = ecInfo
}
// Convert EC volume map to slice and update volume server (after processing all disks)
for _, ecInfo := range ecVolumeMap {
vs.EcShardDetails = append(vs.EcShardDetails, *ecInfo)
vs.EcVolumes++
}
}
}
}
}
return nil
})
if err != nil {
return nil, err
}
// Convert map back to slice
var volumeServers []VolumeServer
for _, vs := range volumeServerMap {
volumeServers = append(volumeServers, *vs)
}
// Sort volume servers by address for consistent ordering on page refresh
sort.Slice(volumeServers, func(i, j int) bool {
return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
})
var totalCapacity int64
var totalVolumes int
for _, vs := range volumeServers {
totalCapacity += vs.DiskCapacity
totalVolumes += vs.Volumes
}
return &ClusterVolumeServersData{
VolumeServers: volumeServers,
TotalVolumeServers: len(volumeServers),
TotalVolumes: totalVolumes,
TotalCapacity: totalCapacity,
LastUpdated: time.Now(),
}, nil
}

View File

@@ -0,0 +1,159 @@
package dash
import (
"context"
"net"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
)
type volumeAccessTestMaster struct {
master_pb.UnimplementedSeaweedServer
topology *master_pb.TopologyInfo
}
type volumeAccessTestServer struct {
volume_server_pb.UnimplementedVolumeServerServer
rpcError error
calls chan any
}
func (s *volumeAccessTestMaster) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error {
if _, err := stream.Recv(); err != nil {
return err
}
if err := stream.Send(&master_pb.KeepConnectedResponse{}); err != nil {
return err
}
<-stream.Context().Done()
return stream.Context().Err()
}
func (s *volumeAccessTestMaster) VolumeList(context.Context, *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) {
// Return unfiltered topology to exercise compatibility with older masters.
return &master_pb.VolumeListResponse{TopologyInfo: s.topology}, nil
}
func (s *volumeAccessTestServer) VolumeMarkReadonly(_ context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
s.calls <- req
return &volume_server_pb.VolumeMarkReadonlyResponse{}, s.rpcError
}
func (s *volumeAccessTestServer) VolumeMarkWritable(_ context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
s.calls <- req
return &volume_server_pb.VolumeMarkWritableResponse{}, s.rpcError
}
func volumeAccessTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
return &master_pb.TopologyInfo{DataCenterInfos: []*master_pb.DataCenterInfo{{
RackInfos: []*master_pb.RackInfo{{DataNodeInfos: nodes}},
}}}
}
func TestSetVolumeReadOnly(t *testing.T) {
for _, tc := range []struct {
name string
readOnly bool
volumeID uint32
server string
rpcError error
wantCall bool
}{
{name: "persist read-only", readOnly: true, volumeID: 7, server: "node-a", wantCall: true},
{name: "restore read-write", volumeID: 7, server: "node-a", wantCall: true},
{name: "propagate volume server error", volumeID: 7, server: "node-a", wantCall: true, rpcError: status.Error(codes.PermissionDenied, "server is in maintenance mode")},
{name: "reject unknown volume", readOnly: true, volumeID: 8, server: "node-a"},
{name: "reject unknown server", readOnly: true, volumeID: 7, server: "127.0.0.1:9999"},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := listener.Addr().(*net.TCPAddr).Port
master := &volumeAccessTestMaster{
topology: volumeAccessTestTopology(&master_pb.DataNodeInfo{
Id: "node-a", Address: "127.0.0.1:8080", GrpcPort: uint32(port),
DiskInfos: map[string]*master_pb.DiskInfo{"hdd": {
VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 7}},
}},
}),
}
fake := &volumeAccessTestServer{
rpcError: tc.rpcError,
calls: make(chan any, 1),
}
grpcServer := grpc.NewServer()
master_pb.RegisterSeaweedServer(grpcServer, master)
volume_server_pb.RegisterVolumeServerServer(grpcServer, fake)
go grpcServer.Serve(listener)
t.Cleanup(grpcServer.Stop)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
address := pb.NewServerAddress("127.0.0.1", 9333, port)
discovery := pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"master": address})
masterClient := wdclient.NewMasterClient(dialOption, "", "admin", "", "", "", *discovery)
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
go masterClient.KeepConnectedToMaster(ctx)
admin := &AdminServer{masterClient: masterClient, grpcDialOption: dialOption}
err = admin.SetVolumeReadOnly(ctx, tc.volumeID, tc.server, tc.readOnly)
if !tc.wantCall {
require.ErrorContains(t, err, "not found on server")
require.Empty(t, fake.calls, "an unknown replica must not receive a mutation")
return
}
if tc.rpcError != nil {
require.Equal(t, status.Code(tc.rpcError), status.Code(err))
} else {
require.NoError(t, err)
}
select {
case call := <-fake.calls:
if tc.readOnly {
req, ok := call.(*volume_server_pb.VolumeMarkReadonlyRequest)
require.True(t, ok)
require.Equal(t, tc.volumeID, req.VolumeId)
require.True(t, req.Persist, "operator changes must survive a restart")
require.False(t, req.CanDelete)
} else {
req, ok := call.(*volume_server_pb.VolumeMarkWritableRequest)
require.True(t, ok)
require.Equal(t, tc.volumeID, req.VolumeId)
}
default:
t.Fatal("volume server did not receive an access mode change")
}
})
}
}
func TestVolumeReplicaAddress(t *testing.T) {
topology := volumeAccessTestTopology(
&master_pb.DataNodeInfo{Id: "legacy:8080", DiskInfos: map[string]*master_pb.DiskInfo{
"hdd": {VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 7}}},
}},
&master_pb.DataNodeInfo{Id: "node-b", Address: "host-b:8081", GrpcPort: 18082, DiskInfos: map[string]*master_pb.DiskInfo{
"ssd": {VolumeInfos: []*master_pb.VolumeInformationMessage{{Id: 7}}, EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{{Id: 8}}},
}},
)
address, err := volumeReplicaAddress(topology, 7, "legacy:8080")
require.NoError(t, err)
require.Equal(t, "legacy:18080", address.ToGrpcAddress())
address, err = volumeReplicaAddress(topology, 7, "node-b")
require.NoError(t, err)
require.Equal(t, "host-b:18082", address.ToGrpcAddress())
_, err = volumeReplicaAddress(topology, 8, "node-b")
require.Error(t, err, "EC shards must not be treated as regular volumes")
_, err = volumeReplicaAddress(nil, 7, "node-b")
require.Error(t, err)
}

View File

@@ -0,0 +1,130 @@
package dash
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// TestMergeWorkerFleetTotals covers the accounting behind
// SeaweedFS_admin_workers_connected / SeaweedFS_admin_worker_slots and the
// dashboard's Workers card. Both used to read the legacy maintenance-worker
// registry only, so a cluster whose admin and workers run as separate
// components reported 0 workers (issue #10525).
func TestMergeWorkerFleetTotals(t *testing.T) {
pluginWorker := func(id string, detectUsed, detectTotal, executeUsed, executeTotal int32) *adminplugin.WorkerSession {
return &adminplugin.WorkerSession{
WorkerID: id,
Heartbeat: &plugin_pb.WorkerHeartbeat{
DetectionSlotsUsed: detectUsed,
DetectionSlotsTotal: detectTotal,
ExecutionSlotsUsed: executeUsed,
ExecutionSlotsTotal: executeTotal,
},
}
}
testCases := []struct {
name string
legacySlots map[string]maintenance.WorkerSlots
pluginWorkers []*adminplugin.WorkerSession
wantWorkers int
wantUsedSlots int
wantMaxSlots int
}{
{
name: "no workers at all",
},
{
name: "legacy workers only, unchanged accounting",
legacySlots: map[string]maintenance.WorkerSlots{
"w-legacy-a": {Used: 1, Max: 2},
"w-legacy-b": {Used: 0, Max: 4},
},
wantWorkers: 2,
wantUsedSlots: 1,
wantMaxSlots: 6,
},
{
// The reported bug: admin and workers deployed separately, so
// nothing ever lands in the legacy registry.
name: "plugin workers only are counted",
pluginWorkers: []*adminplugin.WorkerSession{
pluginWorker("w-plugin-a", 0, 1, 2, 4),
pluginWorker("w-plugin-b", 1, 1, 0, 4),
},
wantWorkers: 2,
wantUsedSlots: 3,
wantMaxSlots: 10,
},
{
name: "distinct workers in both registries are summed",
legacySlots: map[string]maintenance.WorkerSlots{
"w-legacy-a": {Used: 1, Max: 2},
},
pluginWorkers: []*adminplugin.WorkerSession{
pluginWorker("w-plugin-a", 0, 1, 1, 4),
},
wantWorkers: 2,
wantUsedSlots: 2,
wantMaxSlots: 7,
},
{
// `weed mini` runs both worker runtimes out of one working
// directory, so they share the persisted worker ID and must not be
// counted twice. Legacy slots win for such a worker.
name: "same worker ID in both registries counts once",
legacySlots: map[string]maintenance.WorkerSlots{
"w-host-abcd": {Used: 1, Max: 2},
},
pluginWorkers: []*adminplugin.WorkerSession{
pluginWorker("w-host-abcd", 1, 1, 3, 4),
},
wantWorkers: 1,
wantUsedSlots: 1,
wantMaxSlots: 2,
},
{
name: "plugin worker without a heartbeat still counts as connected",
pluginWorkers: []*adminplugin.WorkerSession{
{WorkerID: "w-plugin-a"},
},
wantWorkers: 1,
},
{
name: "nil session is skipped",
pluginWorkers: []*adminplugin.WorkerSession{nil, pluginWorker("w-plugin-a", 0, 1, 0, 2)},
wantWorkers: 1,
wantMaxSlots: 3,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
workers, usedSlots, maxSlots := mergeWorkerFleetTotals(tc.legacySlots, tc.pluginWorkers)
if workers != tc.wantWorkers {
t.Errorf("workers = %d, want %d", workers, tc.wantWorkers)
}
if usedSlots != tc.wantUsedSlots {
t.Errorf("usedSlots = %d, want %d", usedSlots, tc.wantUsedSlots)
}
if maxSlots != tc.wantMaxSlots {
t.Errorf("maxSlots = %d, want %d", maxSlots, tc.wantMaxSlots)
}
})
}
}
// TestWorkerFleetTotalsWithoutMaintenanceManager guards the nil-manager path:
// an admin server with no maintenance manager must still report its plugin
// workers rather than returning early.
func TestWorkerFleetTotalsWithoutMaintenanceManager(t *testing.T) {
server := &AdminServer{}
workers, usedSlots, maxSlots := server.workerFleetTotals()
if workers != 0 || usedSlots != 0 || maxSlots != 0 {
t.Fatalf("empty admin server reported workers=%d used=%d max=%d, want all 0", workers, usedSlots, maxSlots)
}
}

View File

@@ -0,0 +1,860 @@
package dash
import (
"context"
"errors"
"fmt"
"io"
"net"
"strconv"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/peer"
"google.golang.org/grpc/status"
)
const (
maxLogFetchLimit = 1000
maxLogMessageSize = 2000
maxLogFieldsCount = 20
logRequestTimeout = 10 * time.Second
logResponseTimeout = 30 * time.Second
logSendTimeout = 10 * time.Second
)
// WorkerGrpcServer implements the WorkerService gRPC interface
type WorkerGrpcServer struct {
worker_pb.UnimplementedWorkerServiceServer
adminServer *AdminServer
// Worker connection management
connections map[string]*WorkerConnection
connMutex sync.RWMutex
// Log request correlation
pendingLogRequests map[string]*LogRequestContext
logRequestsMutex sync.RWMutex
// gRPC server
grpcServer *grpc.Server
listener net.Listener
running bool
stopChan chan struct{}
mtlsEnabled bool
}
// LogRequestContext tracks pending log requests
type LogRequestContext struct {
TaskID string
WorkerID string
ResponseCh chan *worker_pb.TaskLogResponse
}
// WorkerConnection represents an active worker connection
type WorkerConnection struct {
workerID string
stream worker_pb.WorkerService_WorkerStreamServer
lastSeen time.Time
capabilities []MaintenanceTaskType
address string
maxConcurrent int32
outgoing chan *worker_pb.AdminMessage
ctx context.Context
cancel context.CancelFunc
}
// NewWorkerGrpcServer creates a new gRPC server for worker connections
func NewWorkerGrpcServer(adminServer *AdminServer) *WorkerGrpcServer {
return &WorkerGrpcServer{
adminServer: adminServer,
connections: make(map[string]*WorkerConnection),
pendingLogRequests: make(map[string]*LogRequestContext),
stopChan: make(chan struct{}),
}
}
// StartWithTLS starts the gRPC server on the specified port with optional TLS.
// A caller that already holds the port passes its listener instead. When no
// listener is supplied the server binds to bindIp to honor the operator's -ip.
func (s *WorkerGrpcServer) StartWithTLS(bindIp string, port int, listener net.Listener) error {
if s.running {
return fmt.Errorf("worker gRPC server is already running")
}
if listener == nil {
var err error
listener, err = net.Listen("tcp", util.JoinHostPort(bindIp, port))
if err != nil {
return fmt.Errorf("failed to listen on port %d: %v", port, err)
}
}
// Create gRPC server with optional TLS
tlsOption, _ := security.LoadServerTLS(util.GetViper(), "grpc.admin")
s.mtlsEnabled = tlsOption != nil
grpcServer := pb.NewGrpcServer(tlsOption)
worker_pb.RegisterWorkerServiceServer(grpcServer, s)
if plugin := s.adminServer.GetPlugin(); plugin != nil {
plugin_pb.RegisterPluginControlServiceServer(grpcServer, plugin)
glog.V(0).Infof("Plugin gRPC service registered on worker gRPC server")
}
s.grpcServer = grpcServer
s.listener = listener
s.running = true
// Start background routines
go s.cleanupRoutine()
go s.activeLogFetchLoop()
pb.ServeGrpcOnLocalSocket(grpcServer, port)
// Start serving in a goroutine
go func() {
if err := s.grpcServer.Serve(listener); err != nil {
if s.running {
glog.Errorf("Worker gRPC server error: %v", err)
}
}
}()
return nil
}
// ListenPort returns the currently bound worker gRPC listen port.
func (s *WorkerGrpcServer) ListenPort() int {
if s == nil || s.listener == nil {
return 0
}
if tcpAddr, ok := s.listener.Addr().(*net.TCPAddr); ok {
return tcpAddr.Port
}
_, portStr, err := net.SplitHostPort(s.listener.Addr().String())
if err != nil {
return 0
}
port, err := strconv.Atoi(portStr)
if err != nil {
return 0
}
return port
}
// Stop stops the gRPC server
func (s *WorkerGrpcServer) Stop() error {
if !s.running {
return nil
}
s.running = false
close(s.stopChan)
// Close all worker connections. Cancelling the context stops each
// handleOutgoingMessages goroutine; the outgoing channel is never closed.
s.connMutex.Lock()
for _, conn := range s.connections {
conn.cancel()
}
s.connections = make(map[string]*WorkerConnection)
s.connMutex.Unlock()
// Stop gRPC server
if s.grpcServer != nil {
s.grpcServer.GracefulStop()
}
// Close listener
if s.listener != nil {
s.listener.Close()
}
glog.Infof("Worker gRPC server stopped")
return nil
}
// WorkerStream handles bidirectional communication with workers
func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStreamServer) error {
ctx := stream.Context()
// get client address
address := findClientAddress(ctx)
// Wait for initial registration message
msg, err := stream.Recv()
if err != nil {
return fmt.Errorf("failed to receive registration message: %w", err)
}
registration := msg.GetRegistration()
if registration == nil {
return fmt.Errorf("first message must be registration")
}
registration.Address = address
workerID := registration.WorkerId
if workerID == "" {
return fmt.Errorf("worker ID cannot be empty")
}
glog.Infof("Worker %s connecting from %s", workerID, registration.Address)
// Create worker connection
connCtx, connCancel := context.WithCancel(ctx)
conn := &WorkerConnection{
workerID: workerID,
stream: stream,
lastSeen: time.Now(),
address: registration.Address,
maxConcurrent: registration.MaxConcurrent,
outgoing: make(chan *worker_pb.AdminMessage, 100),
ctx: connCtx,
cancel: connCancel,
}
// Convert capabilities
capabilities := make([]MaintenanceTaskType, len(registration.Capabilities))
for i, cap := range registration.Capabilities {
capabilities[i] = MaintenanceTaskType(cap)
}
conn.capabilities = capabilities
// Register connection - clean up old connection if worker is reconnecting
s.connMutex.Lock()
if oldConn, exists := s.connections[workerID]; exists {
glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
// Cancel old connection to stop its goroutines. Its handleOutgoingMessages
// exits on the cancelled context; the outgoing channel is never closed.
oldConn.cancel()
}
s.connections[workerID] = conn
s.connMutex.Unlock()
stats_collect.AdminWorkerEventsTotal.WithLabelValues("registered").Inc()
// Register worker with maintenance manager
s.registerWorkerWithManager(conn)
// IMPORTANT: Start outgoing message handler BEFORE sending registration response
// This ensures the handler is ready to process messages and prevents race conditions
// where the worker might send requests before we're ready to respond
go s.handleOutgoingMessages(conn)
// Send registration response (after handler is started)
regResponse := &worker_pb.AdminMessage{
Timestamp: time.Now().Unix(),
Message: &worker_pb.AdminMessage_RegistrationResponse{
RegistrationResponse: &worker_pb.RegistrationResponse{
Success: true,
Message: "Worker registered successfully",
},
},
}
if s.sendToWorker(conn, regResponse, 5*time.Second, "registration response") {
glog.V(1).Infof("Registration response sent to worker %s", workerID)
}
// Handle incoming messages
for {
select {
case <-ctx.Done():
glog.Infof("Worker %s connection closed: %v", workerID, ctx.Err())
s.unregisterWorker(conn, "unregistered")
return nil
case <-connCtx.Done():
glog.Infof("Worker %s connection cancelled", workerID)
s.unregisterWorker(conn, "unregistered")
return nil
default:
}
msg, err := stream.Recv()
if err != nil {
switch {
case err == io.EOF:
glog.Infof("Worker %s disconnected", workerID)
case errors.Is(err, context.Canceled), status.Code(err) == codes.Canceled:
// Graceful shutdown on either side cancels the stream.
glog.V(1).Infof("Worker %s stream canceled: %v", workerID, err)
default:
glog.Errorf("Error receiving from worker %s: %v", workerID, err)
}
s.unregisterWorker(conn, "unregistered")
return err
}
s.connMutex.Lock()
conn.lastSeen = time.Now()
s.connMutex.Unlock()
s.handleWorkerMessage(conn, msg)
}
}
// handleOutgoingMessages sends messages to worker
func (s *WorkerGrpcServer) handleOutgoingMessages(conn *WorkerConnection) {
for {
select {
case <-conn.ctx.Done():
return
case msg, ok := <-conn.outgoing:
if !ok {
return
}
if err := conn.stream.Send(msg); err != nil {
glog.Errorf("Failed to send message to worker %s: %v", conn.workerID, err)
conn.cancel()
return
}
}
}
}
// handleWorkerMessage processes incoming messages from workers
func (s *WorkerGrpcServer) handleWorkerMessage(conn *WorkerConnection, msg *worker_pb.WorkerMessage) {
workerID := conn.workerID
switch m := msg.Message.(type) {
case *worker_pb.WorkerMessage_Heartbeat:
s.handleHeartbeat(conn, m.Heartbeat)
case *worker_pb.WorkerMessage_TaskRequest:
s.handleTaskRequest(conn, m.TaskRequest)
case *worker_pb.WorkerMessage_TaskUpdate:
s.handleTaskUpdate(conn, m.TaskUpdate)
case *worker_pb.WorkerMessage_TaskComplete:
s.handleTaskCompletion(conn, m.TaskComplete)
case *worker_pb.WorkerMessage_TaskLogResponse:
s.handleTaskLogResponse(conn, m.TaskLogResponse)
case *worker_pb.WorkerMessage_Shutdown:
glog.Infof("Worker %s shutting down: %s", workerID, m.Shutdown.Reason)
s.unregisterWorker(conn, "unregistered")
default:
glog.Warningf("Unknown message type from worker %s", workerID)
}
}
// registerWorkerWithManager registers the worker with the maintenance manager
func (s *WorkerGrpcServer) registerWorkerWithManager(conn *WorkerConnection) {
if s.adminServer.maintenanceManager == nil {
return
}
worker := &MaintenanceWorker{
ID: conn.workerID,
Address: conn.address,
LastHeartbeat: time.Now(),
Status: "active",
Capabilities: conn.capabilities,
MaxConcurrent: int(conn.maxConcurrent),
CurrentLoad: 0,
}
s.adminServer.maintenanceManager.RegisterWorker(worker)
glog.V(1).Infof("Registered worker %s with maintenance manager", conn.workerID)
}
// handleHeartbeat processes heartbeat messages
func (s *WorkerGrpcServer) handleHeartbeat(conn *WorkerConnection, heartbeat *worker_pb.WorkerHeartbeat) {
if s.adminServer.maintenanceManager != nil {
s.adminServer.maintenanceManager.UpdateWorkerHeartbeat(conn.workerID)
}
// Send heartbeat response
response := &worker_pb.AdminMessage{
Timestamp: time.Now().Unix(),
Message: &worker_pb.AdminMessage_HeartbeatResponse{
HeartbeatResponse: &worker_pb.HeartbeatResponse{
Success: true,
Message: "Heartbeat acknowledged",
},
},
}
s.sendToWorker(conn, response, time.Second, "heartbeat response")
}
// handleTaskRequest processes task requests from workers
func (s *WorkerGrpcServer) handleTaskRequest(conn *WorkerConnection, request *worker_pb.TaskRequest) {
if s.adminServer.maintenanceManager == nil {
return
}
// Get next task from maintenance manager
task := s.adminServer.maintenanceManager.GetNextTask(conn.workerID, conn.capabilities)
if task != nil {
// Use typed params directly - master client should already be configured in the params
var taskParams *worker_pb.TaskParams
if task.TypedParams != nil {
taskParams = task.TypedParams
} else {
// Create basic params if none exist
taskParams = &worker_pb.TaskParams{
VolumeId: task.VolumeID,
Collection: task.Collection,
Sources: []*worker_pb.TaskSource{
{
Node: task.Server,
VolumeId: task.VolumeID,
},
},
}
}
// Send task assignment
assignment := &worker_pb.AdminMessage{
Timestamp: time.Now().Unix(),
Message: &worker_pb.AdminMessage_TaskAssignment{
TaskAssignment: &worker_pb.TaskAssignment{
TaskId: task.ID,
TaskType: string(task.Type),
Params: taskParams,
Priority: int32(task.Priority),
CreatedTime: time.Now().Unix(),
},
},
}
s.sendToWorker(conn, assignment, time.Second, "task assignment")
} else {
// Send explicit "No Task" response to prevent worker timeout
// Workers expect a TaskAssignment message but will sleep if TaskId is empty
noTaskAssignment := &worker_pb.AdminMessage{
Timestamp: time.Now().Unix(),
Message: &worker_pb.AdminMessage_TaskAssignment{
TaskAssignment: &worker_pb.TaskAssignment{
TaskId: "", // Empty TaskId indicates no task available
},
},
}
if s.sendToWorker(conn, noTaskAssignment, time.Second, "no-task response") {
glog.V(4).Infof("Sent 'No Task' response to worker %s", conn.workerID)
}
}
}
// handleTaskUpdate processes task progress updates
func (s *WorkerGrpcServer) handleTaskUpdate(conn *WorkerConnection, update *worker_pb.TaskUpdate) {
if s.adminServer.maintenanceManager != nil {
s.adminServer.maintenanceManager.UpdateTaskProgress(update.TaskId, float64(update.Progress))
glog.V(3).Infof("Updated task %s progress: %.1f%%", update.TaskId, update.Progress)
}
}
// handleTaskCompletion processes task completion notifications
func (s *WorkerGrpcServer) handleTaskCompletion(conn *WorkerConnection, completion *worker_pb.TaskComplete) {
if s.adminServer.maintenanceManager != nil {
errorMsg := ""
if !completion.Success {
errorMsg = completion.ErrorMessage
}
s.adminServer.maintenanceManager.CompleteTask(completion.TaskId, errorMsg)
if completion.Success {
glog.V(1).Infof("Worker %s completed task %s successfully", conn.workerID, completion.TaskId)
} else {
glog.Errorf("Worker %s failed task %s: %s", conn.workerID, completion.TaskId, completion.ErrorMessage)
}
// Fetch and persist logs
go s.FetchAndSaveLogs(conn.workerID, completion.TaskId)
}
}
// FetchAndSaveLogs retrieves logs from a worker and saves them to disk
func (s *WorkerGrpcServer) FetchAndSaveLogs(workerID, taskID string) error {
// Add a small initial delay to allow worker to finalize and sync logs
// especially when this is called immediately after TaskComplete
time.Sleep(300 * time.Millisecond)
var workerLogs []*worker_pb.TaskLogEntry
var err error
// Retry a few times if fetch fails, as logs might be in the middle of a terminal sync
for attempt := 1; attempt <= 3; attempt++ {
workerLogs, err = s.RequestTaskLogs(workerID, taskID, maxLogFetchLimit, "")
if err == nil {
break
}
if attempt < 3 {
glog.V(1).Infof("Fetch logs attempt %d failed for task %s: %v. Retrying in 1s...", attempt, taskID, err)
time.Sleep(1 * time.Second)
}
}
if err != nil {
glog.Warningf("Failed to fetch logs for task %s after 3 attempts: %v", taskID, err)
return err
}
// Convert logs
var maintenanceLogs []*maintenance.TaskExecutionLog
for _, workerLog := range workerLogs {
maintenanceLog := &maintenance.TaskExecutionLog{
Timestamp: time.Unix(workerLog.Timestamp, 0),
Level: workerLog.Level,
Message: workerLog.Message,
Source: "worker",
TaskID: taskID,
WorkerID: workerID,
}
// Truncate very long messages to prevent rendering issues and disk bloat
if len(maintenanceLog.Message) > maxLogMessageSize {
maintenanceLog.Message = maintenanceLog.Message[:maxLogMessageSize] + "... (truncated)"
}
// carry structured fields if present
if len(workerLog.Fields) > 0 {
maintenanceLog.Fields = make(map[string]string)
fieldCount := 0
for k, v := range workerLog.Fields {
if fieldCount >= maxLogFieldsCount {
maintenanceLog.Fields["..."] = fmt.Sprintf("(%d more fields truncated)", len(workerLog.Fields)-maxLogFieldsCount)
break
}
maintenanceLog.Fields[k] = v
fieldCount++
}
}
// carry optional progress/status
if workerLog.Progress != 0 {
p := float64(workerLog.Progress)
maintenanceLog.Progress = &p
}
if workerLog.Status != "" {
maintenanceLog.Status = workerLog.Status
}
maintenanceLogs = append(maintenanceLogs, maintenanceLog)
}
// Persist logs
if s.adminServer.configPersistence != nil {
if err := s.adminServer.configPersistence.SaveTaskExecutionLogs(taskID, maintenanceLogs); err != nil {
glog.Errorf("Failed to persist logs for task %s: %v", taskID, err)
return err
}
}
return nil
}
// handleTaskLogResponse processes task log responses from workers
func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, response *worker_pb.TaskLogResponse) {
requestKey := fmt.Sprintf("%s:%s", response.WorkerId, response.TaskId)
s.logRequestsMutex.RLock()
requestContext, exists := s.pendingLogRequests[requestKey]
s.logRequestsMutex.RUnlock()
if !exists {
glog.Warningf("Received unexpected log response for task %s from worker %s", response.TaskId, response.WorkerId)
return
}
glog.V(1).Infof("Received log response for task %s from worker %s: %d entries", response.TaskId, response.WorkerId, len(response.LogEntries))
// Send response to waiting channel
select {
case requestContext.ResponseCh <- response:
// Response delivered successfully
case <-time.After(time.Second):
glog.Warningf("Failed to deliver log response for task %s from worker %s: timeout", response.TaskId, response.WorkerId)
}
// Clean up the pending request
s.logRequestsMutex.Lock()
delete(s.pendingLogRequests, requestKey)
s.logRequestsMutex.Unlock()
}
// sendToWorker queues a message on conn.outgoing, which is deliberately never
// closed: it has multiple concurrent senders, so closing it could panic one of
// them with "send on closed channel". Teardown is signaled via conn.ctx instead;
// handleOutgoingMessages drains the channel until that context is cancelled.
// Returns false if the connection closed or the send timed out.
func (s *WorkerGrpcServer) sendToWorker(conn *WorkerConnection, msg *worker_pb.AdminMessage, timeout time.Duration, description string) bool {
select {
case conn.outgoing <- msg:
return true
case <-conn.ctx.Done():
glog.V(2).Infof("Dropped %s for worker %s: connection closed", description, conn.workerID)
return false
case <-time.After(timeout):
glog.Warningf("Failed to send %s to worker %s: timeout", description, conn.workerID)
return false
}
}
// unregisterWorker removes a worker connection
func (s *WorkerGrpcServer) unregisterWorker(conn *WorkerConnection, event string) {
s.connMutex.Lock()
existingConn, exists := s.connections[conn.workerID]
if !exists {
s.connMutex.Unlock()
glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", conn.workerID)
return
}
// Only remove if it matches the specific connection instance
if existingConn != conn {
s.connMutex.Unlock()
glog.V(1).Infof("unregisterWorker: worker %s connection replaced, skipping unregister for old connection", conn.workerID)
return
}
// Remove from map first to prevent duplicate cleanup attempts
delete(s.connections, conn.workerID)
s.connMutex.Unlock()
stats_collect.AdminWorkerEventsTotal.WithLabelValues(event).Inc()
// Cancel context to signal goroutines to stop. The outgoing channel is
// never closed (it has multiple senders); handleOutgoingMessages exits on
// the cancelled context.
conn.cancel()
glog.V(1).Infof("Unregistered worker %s", conn.workerID)
}
// cleanupRoutine periodically cleans up stale connections
func (s *WorkerGrpcServer) cleanupRoutine() {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-s.stopChan:
return
case <-ticker.C:
s.cleanupStaleConnections()
}
}
}
// cleanupStaleConnections removes connections that haven't been seen recently
func (s *WorkerGrpcServer) cleanupStaleConnections() {
cutoff := time.Now().Add(-2 * time.Minute)
s.connMutex.Lock()
// collect connections to remove first to avoid deadlock if unregisterWorker locks
var toRemove []*WorkerConnection
for _, conn := range s.connections {
if conn.lastSeen.Before(cutoff) {
toRemove = append(toRemove, conn)
}
}
s.connMutex.Unlock()
for _, conn := range toRemove {
glog.Warningf("Cleaning up stale worker connection: %s", conn.workerID)
s.unregisterWorker(conn, "stale_removed")
}
}
// GetConnectedWorkers returns a list of currently connected workers
func (s *WorkerGrpcServer) GetConnectedWorkers() []string {
s.connMutex.RLock()
defer s.connMutex.RUnlock()
workers := make([]string, 0, len(s.connections))
for workerID := range s.connections {
workers = append(workers, workerID)
}
return workers
}
// RequestTaskLogs requests execution logs from a worker for a specific task
func (s *WorkerGrpcServer) RequestTaskLogs(workerID, taskID string, maxEntries int32, logLevel string) ([]*worker_pb.TaskLogEntry, error) {
s.connMutex.RLock()
conn, exists := s.connections[workerID]
s.connMutex.RUnlock()
if !exists {
return nil, fmt.Errorf("worker %s is not connected", workerID)
}
// Create response channel for this request
responseCh := make(chan *worker_pb.TaskLogResponse, 1)
requestKey := fmt.Sprintf("%s:%s", workerID, taskID)
// Register pending request
requestContext := &LogRequestContext{
TaskID: taskID,
WorkerID: workerID,
ResponseCh: responseCh,
}
s.logRequestsMutex.Lock()
if _, exists := s.pendingLogRequests[requestKey]; exists {
s.logRequestsMutex.Unlock()
return nil, fmt.Errorf("a log request for task %s is already in progress", taskID)
}
s.pendingLogRequests[requestKey] = requestContext
s.logRequestsMutex.Unlock()
// Create log request message
logRequest := &worker_pb.AdminMessage{
AdminId: "admin-server",
Timestamp: time.Now().Unix(),
Message: &worker_pb.AdminMessage_TaskLogRequest{
TaskLogRequest: &worker_pb.TaskLogRequest{
TaskId: taskID,
WorkerId: workerID,
IncludeMetadata: true,
MaxEntries: maxEntries,
LogLevel: logLevel,
},
},
}
// Send the request through the worker's outgoing channel
select {
case conn.outgoing <- logRequest:
glog.V(1).Infof("Log request sent to worker %s for task %s", workerID, taskID)
case <-conn.ctx.Done():
s.logRequestsMutex.Lock()
if s.pendingLogRequests[requestKey] == requestContext {
delete(s.pendingLogRequests, requestKey)
}
s.logRequestsMutex.Unlock()
return nil, fmt.Errorf("worker %s connection closed", workerID)
case <-time.After(logSendTimeout):
// Clean up pending request on timeout
s.logRequestsMutex.Lock()
if s.pendingLogRequests[requestKey] == requestContext {
delete(s.pendingLogRequests, requestKey)
}
s.logRequestsMutex.Unlock()
return nil, fmt.Errorf("timeout sending log request to worker %s", workerID)
}
// Wait for response
select {
case response := <-responseCh:
if !response.Success {
return nil, fmt.Errorf("worker log request failed: %s", response.ErrorMessage)
}
glog.V(1).Infof("Received %d log entries for task %s from worker %s", len(response.LogEntries), taskID, workerID)
return response.LogEntries, nil
case <-conn.ctx.Done():
s.logRequestsMutex.Lock()
if s.pendingLogRequests[requestKey] == requestContext {
delete(s.pendingLogRequests, requestKey)
}
s.logRequestsMutex.Unlock()
return nil, fmt.Errorf("worker %s connection closed", workerID)
case <-time.After(logResponseTimeout):
// Clean up pending request on timeout
s.logRequestsMutex.Lock()
if s.pendingLogRequests[requestKey] == requestContext {
delete(s.pendingLogRequests, requestKey)
}
s.logRequestsMutex.Unlock()
return nil, fmt.Errorf("timeout waiting for log response from worker %s", workerID)
}
}
// RequestTaskLogsFromAllWorkers requests logs for a task from all connected workers
func (s *WorkerGrpcServer) RequestTaskLogsFromAllWorkers(taskID string, maxEntries int32, logLevel string) (map[string][]*worker_pb.TaskLogEntry, error) {
s.connMutex.RLock()
workerIDs := make([]string, 0, len(s.connections))
for workerID := range s.connections {
workerIDs = append(workerIDs, workerID)
}
s.connMutex.RUnlock()
results := make(map[string][]*worker_pb.TaskLogEntry)
for _, workerID := range workerIDs {
logs, err := s.RequestTaskLogs(workerID, taskID, maxEntries, logLevel)
if err != nil {
glog.V(1).Infof("Failed to get logs from worker %s for task %s: %v", workerID, taskID, err)
// Store empty result with error information for debugging
results[workerID+"_error"] = []*worker_pb.TaskLogEntry{
{
Timestamp: time.Now().Unix(),
Level: "ERROR",
Message: fmt.Sprintf("Failed to retrieve logs from worker %s: %v", workerID, err),
Fields: map[string]string{"source": "admin"},
},
}
continue
}
if len(logs) > 0 {
results[workerID] = logs
} else {
glog.V(2).Infof("No logs found for task %s on worker %s", taskID, workerID)
}
}
return results, nil
}
func findClientAddress(ctx context.Context) string {
// fmt.Printf("FromContext %+v\n", ctx)
pr, ok := peer.FromContext(ctx)
if !ok {
glog.Error("failed to get peer from ctx")
return ""
}
if pr.Addr == net.Addr(nil) {
glog.Error("failed to get peer address")
return ""
}
return pr.Addr.String()
}
// activeLogFetchLoop periodically fetches logs for all in-progress tasks
func (s *WorkerGrpcServer) activeLogFetchLoop() {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-s.stopChan:
return
case <-ticker.C:
if !s.running || s.adminServer == nil || s.adminServer.maintenanceManager == nil {
continue
}
// Get all in-progress tasks
tasks := s.adminServer.maintenanceManager.GetTasks(maintenance.TaskStatusInProgress, "", 0)
if len(tasks) == 0 {
continue
}
glog.V(2).Infof("Background log fetcher: found %d in-progress tasks", len(tasks))
for _, task := range tasks {
if task.WorkerID != "" {
// Use a goroutine to avoid blocking the loop
go func(wID, tID string) {
if err := s.FetchAndSaveLogs(wID, tID); err != nil {
glog.V(2).Infof("Background log fetch failed for task %s on worker %s: %v", tID, wID, err)
}
}(task.WorkerID, task.ID)
}
}
}
}
}

View File

@@ -0,0 +1,81 @@
package dash
import (
"context"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
)
func newTestWorkerConn(bufferSize int) *WorkerConnection {
ctx, cancel := context.WithCancel(context.Background())
return &WorkerConnection{
workerID: "w-test",
outgoing: make(chan *worker_pb.AdminMessage, bufferSize),
ctx: ctx,
cancel: cancel,
}
}
// TestSendToWorkerDropsOnCancelledContext verifies a send bails out via the
// connection context instead of blocking for the full timeout once the
// connection is torn down and its buffer is full.
func TestSendToWorkerDropsOnCancelledContext(t *testing.T) {
s := &WorkerGrpcServer{adminServer: &AdminServer{}}
conn := newTestWorkerConn(1)
// Fill the buffer so the next send cannot enqueue.
conn.outgoing <- &worker_pb.AdminMessage{}
conn.cancel()
start := time.Now()
if s.sendToWorker(conn, &worker_pb.AdminMessage{}, 5*time.Second, "heartbeat response") {
t.Fatal("expected send to be dropped when context cancelled and buffer full")
}
if elapsed := time.Since(start); elapsed > time.Second {
t.Fatalf("send blocked for %v; expected an immediate context-cancelled bail", elapsed)
}
}
// TestHeartbeatDuringUnregisterDoesNotPanic is a regression test for the
// "panic: send on closed channel" crash: unregisterWorker (called from the
// stale-connection cleanup routine) used to close conn.outgoing while the
// WorkerStream goroutine was still sending heartbeat responses to it.
func TestHeartbeatDuringUnregisterDoesNotPanic(t *testing.T) {
s := &WorkerGrpcServer{
adminServer: &AdminServer{},
connections: make(map[string]*WorkerConnection),
}
conn := newTestWorkerConn(100)
s.connections[conn.workerID] = conn
// Drain outgoing like handleOutgoingMessages would, until teardown.
go func() {
for {
select {
case <-conn.ctx.Done():
return
case <-conn.outgoing:
}
}
}()
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for i := 0; i < 2000; i++ {
s.handleHeartbeat(conn, &worker_pb.WorkerHeartbeat{})
}
}()
go func() {
defer wg.Done()
s.unregisterWorker(conn, "test")
}()
// Under the old close()-based teardown this would panic the sender goroutine
// and crash the test binary.
wg.Wait()
}