snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/shell/command.go

@@ -0,0 +1,21 @@
+package shell
+
+import "io"
+
+type command interface {
+	Name() string
+	Help() string
+	Do([]string, *CommandEnv, io.Writer) error
+	HasTag(tag CommandTag) bool
+}
+
+var (
+	Commands = []command{}
+)
+
+type CommandTag string
+
+const (
+	ResourceHeavy CommandTag = "resourceHeavy"
+	Hidden        CommandTag = "hidden"
+)

weed/shell/command_cluster_check.go

@@ -0,0 +1,249 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func init() {
+	Commands = append(Commands, &commandClusterCheck{})
+}
+
+type commandClusterCheck struct {
+}
+
+func (c *commandClusterCheck) Name() string {
+	return "cluster.check"
+}
+
+func (c *commandClusterCheck) Help() string {
+	return `check current cluster network connectivity
+
+	cluster.check
+
+`
+}
+
+func (c *commandClusterCheck) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = clusterPsCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	// collect topology information
+	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(topologyInfo.DiskInfos))
+
+	if len(topologyInfo.DiskInfos) == 0 {
+		return fmt.Errorf("no disk type defined")
+	}
+	for diskType, diskInfo := range topologyInfo.DiskInfos {
+		if diskInfo.MaxVolumeCount == 0 {
+			return fmt.Errorf("no volume available for \"%s\" disk type", diskType)
+		}
+	}
+
+	// collect filers
+	var filers []pb.ServerAddress
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+			ClientType: cluster.FilerType,
+			FilerGroup: *commandEnv.option.FilerGroup,
+		})
+
+		for _, node := range resp.ClusterNodes {
+			filers = append(filers, pb.ServerAddress(node.Address))
+		}
+		return err
+	})
+	if err != nil {
+		return
+	}
+	fmt.Fprintf(writer, "the cluster has %d filers: %+v\n", len(filers), filers)
+
+	if len(filers) > 0 {
+		genericDiskInfo, genericDiskInfoOk := topologyInfo.DiskInfos[""]
+		hddDiskInfo, hddDiskInfoOk := topologyInfo.DiskInfos[types.HddType]
+
+		if !genericDiskInfoOk && !hddDiskInfoOk {
+			return fmt.Errorf("filer metadata logs need generic or hdd disk type to be defined")
+		}
+
+		if (genericDiskInfoOk && genericDiskInfo.MaxVolumeCount == 0) || (hddDiskInfoOk && hddDiskInfo.MaxVolumeCount == 0) {
+			return fmt.Errorf("filer metadata logs need generic or hdd volumes to be available")
+		}
+	}
+
+	// collect volume servers
+	var volumeServers []pb.ServerAddress
+	t, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	for _, dc := range t.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn))
+			}
+		}
+	}
+	fmt.Fprintf(writer, "the cluster has %d volume servers: %+v\n", len(volumeServers), volumeServers)
+
+	// collect all masters
+	var masters []pb.ServerAddress
+	masters = append(masters, commandEnv.MasterClient.GetMasters(context.Background())...)
+
+	// check from master to volume servers
+	for _, master := range masters {
+		for _, volumeServer := range volumeServers {
+			fmt.Fprintf(writer, "checking master %s to volume server %s ... ", string(master), string(volumeServer))
+			err := pb.WithMasterClient(context.Background(), false, master, commandEnv.option.GrpcDialOption, false, func(client master_pb.SeaweedClient) error {
+				pong, err := client.Ping(context.Background(), &master_pb.PingRequest{
+					Target:     string(volumeServer),
+					TargetType: cluster.VolumeServerType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	// check between masters
+	for _, sourceMaster := range masters {
+		for _, targetMaster := range masters {
+			if sourceMaster == targetMaster {
+				continue
+			}
+			fmt.Fprintf(writer, "checking master %s to %s ... ", string(sourceMaster), string(targetMaster))
+			err := pb.WithMasterClient(context.Background(), false, sourceMaster, commandEnv.option.GrpcDialOption, false, func(client master_pb.SeaweedClient) error {
+				pong, err := client.Ping(context.Background(), &master_pb.PingRequest{
+					Target:     string(targetMaster),
+					TargetType: cluster.MasterType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	// check from volume servers to masters
+	for _, volumeServer := range volumeServers {
+		for _, master := range masters {
+			fmt.Fprintf(writer, "checking volume server %s to master %s ... ", string(volumeServer), string(master))
+			err := pb.WithVolumeServerClient(false, volumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+				pong, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
+					Target:     string(master),
+					TargetType: cluster.MasterType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	// check from filers to masters
+	for _, filer := range filers {
+		for _, master := range masters {
+			fmt.Fprintf(writer, "checking filer %s to master %s ... ", string(filer), string(master))
+			err := pb.WithFilerClient(false, 0, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+					Target:     string(master),
+					TargetType: cluster.MasterType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	// check from filers to volume servers
+	for _, filer := range filers {
+		for _, volumeServer := range volumeServers {
+			fmt.Fprintf(writer, "checking filer %s to volume server %s ... ", string(filer), string(volumeServer))
+			err := pb.WithFilerClient(false, 0, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+					Target:     string(volumeServer),
+					TargetType: cluster.VolumeServerType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	// Direct volume-to-volume connectivity is intentionally not validated
+	// here. Each volume server now restricts Ping to peers it can identify
+	// (its configured/current masters), so it does not carry a peer-volume
+	// list to drive a mesh check from. The master->volume and filer->volume
+	// probes above do not exercise volume-to-volume reachability.
+
+	// check between filers, and need to connect to itself
+	for _, sourceFiler := range filers {
+		for _, targetFiler := range filers {
+			fmt.Fprintf(writer, "checking filer %s to %s ... ", string(sourceFiler), string(targetFiler))
+			err := pb.WithFilerClient(false, 0, sourceFiler, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				pong, err := client.Ping(context.Background(), &filer_pb.PingRequest{
+					Target:     string(targetFiler),
+					TargetType: cluster.FilerType,
+				})
+				if err == nil {
+					printTiming(writer, pong.StartTimeNs, pong.RemoteTimeNs, pong.StopTimeNs)
+				}
+				return err
+			})
+			if err != nil {
+				fmt.Fprintf(writer, "%v\n", err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func printTiming(writer io.Writer, startNs, remoteNs, stopNs int64) {
+	roundTripTimeMs := float32(stopNs-startNs) / 1000000
+	deltaTimeMs := float32(remoteNs-(startNs+stopNs)/2) / 1000000
+	fmt.Fprintf(writer, "ok round trip %.3fms clock delta %.3fms\n", roundTripTimeMs, deltaTimeMs)
+}

weed/shell/command_cluster_ps.go

@@ -0,0 +1,178 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/cluster"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandClusterPs{})
+}
+
+type commandClusterPs struct {
+}
+
+func (c *commandClusterPs) Name() string {
+	return "cluster.ps"
+}
+
+func (c *commandClusterPs) Help() string {
+	return `check current cluster process status
+
+	cluster.ps
+
+`
+}
+
+func (c *commandClusterPs) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = clusterPsCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	listClusterNodes := func(clientType string) ([]*master_pb.ListClusterNodesResponse_ClusterNode, error) {
+		var nodes []*master_pb.ListClusterNodesResponse_ClusterNode
+		err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+			resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
+				ClientType: clientType,
+				FilerGroup: *commandEnv.option.FilerGroup,
+			})
+			if err != nil {
+				return err
+			}
+			nodes = resp.ClusterNodes
+			return nil
+		})
+		return nodes, err
+	}
+
+	filerNodes, err := listClusterNodes(cluster.FilerType)
+	if err != nil {
+		return
+	}
+	mqBrokerNodes, err := listClusterNodes(cluster.BrokerType)
+	if err != nil {
+		return
+	}
+	s3Nodes, err := listClusterNodes(cluster.S3Type)
+	if err != nil {
+		return
+	}
+
+	if len(mqBrokerNodes) > 0 {
+		fmt.Fprintf(writer, "* message queue brokers %d\n", len(mqBrokerNodes))
+		for _, node := range mqBrokerNodes {
+			fmt.Fprintf(writer, "  * %s (%v)\n", node.Address, node.Version)
+			if node.DataCenter != "" {
+				fmt.Fprintf(writer, "    DataCenter: %v\n", node.DataCenter)
+			}
+			if node.Rack != "" {
+				fmt.Fprintf(writer, "    Rack: %v\n", node.Rack)
+			}
+		}
+	}
+
+	if len(s3Nodes) > 0 {
+		fmt.Fprintf(writer, "* s3 servers %d\n", len(s3Nodes))
+		for _, node := range s3Nodes {
+			fmt.Fprintf(writer, "  * %s (%v)\n", node.Address, node.Version)
+			if node.DataCenter != "" {
+				fmt.Fprintf(writer, "    DataCenter: %v\n", node.DataCenter)
+			}
+			if node.Rack != "" {
+				fmt.Fprintf(writer, "    Rack: %v\n", node.Rack)
+			}
+		}
+	}
+
+	filerSignatures := make(map[*master_pb.ListClusterNodesResponse_ClusterNode]int32)
+	fmt.Fprintf(writer, "* filers %d\n", len(filerNodes))
+	for _, node := range filerNodes {
+		fmt.Fprintf(writer, "  * %s (%v) %v\n", node.Address, node.Version, time.Unix(0, node.CreatedAtNs).UTC())
+		if node.DataCenter != "" {
+			fmt.Fprintf(writer, "    DataCenter: %v\n", node.DataCenter)
+		}
+		if node.Rack != "" {
+			fmt.Fprintf(writer, "    Rack: %v\n", node.Rack)
+		}
+		pb.WithFilerClient(false, 0, pb.ServerAddress(node.Address), commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			if err == nil {
+				if resp.FilerGroup != "" {
+					fmt.Fprintf(writer, "    filer group: %s\n", resp.FilerGroup)
+				}
+				fmt.Fprintf(writer, "    signature: %d\n", resp.Signature)
+				filerSignatures[node] = resp.Signature
+			} else {
+				fmt.Fprintf(writer, "    failed to connect: %v\n", err)
+			}
+			return err
+		})
+	}
+	for _, node := range filerNodes {
+		pb.WithFilerClient(false, 0, pb.ServerAddress(node.Address), commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+			fmt.Fprintf(writer, "* filer %s metadata sync time\n", node.Address)
+			selfSignature := filerSignatures[node]
+			for peer, peerSignature := range filerSignatures {
+				if selfSignature == peerSignature {
+					continue
+				}
+				if resp, err := client.KvGet(context.Background(), &filer_pb.KvGetRequest{Key: filer.GetPeerMetaOffsetKey(peerSignature)}); err == nil && len(resp.Value) == 8 {
+					lastTsNs := int64(util.BytesToUint64(resp.Value))
+					fmt.Fprintf(writer, "    %s: %v\n", peer.Address, time.Unix(0, lastTsNs).UTC())
+				}
+			}
+			return nil
+		})
+	}
+
+	// collect volume servers
+	var volumeServers []pb.ServerAddress
+	t, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	for _, dc := range t.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn))
+			}
+		}
+	}
+
+	fmt.Fprintf(writer, "* volume servers %d\n", len(volumeServers))
+	for _, dc := range t.DataCenterInfos {
+		fmt.Fprintf(writer, "  * data center: %s\n", dc.Id)
+		for _, r := range dc.RackInfos {
+			fmt.Fprintf(writer, "    * rack: %s\n", r.Id)
+			for _, dn := range r.DataNodeInfos {
+				pb.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dn), commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+					resp, err := client.VolumeServerStatus(context.Background(), &volume_server_pb.VolumeServerStatusRequest{})
+					if err == nil {
+						fmt.Fprintf(writer, "      * %s (%v)\n", dn.Id, resp.Version)
+					}
+					return err
+				})
+			}
+		}
+	}
+
+	return nil
+}

weed/shell/command_cluster_raft_add.go

@@ -0,0 +1,64 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandRaftServerAdd{})
+}
+
+type commandRaftServerAdd struct {
+}
+
+func (c *commandRaftServerAdd) Name() string {
+	return "cluster.raft.add"
+}
+
+func (c *commandRaftServerAdd) Help() string {
+	return `add a server to the raft cluster
+
+	Example:
+		cluster.raft.add -id <server_name> -address <server_host:port> -voter
+`
+}
+
+func (c *commandRaftServerAdd) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRaftServerAdd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	serverId := raftServerAddCommand.String("id", "", "server id")
+	serverAddress := raftServerAddCommand.String("address", "", "server grpc address")
+	serverVoter := raftServerAddCommand.Bool("voter", true, "assign it a vote")
+	if err = raftServerAddCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *serverId == "" || *serverAddress == "" {
+		return fmt.Errorf("empty server id or address")
+	}
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		_, err := client.RaftAddServer(context.Background(), &master_pb.RaftAddServerRequest{
+			Id:      *serverId,
+			Address: *serverAddress,
+			Voter:   *serverVoter,
+		})
+		if err != nil {
+			return fmt.Errorf("raft add server: %w", err)
+		}
+		println("added server", *serverId)
+		return nil
+	})
+
+	return err
+
+}

weed/shell/command_cluster_raft_leader_transfer.go

@@ -0,0 +1,143 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandRaftLeaderTransfer{})
+}
+
+type commandRaftLeaderTransfer struct{}
+
+func (c *commandRaftLeaderTransfer) Name() string {
+	return "cluster.raft.transferLeader"
+}
+
+func (c *commandRaftLeaderTransfer) Help() string {
+	return `transfer raft leadership to another master server
+
+	This command initiates a graceful leadership transfer from the current
+	leader to another server. Use this before performing maintenance on
+	the current leader to reduce errors in filers and other components.
+
+	Examples:
+		# Transfer to any eligible follower (auto-selection)
+		cluster.raft.transferLeader
+
+		# Transfer to a specific server
+		cluster.raft.transferLeader -id <server_id> -address <server_grpc_address>
+
+	Notes:
+		- Requires hashicorp raft (-raftHashicorp=true on master)
+		- This command must be sent to the current leader
+		- The target server must be a voting member of the raft cluster
+		- Use 'cluster.raft.ps' to list available servers and identify the leader
+`
+}
+
+func (c *commandRaftLeaderTransfer) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRaftLeaderTransfer) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	leaderTransferCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	targetId := leaderTransferCommand.String("id", "", "target server id (must be used with -address)")
+	targetAddress := leaderTransferCommand.String("address", "", "target server grpc address (must be used with -id)")
+
+	if err := leaderTransferCommand.Parse(args); err != nil {
+		return err
+	}
+
+	// Validate: id and address must be specified together
+	if *targetId != "" && *targetAddress == "" {
+		return fmt.Errorf("-address is required when -id is specified")
+	}
+	if *targetAddress != "" && *targetId == "" {
+		return fmt.Errorf("-id is required when -address is specified")
+	}
+
+	// First, show current cluster status
+	fmt.Fprintf(writer, "Checking current raft cluster status...\n")
+
+	var currentLeader string
+	err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		resp, err := client.RaftListClusterServers(ctx, &master_pb.RaftListClusterServersRequest{})
+		if err != nil {
+			return fmt.Errorf("failed to list cluster servers: %v", err)
+		}
+
+		if len(resp.ClusterServers) == 0 {
+			fmt.Fprintf(writer, "No raft cluster configured (single master mode)\n")
+			return fmt.Errorf("leadership transfer not available in single master mode")
+		}
+
+		fmt.Fprintf(writer, "Raft cluster has %d servers:\n", len(resp.ClusterServers))
+		for _, server := range resp.ClusterServers {
+			suffix := ""
+			if server.IsLeader {
+				suffix = " <- current leader"
+				currentLeader = server.Id
+			}
+			fmt.Fprintf(writer, "  %s %s [%s]%s\n", server.Id, server.Address, server.Suffrage, suffix)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	if currentLeader == "" {
+		return fmt.Errorf("no leader found in cluster")
+	}
+
+	// Perform the transfer
+	targetDesc := "any eligible follower"
+	if *targetId != "" {
+		targetDesc = fmt.Sprintf("server %s (%s)", *targetId, *targetAddress)
+	}
+	fmt.Fprintf(writer, "\nTransferring leadership from %s to %s...\n", currentLeader, targetDesc)
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), true, func(client master_pb.SeaweedClient) error {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		resp, err := client.RaftLeadershipTransfer(ctx, &master_pb.RaftLeadershipTransferRequest{
+			TargetId:      *targetId,
+			TargetAddress: *targetAddress,
+		})
+		if err != nil {
+			return fmt.Errorf("leadership transfer failed: %v", err)
+		}
+
+		if resp.PreviousLeader != resp.NewLeader {
+			fmt.Fprintf(writer, "Leadership successfully transferred.\n")
+			fmt.Fprintf(writer, "  Previous leader: %s\n", resp.PreviousLeader)
+			fmt.Fprintf(writer, "  New leader: %s\n", resp.NewLeader)
+		} else {
+			fmt.Fprintf(writer, "Leadership transfer initiated, but the same leader was re-elected.\n")
+			fmt.Fprintf(writer, "  Current leader: %s\n", resp.NewLeader)
+		}
+		return nil
+	})
+
+	if err != nil {
+		fmt.Fprintf(writer, "\nLeadership transfer failed: %v\n", err)
+		fmt.Fprintf(writer, "\nTroubleshooting:\n")
+		fmt.Fprintf(writer, "  - Ensure you are connected to the current leader\n")
+		fmt.Fprintf(writer, "  - Ensure target server is a voting member (use 'cluster.raft.ps')\n")
+		fmt.Fprintf(writer, "  - Ensure target server is healthy and reachable\n")
+		return err
+	}
+
+	return nil
+}

weed/shell/command_cluster_raft_leader_transfer_test.go

@@ -0,0 +1,88 @@
+package shell
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestRaftLeaderTransfer_Name(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	expected := "cluster.raft.transferLeader"
+	if cmd.Name() != expected {
+		t.Errorf("expected name %q, got %q", expected, cmd.Name())
+	}
+}
+
+func TestRaftLeaderTransfer_Help(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	help := cmd.Help()
+
+	// Verify help text contains key information
+	expectedPhrases := []string{
+		"transfer raft leadership",
+		"cluster.raft.transferLeader",
+		"-id",
+		"-address",
+		"cluster.raft.ps",
+		"-raftHashicorp",
+	}
+
+	for _, phrase := range expectedPhrases {
+		if !strings.Contains(help, phrase) {
+			t.Errorf("help text should contain %q", phrase)
+		}
+	}
+}
+
+func TestRaftLeaderTransfer_HasTag(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	// The command should not have any special tags
+	if cmd.HasTag(ResourceHeavy) {
+		t.Error("expected HasTag to return false for ResourceHeavy")
+	}
+}
+
+func TestRaftLeaderTransfer_ValidateTargetIdWithoutAddress(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	var buf bytes.Buffer
+
+	// Create a mock command environment - this will fail because no master client
+	// but we can verify argument parsing
+	err := cmd.Do([]string{"-id", "test-server"}, nil, &buf)
+
+	// Should fail because -address is required when -id is specified
+	if err == nil {
+		t.Error("expected error when -id is specified without -address")
+	}
+	if err != nil && !strings.Contains(err.Error(), "-address is required") {
+		t.Errorf("expected error about missing -address, got: %v", err)
+	}
+}
+
+func TestRaftLeaderTransfer_ValidateTargetAddressWithoutId(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	var buf bytes.Buffer
+
+	// Verify argument parsing - address without id should fail
+	err := cmd.Do([]string{"-address", "localhost:19333"}, nil, &buf)
+
+	// Should fail because -id is required when -address is specified
+	if err == nil {
+		t.Error("expected error when -address is specified without -id")
+	}
+	if err != nil && !strings.Contains(err.Error(), "-id is required") {
+		t.Errorf("expected error about missing -id, got: %v", err)
+	}
+}
+
+func TestRaftLeaderTransfer_UnknownFlag(t *testing.T) {
+	cmd := &commandRaftLeaderTransfer{}
+	var buf bytes.Buffer
+
+	// Unknown flag should return an error
+	err := cmd.Do([]string{"-unknown-flag"}, nil, &buf)
+	if err == nil {
+		t.Error("expected error for unknown flag")
+	}
+}

weed/shell/command_cluster_raft_ps.go

@@ -0,0 +1,60 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandRaftClusterPs{})
+}
+
+type commandRaftClusterPs struct {
+}
+
+func (c *commandRaftClusterPs) Name() string {
+	return "cluster.raft.ps"
+}
+
+func (c *commandRaftClusterPs) Help() string {
+	return `check current raft cluster status
+
+	cluster.raft.ps
+`
+}
+
+func (c *commandRaftClusterPs) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRaftClusterPs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	raftClusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = raftClusterPsCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err := client.RaftListClusterServers(context.Background(), &master_pb.RaftListClusterServersRequest{})
+		if err != nil {
+			return fmt.Errorf("raft list cluster: %w", err)
+		}
+		fmt.Fprintf(writer, "the raft cluster has %d servers\n", len(resp.ClusterServers))
+		for _, server := range resp.ClusterServers {
+			suffrage := server.Suffrage
+			if server.IsLeader {
+				suffrage = "Leader"
+			}
+			fmt.Fprintf(writer, "  * %s %s (%s)\n", server.Id, server.Address, suffrage)
+		}
+
+		return nil
+	})
+
+	return err
+
+}

weed/shell/command_cluster_raft_remove.go

@@ -0,0 +1,61 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandRaftServerRemove{})
+}
+
+type commandRaftServerRemove struct {
+}
+
+func (c *commandRaftServerRemove) Name() string {
+	return "cluster.raft.remove"
+}
+
+func (c *commandRaftServerRemove) Help() string {
+	return `remove a server from the raft cluster
+
+	Example:
+		cluster.raft.remove -id <server_name>
+`
+}
+
+func (c *commandRaftServerRemove) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRaftServerRemove) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	raftServerAddCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	serverId := raftServerAddCommand.String("id", "", "server id")
+	if err = raftServerAddCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *serverId == "" {
+		return fmt.Errorf("empty server id")
+	}
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		_, err := client.RaftRemoveServer(context.Background(), &master_pb.RaftRemoveServerRequest{
+			Id:    *serverId,
+			Force: true,
+		})
+		if err != nil {
+			return fmt.Errorf("raft remove server: %w", err)
+		}
+		println("removed server", *serverId)
+		return nil
+	})
+
+	return err
+
+}

weed/shell/command_cluster_status.go

@@ -0,0 +1,542 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math"
+	"strings"
+	"sync"
+
+	"github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize/english"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+
+	"io"
+)
+
+func init() {
+	Commands = append(Commands, &commandClusterStatus{})
+}
+
+// Map of volume_id -> [volume replicas] with stat details.
+type VolumeReplicaStats struct {
+	Id       string
+	VolumeId uint32
+
+	Files        uint64
+	FilesDeleted uint64
+	TotalSize    uint64
+}
+type RegularVolumesStats map[uint32][]*VolumeReplicaStats
+
+// Map of ec_volume_id -> stat details.
+type EcVolumeStats struct {
+	VolumeId uint32
+
+	Files        uint64
+	FilesDeleted uint64
+	TotalSize    uint64
+}
+type EcVolumesStats map[uint32]*EcVolumeStats
+
+type commandClusterStatus struct{}
+type ClusterStatusPrinter struct {
+	writer             io.Writer
+	writerMu           sync.Mutex
+	humanize           bool
+	maxParallelization int
+
+	locked              bool
+	lockHeld            bool
+	lockHolder          string
+	lockMessage         string
+	collections         []string
+	topology            *master_pb.TopologyInfo
+	volumeSizeLimitMb   uint64
+	regularVolumesStats RegularVolumesStats
+	ecVolumesStats      EcVolumesStats
+}
+
+func (c *commandClusterStatus) Name() string {
+	return "cluster.status"
+}
+
+func (c *commandClusterStatus) Help() string {
+	return `outputs a quick overview of the cluster status`
+}
+
+func (c *commandClusterStatus) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandClusterStatus) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	flags := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	humanize := flags.Bool("humanize", true, "human-readable output")
+	includeFiles := flags.Bool("files", false, "include detailed file metrics, from all volume servers")
+	maxParallelization := flags.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
+
+	if err = flags.Parse(args); err != nil {
+		return err
+	}
+
+	collections, err := ListCollectionNames(commandEnv, true, true)
+	if err != nil {
+		return err
+	}
+	topology, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	lockHolder, lockMessage, lockHeld := commandEnv.shellLockHolder()
+
+	sp := &ClusterStatusPrinter{
+		writer:             writer,
+		humanize:           *humanize,
+		maxParallelization: *maxParallelization,
+
+		locked:            commandEnv.isLocked(),
+		lockHeld:          lockHeld,
+		lockHolder:        lockHolder,
+		lockMessage:       lockMessage,
+		collections:       collections,
+		topology:          topology,
+		volumeSizeLimitMb: volumeSizeLimitMb,
+	}
+	if *includeFiles {
+		if err := sp.loadFileStats(commandEnv); err != nil {
+			return err
+		}
+	}
+
+	sp.Print()
+
+	return nil
+}
+
+func (sp *ClusterStatusPrinter) uint64(n uint64) string {
+	if !sp.humanize {
+		return fmt.Sprintf("%d", n)
+	}
+	return humanize.Comma(int64(n))
+}
+
+func (sp *ClusterStatusPrinter) int(n int) string {
+	return sp.uint64(uint64(n))
+}
+
+func (sp *ClusterStatusPrinter) uint64Plural(n uint64, str string) string {
+	if !sp.humanize {
+		return fmt.Sprintf("%s(s)", str)
+	}
+	uin := math.MaxInt
+	if n < math.MaxInt {
+		uin = int(n)
+	}
+	return english.PluralWord(int(uin), str, "")
+}
+
+func (sp *ClusterStatusPrinter) plural(n int, str string) string {
+	return sp.uint64Plural(uint64(n), str)
+}
+
+func (sp *ClusterStatusPrinter) bytes(b uint64) string {
+	if !sp.humanize {
+		return fmt.Sprintf("%d %s", b, sp.plural(int(b), "byte"))
+	}
+	return fmt.Sprintf("%s", humanize.Bytes(b))
+}
+
+func (sp *ClusterStatusPrinter) uint64Ratio(a, b uint64) string {
+	var p float64
+	if b != 0 {
+		if a%b == 0 {
+			// Avoid float precision issues on integer ratios.
+			p = float64(a / b)
+		} else {
+			p = float64(a) / float64(b)
+		}
+	}
+	if !sp.humanize {
+		return fmt.Sprintf("%.02f", p)
+	}
+	return fmt.Sprintf("%s", humanize.FtoaWithDigits(p, 2))
+}
+
+func (sp *ClusterStatusPrinter) intRatio(a, b int) string {
+	return sp.uint64Ratio(uint64(a), uint64(b))
+}
+
+func (sp *ClusterStatusPrinter) uint64Pct(a, b uint64) string {
+	var p float64
+
+	if b != 0 {
+		if a%b == 0 {
+			// avoid float rounding errors on exact ratios
+			p = float64(a / b * 100)
+		} else {
+			p = 100 * float64(a) / float64(b)
+		}
+	}
+	if !sp.humanize {
+		return fmt.Sprintf("%.02f%%", p)
+	}
+	return fmt.Sprintf("%s%%", humanize.FtoaWithDigits(p, 2))
+}
+
+func (sp *ClusterStatusPrinter) intPct(a, b int) string {
+	return sp.uint64Pct(uint64(a), uint64(b))
+}
+
+func (sp *ClusterStatusPrinter) write(format string, a ...any) {
+	sp.writerMu.Lock()
+	defer sp.writerMu.Unlock()
+
+	format = strings.TrimRight(format, " ")
+	if len(format) == 0 {
+		format = "\n"
+	}
+	fmt.Fprintf(sp.writer, format, a...)
+
+	last := format[len(format)-1:]
+	if last != "\n" && last != "\r" {
+		fmt.Fprint(sp.writer, "\n")
+	}
+}
+
+func (sp *ClusterStatusPrinter) Print() {
+	sp.write("")
+	sp.printClusterInfo()
+	sp.printVolumeInfo()
+	sp.printStorageInfo()
+	sp.printFilesInfo()
+}
+
+func (sp *ClusterStatusPrinter) loadFileStats(commandEnv *CommandEnv) error {
+	sp.regularVolumesStats = RegularVolumesStats{}
+	sp.ecVolumesStats = EcVolumesStats{}
+
+	var statsMu, writerMu sync.Mutex
+	var progressTotal, progressDone uint64
+	ewg := NewErrorWaitGroup(sp.maxParallelization)
+
+	updateProgress := func() {
+		writerMu.Lock()
+		defer writerMu.Unlock()
+
+		progressDone++
+		sp.write("collecting file stats: %s     \r", sp.uint64Pct(progressDone, progressTotal))
+	}
+
+	for _, dci := range sp.topology.DataCenterInfos {
+		for _, ri := range dci.RackInfos {
+			for _, dni := range ri.DataNodeInfos {
+				for _, d := range dni.DiskInfos {
+					statsMu.Lock()
+					progressTotal += uint64(len(d.VolumeInfos))
+					progressTotal += uint64(len(d.EcShardInfos))
+					statsMu.Unlock()
+
+					for _, v := range d.VolumeInfos {
+						ewg.Add(func() error {
+							defer updateProgress()
+
+							// Collect regular volume stats
+							err := operation.WithVolumeServerClient(false, pb.NewServerAddressWithGrpcPort(dni.Id, int(dni.GrpcPort)), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+								resp, reqErr := volumeServerClient.VolumeStatus(context.Background(), &volume_server_pb.VolumeStatusRequest{
+									VolumeId: uint32(v.Id),
+								})
+								if reqErr != nil {
+									return reqErr
+								}
+
+								statsMu.Lock()
+								defer statsMu.Unlock()
+								if resp != nil {
+									if _, ok := sp.regularVolumesStats[v.Id]; !ok {
+										sp.regularVolumesStats[v.Id] = []*VolumeReplicaStats{}
+									}
+									sp.regularVolumesStats[v.Id] = append(sp.regularVolumesStats[v.Id], &VolumeReplicaStats{
+										Id:           dni.Id,
+										VolumeId:     v.Id,
+										Files:        resp.FileCount,
+										FilesDeleted: resp.FileDeletedCount,
+										TotalSize:    resp.VolumeSize,
+									})
+								}
+								return nil
+							})
+
+							return err
+						})
+					}
+
+					for _, eci := range d.EcShardInfos {
+						ewg.Add(func() error {
+							defer updateProgress()
+
+							// Collect EC shard stats
+							statsMu.Lock()
+							_, ok := sp.ecVolumesStats[eci.Id]
+							statsMu.Unlock()
+							if ok {
+								// this EC volume has been already processed, likely on a different node
+								return nil
+							}
+
+							err := operation.WithVolumeServerClient(false, pb.NewServerAddressWithGrpcPort(dni.Id, int(dni.GrpcPort)), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+								resp, reqErr := volumeServerClient.VolumeEcShardsInfo(context.Background(), &volume_server_pb.VolumeEcShardsInfoRequest{
+									VolumeId: uint32(eci.Id),
+								})
+								if reqErr != nil {
+									return reqErr
+								}
+
+								statsMu.Lock()
+								defer statsMu.Unlock()
+								if resp != nil {
+									sp.ecVolumesStats[eci.Id] = &EcVolumeStats{
+										VolumeId:     eci.Id,
+										Files:        resp.FileCount,
+										FilesDeleted: resp.FileDeletedCount,
+										TotalSize:    resp.VolumeSize,
+									}
+								}
+								return nil
+							})
+
+							return err
+						})
+					}
+				}
+			}
+		}
+	}
+
+	err := ewg.Wait()
+	sp.write("")
+	return err
+}
+
+func (sp *ClusterStatusPrinter) printClusterInfo() {
+	dcs := len(sp.topology.DataCenterInfos)
+
+	racks := 0
+	nodes := 0
+	disks := 0
+	for _, dci := range sp.topology.DataCenterInfos {
+		racks += len(dci.RackInfos)
+		for _, ri := range dci.RackInfos {
+			for _, dni := range ri.DataNodeInfos {
+				nodes++
+				// The master keys DiskInfos by disk type, so multiple
+				// same-type physical disks on one node collapse into a single
+				// entry. Count physical disks via SplitByPhysicalDisk so a node
+				// with N disks of one type reports N, not 1.
+				for _, di := range dni.DiskInfos {
+					disks += len(di.SplitByPhysicalDisk())
+				}
+			}
+		}
+	}
+
+	status := "unlocked"
+	if sp.lockHeld || sp.locked {
+		status = "LOCKED"
+		if sp.lockHolder != "" {
+			status += " by " + sp.lockHolder
+		}
+		if sp.lockMessage != "" {
+			status += " (" + sp.lockMessage + ")"
+		}
+	}
+
+	sp.write("cluster:")
+	sp.write("\tid:       %s", sp.topology.Id)
+	sp.write("\tstatus:   %s", status)
+	sp.write("\tnodes:    %s", sp.int(nodes))
+	sp.write("\ttopology: %s %s, %s %s on %s %s",
+		sp.int(dcs), sp.plural(dcs, "DC"),
+		sp.int(disks), sp.plural(disks, "disk"),
+		sp.int(racks), sp.plural(racks, "rack"))
+	sp.write("")
+
+}
+
+func (sp *ClusterStatusPrinter) printVolumeInfo() {
+	collections := len(sp.collections)
+	var maxVolumes uint64
+	volumeIds := map[needle.VolumeId]bool{}
+	ecVolumeIds := map[needle.VolumeId]bool{}
+
+	var replicas, roReplicas, rwReplicas, ecShards int
+
+	for _, dci := range sp.topology.DataCenterInfos {
+		for _, ri := range dci.RackInfos {
+			for _, dni := range ri.DataNodeInfos {
+				for _, di := range dni.DiskInfos {
+					maxVolumes += uint64(di.MaxVolumeCount)
+					for _, vi := range di.VolumeInfos {
+						vid := needle.VolumeId(vi.Id)
+						volumeIds[vid] = true
+						replicas++
+						if vi.ReadOnly {
+							roReplicas++
+						} else {
+							rwReplicas++
+						}
+					}
+					for _, eci := range di.EcShardInfos {
+						vid := needle.VolumeId(eci.Id)
+						ecVolumeIds[vid] = true
+						ecShards += erasure_coding.GetShardCount(eci)
+					}
+				}
+			}
+		}
+	}
+
+	volumes := len(volumeIds)
+	ecVolumes := len(ecVolumeIds)
+	totalVolumes := volumes + ecVolumes
+
+	sp.write("volumes:")
+	sp.write("\ttotal:    %s %s, %s %s",
+		sp.int(totalVolumes), sp.plural(totalVolumes, "volume"),
+		sp.int(collections), sp.plural(collections, "collection"))
+	sp.write("\tmax size: %s", sp.bytes(sp.volumeSizeLimitMb*1024*1024))
+	sp.write("\tregular:  %s/%s %s on %s %s, %s writable (%s), %s read-only (%s)",
+		sp.int(volumes), sp.uint64(maxVolumes), sp.plural(volumes, "volume"),
+		sp.int(replicas), sp.plural(replicas, "replica"),
+		sp.int(rwReplicas), sp.intPct(rwReplicas, replicas),
+		sp.int(roReplicas), sp.intPct(roReplicas, replicas))
+	sp.write("\tEC:       %s EC %s on %s %s (%s shards/volume)",
+		sp.int(ecVolumes), sp.plural(ecVolumes, "volume"),
+		sp.int(ecShards), sp.plural(ecShards, "shard"),
+		sp.intRatio(ecShards, ecVolumes))
+	sp.write("")
+
+}
+
+func (sp *ClusterStatusPrinter) printStorageInfo() {
+	perVolumeSize := map[needle.VolumeId]uint64{}
+	perEcVolumeSize := map[needle.VolumeId]uint64{}
+	var rawVolumeSize, rawEcVolumeSize uint64
+
+	for _, dci := range sp.topology.DataCenterInfos {
+		for _, ri := range dci.RackInfos {
+			for _, dni := range ri.DataNodeInfos {
+				for _, di := range dni.DiskInfos {
+					for _, vi := range di.VolumeInfos {
+						vid := needle.VolumeId(vi.Id)
+						perVolumeSize[vid] = vi.Size
+						rawVolumeSize += vi.Size
+					}
+					for _, eci := range di.EcShardInfos {
+						vid := needle.VolumeId(eci.Id)
+						var size uint64
+						for _, ss := range eci.ShardSizes {
+							size += uint64(ss)
+						}
+						perEcVolumeSize[vid] += size
+						rawEcVolumeSize += size
+					}
+
+				}
+			}
+		}
+	}
+	// normalize EC logical volume sizes given shard settings
+	for vid := range perEcVolumeSize {
+		perEcVolumeSize[vid] = perEcVolumeSize[vid] * erasure_coding.DataShardsCount / erasure_coding.TotalShardsCount
+	}
+
+	var volumeSize, ecVolumeSize uint64
+	for _, s := range perVolumeSize {
+		volumeSize += s
+	}
+	for _, s := range perEcVolumeSize {
+		ecVolumeSize += s
+	}
+	totalSize := volumeSize + ecVolumeSize
+	totalRawSize := rawVolumeSize + rawEcVolumeSize
+
+	sp.write("storage:")
+	sp.write("\ttotal:           %s (%s raw, %s)", sp.bytes(totalSize), sp.bytes(totalRawSize), sp.uint64Pct(totalRawSize, totalSize))
+	sp.write("\tregular volumes: %s (%s raw, %s)", sp.bytes(volumeSize), sp.bytes(rawVolumeSize), sp.uint64Pct(rawVolumeSize, volumeSize))
+	sp.write("\tEC volumes:      %s (%s raw, %s)", sp.bytes(ecVolumeSize), sp.bytes(rawEcVolumeSize), sp.uint64Pct(rawEcVolumeSize, ecVolumeSize))
+	sp.write("")
+}
+
+func (sp *ClusterStatusPrinter) printFilesInfo() {
+	if len(sp.regularVolumesStats) == 0 && len(sp.ecVolumesStats) == 0 {
+		return
+	}
+
+	var regularFilesTotal, regularFilesDeleted, regularFilesSize uint64
+
+	for _, replicaStats := range sp.regularVolumesStats {
+		rc := uint64(len(replicaStats))
+
+		var volumeFilesTotal, volumeFilesSize, volumeFilesDeleted uint64
+		for _, rs := range replicaStats {
+			volumeFilesTotal += rs.Files
+			volumeFilesSize += rs.TotalSize
+			volumeFilesDeleted += rs.FilesDeleted
+		}
+		regularFilesTotal += (volumeFilesTotal / rc)
+		regularFilesSize += (volumeFilesSize / rc)
+		regularFilesDeleted += (volumeFilesDeleted / rc)
+	}
+	regularFiles := regularFilesTotal - regularFilesDeleted
+	var avgRegularFileSize uint64
+	if regularFilesTotal != 0 {
+		avgRegularFileSize = regularFilesSize / regularFilesTotal
+	}
+
+	var ecFilesTotal, ecFilesDeleted, ecFilesSize uint64
+
+	for _, ecStats := range sp.ecVolumesStats {
+		ecFilesTotal += ecStats.Files
+		ecFilesSize += ecStats.TotalSize
+		ecFilesDeleted += ecStats.FilesDeleted
+	}
+	ecFiles := ecFilesTotal - ecFilesDeleted
+	var avgEcFileSize uint64
+	if ecFilesTotal != 0 {
+		avgEcFileSize = ecFilesSize / ecFilesTotal
+	}
+
+	files := regularFiles + ecFiles
+	filesDeleted := regularFilesDeleted + ecFilesDeleted
+	filesTotal := regularFilesTotal + ecFilesTotal
+	filesSize := regularFilesSize + ecFilesSize
+	var avgFileSize uint64
+	if filesTotal != 0 {
+		avgFileSize = filesSize / filesTotal
+	}
+
+	sp.write("files:")
+	sp.write("\ttotal:   %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
+		sp.uint64(filesTotal), sp.uint64Plural(filesTotal, "file"),
+		sp.uint64(files), sp.uint64Pct(files, filesTotal),
+		sp.uint64(filesDeleted), sp.uint64Pct(filesDeleted, filesTotal),
+		sp.bytes(avgFileSize))
+	sp.write("\tregular: %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
+		sp.uint64(regularFilesTotal), sp.uint64Plural(regularFilesTotal, "file"),
+		sp.uint64(regularFiles), sp.uint64Pct(regularFiles, regularFilesTotal),
+		sp.uint64(regularFilesDeleted), sp.uint64Pct(regularFilesDeleted, regularFilesTotal),
+		sp.bytes(avgRegularFileSize))
+	sp.write("\tEC:      %s %s, %s readable (%s), %s deleted (%s), avg %s per file",
+		sp.uint64(ecFilesTotal), sp.uint64Plural(ecFilesTotal, "file"),
+		sp.uint64(ecFiles), sp.uint64Pct(ecFiles, ecFilesTotal),
+		sp.uint64(ecFilesDeleted), sp.uint64Pct(ecFilesDeleted, ecFilesTotal),
+		sp.bytes(avgEcFileSize))
+	sp.write("")
+}

weed/shell/command_cluster_status_test.go

@@ -0,0 +1,306 @@
+package shell
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func TestPrintClusterInfo(t *testing.T) {
+	testCases := []struct {
+		topology    *master_pb.TopologyInfo
+		humanize    bool
+		lockHeld    bool
+		lockHolder  string
+		lockMessage string
+		want        string
+	}{
+		{
+			testTopology1, true, false, "", "",
+			`cluster:
+	id:       test_topo_1
+	status:   unlocked
+	nodes:    5
+	topology: 5 DCs, 5 disks on 6 racks
+
+`,
+		},
+		{
+			testTopology1, false, false, "", "",
+			`cluster:
+	id:       test_topo_1
+	status:   unlocked
+	nodes:    5
+	topology: 5 DC(s), 5 disk(s) on 6 rack(s)
+
+`,
+		},
+		{
+			testTopology1, true, true, "192.168.1.5", "",
+			`cluster:
+	id:       test_topo_1
+	status:   LOCKED by 192.168.1.5
+	nodes:    5
+	topology: 5 DCs, 5 disks on 6 racks
+
+`,
+		},
+		{
+			testTopology1, true, true, "192.168.1.5", "ec.encode",
+			`cluster:
+	id:       test_topo_1
+	status:   LOCKED by 192.168.1.5 (ec.encode)
+	nodes:    5
+	topology: 5 DCs, 5 disks on 6 racks
+
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		var buf bytes.Buffer
+		sp := &ClusterStatusPrinter{
+			writer:      &buf,
+			humanize:    tc.humanize,
+			topology:    tc.topology,
+			lockHeld:    tc.lockHeld,
+			lockHolder:  tc.lockHolder,
+			lockMessage: tc.lockMessage,
+		}
+		sp.printClusterInfo()
+		got := buf.String()
+
+		if got != tc.want {
+			t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want)
+		}
+	}
+}
+
+// TestPrintClusterInfo_multiDiskPerNode covers a node whose several physical
+// disks of the same type collapse into a single DiskInfo on the wire (keyed by
+// disk type), so counting len(DiskInfos) under-reports the physical disk count.
+// Three nodes with six disks each must report 18 disks, not 3.
+func TestPrintClusterInfo_multiDiskPerNode(t *testing.T) {
+	makeNode := func(id string) *master_pb.DataNodeInfo {
+		var ecShardInfos []*master_pb.VolumeEcShardInformationMessage
+		// One EC volume per physical disk, each carrying its own DiskId 0..5.
+		for diskId := uint32(0); diskId < 6; diskId++ {
+			ecShardInfos = append(ecShardInfos, &master_pb.VolumeEcShardInformationMessage{
+				Id:          diskId + 1,
+				DiskId:      diskId,
+				EcIndexBits: 1, // a single shard present
+			})
+		}
+		return &master_pb.DataNodeInfo{
+			Id: id,
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"": {
+					Type:           "",
+					MaxVolumeCount: 60,
+					EcShardInfos:   ecShardInfos,
+				},
+			},
+		}
+	}
+	topo := &master_pb.TopologyInfo{
+		Id: "multi_disk_topo",
+		DataCenterInfos: []*master_pb.DataCenterInfo{{
+			Id: "dc1",
+			RackInfos: []*master_pb.RackInfo{{
+				Id: "rack1",
+				DataNodeInfos: []*master_pb.DataNodeInfo{
+					makeNode("node1"), makeNode("node2"), makeNode("node3"),
+				},
+			}},
+		}},
+	}
+
+	var buf bytes.Buffer
+	sp := &ClusterStatusPrinter{
+		writer:   &buf,
+		humanize: true,
+		topology: topo,
+	}
+	sp.printClusterInfo()
+	got := buf.String()
+	want := `cluster:
+	id:       multi_disk_topo
+	status:   unlocked
+	nodes:    3
+	topology: 1 DC, 18 disks on 1 rack
+
+`
+	if got != want {
+		t.Errorf("multi-disk cluster info:\ngot:\n%s\nwant:\n%s", got, want)
+	}
+}
+
+func TestPrintVolumeInfo(t *testing.T) {
+	testCases := []struct {
+		topology *master_pb.TopologyInfo
+		humanize bool
+		want     string
+	}{
+		{
+			testTopology2, true,
+			`volumes:
+	total:    12,056 volumes, 0 collections
+	max size: 0 B
+	regular:  5,302/25,063 volumes on 15,900 replicas, 15,900 writable (100%), 0 read-only (0%)
+	EC:       6,754 EC volumes on 91,662 shards (13.57 shards/volume)
+
+`,
+		},
+		{
+			testTopology2, false,
+			`volumes:
+	total:    12056 volume(s), 0 collection(s)
+	max size: 0 byte(s)
+	regular:  5302/25063 volume(s) on 15900 replica(s), 15900 writable (100.00%), 0 read-only (0.00%)
+	EC:       6754 EC volume(s) on 91662 shard(s) (13.57 shards/volume)
+
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		var buf bytes.Buffer
+		sp := &ClusterStatusPrinter{
+			writer:   &buf,
+			humanize: tc.humanize,
+			topology: tc.topology,
+		}
+		sp.printVolumeInfo()
+		got := buf.String()
+
+		if got != tc.want {
+			t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want)
+		}
+	}
+}
+
+func TestPrintStorageInfo(t *testing.T) {
+	testCases := []struct {
+		topology *master_pb.TopologyInfo
+		humanize bool
+		want     string
+	}{
+		{
+			testTopology2, true,
+			`storage:
+	total:           5.9 TB (18 TB raw, 299.97%)
+	regular volumes: 5.9 TB (18 TB raw, 299.97%)
+	EC volumes:      0 B (0 B raw, 0%)
+
+`,
+		},
+		{
+			testTopology2, false,
+			`storage:
+	total:           5892610895448 byte(s) (17676186754616 byte(s) raw, 299.97%)
+	regular volumes: 5892610895448 byte(s) (17676186754616 byte(s) raw, 299.97%)
+	EC volumes:      0 byte(s) (0 byte(s) raw, 0.00%)
+
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		var buf bytes.Buffer
+		sp := &ClusterStatusPrinter{
+			writer:   &buf,
+			humanize: tc.humanize,
+			topology: tc.topology,
+		}
+		sp.printStorageInfo()
+		got := buf.String()
+
+		if got != tc.want {
+			t.Errorf("for %v: got %v, want %v", tc.topology.Id, got, tc.want)
+		}
+	}
+}
+
+func TestPrintFilesInfo(t *testing.T) {
+	testCases := []struct {
+		regularVolumesStats RegularVolumesStats
+		ecVolumesStats      EcVolumesStats
+		humanize            bool
+		want                string
+	}{
+		{
+			regularVolumesStats: RegularVolumesStats{
+				1: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9001", VolumeId: 1, Files: 159, FilesDeleted: 8, TotalSize: 89762704},
+					&VolumeReplicaStats{Id: "10.200.17.13:9002", VolumeId: 1, Files: 159, FilesDeleted: 8, TotalSize: 89762704},
+					&VolumeReplicaStats{Id: "10.200.17.13:9008", VolumeId: 1, Files: 159, FilesDeleted: 8, TotalSize: 89762704},
+				},
+				2: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9003", VolumeId: 2, Files: 192, FilesDeleted: 21, TotalSize: 93788632},
+					&VolumeReplicaStats{Id: "10.200.17.13:9004", VolumeId: 2, Files: 192, FilesDeleted: 21, TotalSize: 93788632},
+					&VolumeReplicaStats{Id: "10.200.17.13:9005", VolumeId: 2, Files: 192, FilesDeleted: 21, TotalSize: 93788632},
+				},
+				3: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9001", VolumeId: 3, Files: 149, FilesDeleted: 0, TotalSize: 81643872},
+					&VolumeReplicaStats{Id: "10.200.17.13:9006", VolumeId: 3, Files: 149, FilesDeleted: 0, TotalSize: 81643872},
+					&VolumeReplicaStats{Id: "10.200.17.13:9009", VolumeId: 3, Files: 149, FilesDeleted: 0, TotalSize: 81643872},
+				},
+			},
+			ecVolumesStats: EcVolumesStats{
+				10: &EcVolumeStats{VolumeId: 10, Files: 30, FilesDeleted: 0, TotalSize: 34879032},
+				11: &EcVolumeStats{VolumeId: 11, Files: 55, FilesDeleted: 5, TotalSize: 55540341},
+			},
+			humanize: false,
+			want: `files:
+	total:   585 file(s), 551 readable (94.19%), 34 deleted (5.81%), avg 607888 byte(s) per file
+	regular: 500 file(s), 471 readable (94.20%), 29 deleted (5.80%), avg 530390 byte(s) per file
+	EC:      85 file(s), 80 readable (94.12%), 5 deleted (5.88%), avg 1063757 byte(s) per file
+
+`,
+		},
+		{
+			regularVolumesStats: RegularVolumesStats{
+				1: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9001", VolumeId: 1, Files: 184, FilesDeleted: 33, TotalSize: 79187475},
+					&VolumeReplicaStats{Id: "10.200.17.13:9008", VolumeId: 1, Files: 184, FilesDeleted: 33, TotalSize: 79187475},
+				},
+				2: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9004", VolumeId: 2, Files: 245, FilesDeleted: 4, TotalSize: 89501070},
+					&VolumeReplicaStats{Id: "10.200.17.13:9005", VolumeId: 2, Files: 245, FilesDeleted: 4, TotalSize: 89501070},
+				},
+				3: []*VolumeReplicaStats{
+					&VolumeReplicaStats{Id: "10.200.17.13:9006", VolumeId: 3, Files: 171, FilesDeleted: 12, TotalSize: 124049530},
+					&VolumeReplicaStats{Id: "10.200.17.13:9009", VolumeId: 3, Files: 171, FilesDeleted: 12, TotalSize: 124049530},
+				},
+			},
+			ecVolumesStats: EcVolumesStats{
+				20: &EcVolumeStats{VolumeId: 20, Files: 22, FilesDeleted: 10, TotalSize: 27328233},
+				30: &EcVolumeStats{VolumeId: 30, Files: 16, FilesDeleted: 11, TotalSize: 11193827},
+			},
+			humanize: true,
+			want: `files:
+	total:   638 files, 568 readable (89.02%), 70 deleted (10.97%), avg 519 kB per file
+	regular: 600 files, 551 readable (91.83%), 49 deleted (8.16%), avg 488 kB per file
+	EC:      38 files, 17 readable (44.73%), 21 deleted (55.26%), avg 1.0 MB per file
+
+`,
+		},
+	}
+
+	for i, tc := range testCases {
+		var buf bytes.Buffer
+		sp := &ClusterStatusPrinter{
+			writer:              &buf,
+			humanize:            tc.humanize,
+			regularVolumesStats: tc.regularVolumesStats,
+			ecVolumesStats:      tc.ecVolumesStats,
+		}
+		sp.printFilesInfo()
+		got := buf.String()
+
+		if got != tc.want {
+			t.Errorf("#%d: got %v, want %v", i, got, tc.want)
+		}
+	}
+}

weed/shell/command_collection_delete.go

@@ -0,0 +1,80 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandCollectionDelete{})
+}
+
+type commandCollectionDelete struct {
+}
+
+func (c *commandCollectionDelete) Name() string {
+	return "collection.delete"
+}
+
+func (c *commandCollectionDelete) Help() string {
+	return `delete specified collection
+
+	collection.delete -collection <collection_name> -apply
+
+`
+}
+
+func (c *commandCollectionDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandCollectionDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	colDeleteCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	collectionName := colDeleteCommand.String("collection", "", "collection to delete. Use '_default_' for the empty-named collection.")
+	applyBalancing := colDeleteCommand.Bool("apply", false, "apply the collection")
+	// TODO: remove this alias
+	applyBalancingAlias := colDeleteCommand.Bool("force", false, "apply the collection (alias for -apply)")
+
+	if err = colDeleteCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	handleDeprecatedForceFlag(writer, colDeleteCommand, applyBalancingAlias, applyBalancing)
+	infoAboutSimulationMode(writer, *applyBalancing, "-apply")
+
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	if *collectionName == "" {
+		return fmt.Errorf("empty collection name is not allowed")
+	}
+
+	if *collectionName == "_default_" {
+		*collectionName = ""
+	}
+
+	if !*applyBalancing {
+		fmt.Fprintf(writer, "collection '%s' will be deleted. Use -apply to apply the change.\n", *collectionName)
+		return nil
+	}
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		_, err = client.CollectionDelete(context.Background(), &master_pb.CollectionDeleteRequest{
+			Name: *collectionName,
+		})
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	fmt.Fprintf(writer, "collection %s is deleted.\n", *collectionName)
+
+	return nil
+}

weed/shell/command_collection_list.go

@@ -0,0 +1,192 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+)
+
+func init() {
+	Commands = append(Commands, &commandCollectionList{})
+}
+
+type commandCollectionList struct {
+}
+
+func (c *commandCollectionList) Name() string {
+	return "collection.list"
+}
+
+func (c *commandCollectionList) Help() string {
+	return `list all collections`
+}
+
+func (c *commandCollectionList) HasTag(CommandTag) bool {
+	return false
+}
+
+type CollectionInfo struct {
+	FileCount        float64
+	DeleteCount      float64
+	DeletedByteCount float64
+	Size             float64
+	VolumeCount      int
+}
+
+// LogicalSize is the live data size: single-copy volume size minus the
+// un-vacuumed deleted/overwritten bytes. Quota enforcement uses this so
+// vacuum lag never counts against a bucket.
+func (c *CollectionInfo) LogicalSize() float64 {
+	if c.Size < c.DeletedByteCount {
+		return 0
+	}
+	return c.Size - c.DeletedByteCount
+}
+
+func (c *commandCollectionList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	collections, err := ListCollectionNames(commandEnv, true, true)
+
+	if err != nil {
+		return err
+	}
+
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	collectionInfos := make(map[string]*CollectionInfo)
+
+	collectCollectionInfo(topologyInfo, collectionInfos)
+
+	for _, c := range collections {
+		cif, found := collectionInfos[c]
+		if !found {
+			continue
+		}
+		fmt.Fprintf(writer, "collection:\"%s\"\tvolumeCount:%d\tsize:%.0f\tfileCount:%.0f\tdeletedBytes:%.0f\tdeletion:%.0f\n", c, cif.VolumeCount, cif.Size, cif.FileCount, cif.DeletedByteCount, cif.DeleteCount)
+	}
+
+	fmt.Fprintf(writer, "Total %d collections.\n", len(collections))
+
+	return nil
+}
+
+func ListCollectionNames(commandEnv *CommandEnv, includeNormalVolumes, includeEcVolumes bool) (collections []string, err error) {
+	var resp *master_pb.CollectionListResponse
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err = client.CollectionList(context.Background(), &master_pb.CollectionListRequest{
+			IncludeNormalVolumes: includeNormalVolumes,
+			IncludeEcVolumes:     includeEcVolumes,
+		})
+		return err
+	})
+	if err != nil {
+		return
+	}
+	for _, c := range resp.Collections {
+		collections = append(collections, c.Name)
+	}
+	return
+}
+
+// volumeKey uniquely identifies a volume for per-collection dedupe. Volume
+// IDs are scoped to a collection, so we key by (collection, volumeId) to
+// avoid cross-collection aliasing if the same numeric ID is ever reused.
+type volumeKey struct {
+	collection string
+	volumeId   uint32
+}
+
+// addToCollection folds one replica of a regular volume into the collection
+// totals. Size/FileCount/DeleteCount/DeletedByteCount are divided by the
+// replication factor so that summing over all replicas yields the whole-
+// volume value. VolumeCount is deduped across replicas via seenVolumes so
+// it reports logical volumes (same semantics as the S3 bucket metrics
+// collector and the EC branch below), not shard/replica presences.
+func addToCollection(collectionInfos map[string]*CollectionInfo, seenVolumes map[volumeKey]bool, vif *master_pb.VolumeInformationMessage) {
+	c := vif.Collection
+	cif, found := collectionInfos[c]
+	if !found {
+		cif = &CollectionInfo{}
+		collectionInfos[c] = cif
+	}
+	replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(vif.ReplicaPlacement))
+	copyCount := float64(replicaPlacement.GetCopyCount())
+	cif.Size += float64(vif.Size) / copyCount
+	cif.DeleteCount += float64(vif.DeleteCount) / copyCount
+	cif.FileCount += float64(vif.FileCount) / copyCount
+	cif.DeletedByteCount += float64(vif.DeletedByteCount) / copyCount
+
+	key := volumeKey{collection: c, volumeId: vif.Id}
+	if !seenVolumes[key] {
+		seenVolumes[key] = true
+		cif.VolumeCount++
+	}
+}
+
+// ecCollectionAgg accumulates per-EC-volume counts across the shard holders.
+// fileCount is volume-wide (every holder reports the same .ecx count) so it
+// is deduped via max; deleteCount is node-local to each .ecj and summed.
+type ecCollectionAgg struct {
+	collection  string
+	fileCount   uint64
+	deleteCount uint64
+}
+
+func collectCollectionInfo(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) {
+	seenVolumes := make(map[volumeKey]bool)
+	ecVolumes := make(map[volumeKey]*ecCollectionAgg)
+	for _, dc := range t.DataCenterInfos {
+		for _, r := range dc.RackInfos {
+			for _, dn := range r.DataNodeInfos {
+				for _, diskInfo := range dn.DiskInfos {
+					for _, vi := range diskInfo.VolumeInfos {
+						addToCollection(collectionInfos, seenVolumes, vi)
+					}
+					for _, esi := range diskInfo.EcShardInfos {
+						c := esi.Collection
+						cif, found := collectionInfos[c]
+						if !found {
+							cif = &CollectionInfo{}
+							collectionInfos[c] = cif
+						}
+
+						// EC shards are node-local, so data-shard sizes sum
+						// across nodes to give the logical volume size.
+						// Upstream OSS uses the fixed 10+4 ratio; forks with
+						// per-volume ratio metadata should pass the
+						// configured dataShards value here.
+						cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0))
+
+						key := volumeKey{collection: c, volumeId: esi.Id}
+						agg, ok := ecVolumes[key]
+						if !ok {
+							agg = &ecCollectionAgg{collection: c}
+							ecVolumes[key] = agg
+							cif.VolumeCount++
+						}
+						if esi.FileCount > agg.fileCount {
+							agg.fileCount = esi.FileCount
+						}
+						agg.deleteCount += esi.DeleteCount
+					}
+				}
+			}
+		}
+	}
+
+	for _, agg := range ecVolumes {
+		cif := collectionInfos[agg.collection]
+		if cif == nil {
+			continue
+		}
+		cif.FileCount += float64(agg.fileCount)
+		cif.DeleteCount += float64(agg.deleteCount)
+	}
+}

weed/shell/command_collection_list_test.go

@@ -0,0 +1,206 @@
+package shell
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestCollectCollectionInfoEC verifies that EC-encoded volumes contribute to
+// collection.list totals and s3.bucket.quota.enforce sums. Before this fix
+// the EC branch was a no-op, so encoded volumes silently dropped out of
+// collection size accounting.
+func TestCollectCollectionInfoEC(t *testing.T) {
+	// One 10+4 EC volume split across two nodes: 14 shards * 1000 bytes.
+	// Data shards 0..9 live on (nodeA: 0..6, nodeB: 7..9), parity 10..13
+	// on nodeB. Every shard holder reports file_count=100; node-local
+	// delete counts are 2 + 3 = 5.
+	nodeA := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "bucket-a",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 2,
+					},
+				},
+			},
+		},
+	}
+	nodeB := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "bucket-a",
+						EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 3,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-a"]
+	if !ok {
+		t.Fatalf("expected collection bucket-a in infos, got: %v", infos)
+	}
+	// 10 data shards * 1000 bytes.
+	if cif.Size != 10000 {
+		t.Errorf("Size: got %.0f, want 10000", cif.Size)
+	}
+	if cif.FileCount != 100 {
+		t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", cif.FileCount)
+	}
+	if cif.DeleteCount != 5 {
+		t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", cif.DeleteCount)
+	}
+	if cif.VolumeCount != 1 {
+		t.Errorf("VolumeCount: got %d, want 1", cif.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoMixed verifies that a collection with both a
+// regular volume and an EC volume sums both branches without either clobbering
+// the other, matching the state mid-EC-conversion.
+func TestCollectCollectionInfoMixed(t *testing.T) {
+	node := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				VolumeInfos: []*master_pb.VolumeInformationMessage{
+					{
+						Id:          1,
+						Collection:  "bucket-mix",
+						Size:        5000,
+						FileCount:   50,
+						DeleteCount: 1,
+					},
+				},
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          2,
+						Collection:  "bucket-mix",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity
+						ShardSizes:  []int64{3000, 3000, 3000},
+						FileCount:   80,
+						DeleteCount: 4,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{node},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-mix"]
+	if !ok {
+		t.Fatalf("expected collection bucket-mix in infos, got: %v", infos)
+	}
+	// Regular: 5000 (replicaCount=1). EC data shards: 6000.
+	if cif.Size != 5000+6000 {
+		t.Errorf("Size: got %.0f, want 11000", cif.Size)
+	}
+	if cif.FileCount != 50+80 {
+		t.Errorf("FileCount: got %.0f, want 130", cif.FileCount)
+	}
+	if cif.DeleteCount != 1+4 {
+		t.Errorf("DeleteCount: got %.0f, want 5", cif.DeleteCount)
+	}
+	if cif.VolumeCount != 2 {
+		t.Errorf("VolumeCount: got %d, want 2", cif.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoRegularVolumeDedupesReplicas verifies that a
+// regular volume replicated across three nodes is counted once in
+// VolumeCount (logical, like the S3 metrics path and the EC branch) rather
+// than three times (one per replica presence).
+func TestCollectCollectionInfoRegularVolumeDedupesReplicas(t *testing.T) {
+	// 001 = one-copy replication placement byte; three copies of volume id=7
+	// on three distinct nodes. Per-replica Size/FileCount are divided by
+	// copyCount so summing yields the whole-volume totals.
+	makeNode := func() *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"disk1": {
+					VolumeInfos: []*master_pb.VolumeInformationMessage{
+						{
+							Id:               7,
+							Collection:       "bucket-rep",
+							Size:             3000,
+							FileCount:        30,
+							DeleteCount:      3,
+							DeletedByteCount: 300,
+							ReplicaPlacement: 002, // 0x02 = 3 total copies
+						},
+					},
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{makeNode(), makeNode(), makeNode()},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-rep"]
+	if !ok {
+		t.Fatalf("expected collection bucket-rep in infos, got: %v", infos)
+	}
+	if cif.VolumeCount != 1 {
+		t.Errorf("VolumeCount: got %d, want 1 (deduped by volume id)", cif.VolumeCount)
+	}
+	// Per-replica values are Size/3, summed across 3 replicas gives 3000.
+	if cif.Size != 3000 {
+		t.Errorf("Size: got %.0f, want 3000", cif.Size)
+	}
+	if cif.FileCount != 30 {
+		t.Errorf("FileCount: got %.0f, want 30", cif.FileCount)
+	}
+}

weed/shell/command_ec_balance.go

@@ -0,0 +1,93 @@
+package shell
+
+import (
+	"flag"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcBalance{})
+}
+
+type commandEcBalance struct {
+}
+
+func (c *commandEcBalance) Name() string {
+	return "ec.balance"
+}
+
+func (c *commandEcBalance) Help() string {
+	return `balance all ec shards among all racks and volume servers
+
+	ec.balance [-c EACH_COLLECTION|<collection_name>] [-apply] [-dataCenter <data_center>] [-shardReplicaPlacement <replica_placement>] [-diskType <disk_type>] [-volumeIds <id>[,<id>...]]
+
+	Options:
+	  -diskType: the disk type for EC shards (hdd, ssd, or empty for default hdd)
+	  -volumeIds: only balance these ec volume ids, e.g. -volumeIds 123,456. The plan is
+	    built as if no other ec volume existed, so nothing else is moved or deduplicated.
+	    Ids without any ec shard in the selected collection, dataCenter and disk type are
+	    an error.
+
+	Algorithm:
+	` + ecBalanceAlgorithmDescription
+}
+
+func (c *commandEcBalance) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	balanceCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	collection := balanceCommand.String("collection", "EACH_COLLECTION", "collection name, or \"EACH_COLLECTION\" for each collection")
+	dc := balanceCommand.String("dataCenter", "", "only apply the balancing for this dataCenter")
+	shardReplicaPlacement := balanceCommand.String("shardReplicaPlacement", "", "replica placement for EC shards, or master default if empty")
+	diskTypeStr := balanceCommand.String("diskType", "", "the disk type for EC shards (hdd, ssd, or empty for default hdd)")
+	volumeIdsStr := balanceCommand.String("volumeIds", "", "optional comma-separated list of ec volume ids to balance; defaults to all")
+	maxParallelization := balanceCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
+	ioBytePerSecond := balanceCommand.Int64("ioBytePerSecond", 0, "limit the speed of each shard copy; 0 falls back to the volume server's maintenance rate")
+	applyBalancing := balanceCommand.Bool("apply", false, "apply the balancing plan")
+	// TODO: remove this alias
+	applyBalancingAlias := balanceCommand.Bool("force", false, "apply the balancing plan (alias for -apply)")
+
+	if err = balanceCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	handleDeprecatedForceFlag(writer, balanceCommand, applyBalancingAlias, applyBalancing)
+	infoAboutSimulationMode(writer, *applyBalancing, "-apply")
+
+	var volumeIds []needle.VolumeId
+	if *volumeIdsStr != "" {
+		if volumeIds, err = parseVolumeIdsFlag(*volumeIdsStr); err != nil {
+			return err
+		}
+	}
+
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	var collections []string
+	if *collection == "EACH_COLLECTION" {
+		collections, err = ListCollectionNames(commandEnv, false, true)
+		if err != nil {
+			return err
+		}
+	} else {
+		collections = append(collections, *collection)
+	}
+	glog.V(1).Infof("balanceEcVolumes collections %+v\n", len(collections))
+
+	rp, err := parseReplicaPlacementArg(commandEnv, *shardReplicaPlacement)
+	if err != nil {
+		return err
+	}
+
+	diskType := types.ToDiskType(*diskTypeStr)
+
+	return EcBalance(commandEnv, collections, *dc, rp, diskType, *maxParallelization, *ioBytePerSecond, *applyBalancing, nil, volumeIds)
+}

weed/shell/command_ec_check_replication.go

@@ -0,0 +1,87 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcCheckReplication{})
+}
+
+type commandEcCheckReplication struct {
+}
+
+func (c *commandEcCheckReplication) Name() string {
+	return "ec.check.replication"
+}
+
+func (c *commandEcCheckReplication) Help() string {
+	return `check EC volumes for under- or over-replicated shards
+
+	ec.check.replication [-volumeIds=<id>,<id>...] [-details]
+
+	Reports EC volumes whose shards are:
+	  - under-replicated: at least one shard is missing from every node
+	  - over-replicated:  at least one shard has more than one copy, whether on
+	                      different nodes or on more than one disk of a node
+
+	Over-replication is normal and transient while ec.balance or ec.encode is
+	running, since shards are copied before the redundant copies are deleted;
+	re-check once those operations finish before treating it as a problem.
+
+	Each volume is checked against its own data+parity ratio rather than a fixed
+	shard count, so volumes with non-default erasure coding ratios are reported
+	correctly.
+
+	Options:
+	  -volumeIds: comma-separated EC volume IDs to check (default: all EC volumes)
+	  -details:   print the per-shard node placement for each flagged volume
+`
+}
+
+func (c *commandEcCheckReplication) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcCheckReplication) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	checkReplicationCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	volumeIDsStr := checkReplicationCommand.String("volumeIds", "", "comma-separated EC volume IDs to process (optional)")
+	showDetails := checkReplicationCommand.Bool("details", false, "display result details, if available")
+
+	if err = checkReplicationCommand.Parse(args); err != nil {
+		return err
+	}
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	volumeIDMap := map[uint32]bool{}
+	if *volumeIDsStr != "" {
+		for _, vids := range strings.Split(*volumeIDsStr, ",") {
+			vids = strings.TrimSpace(vids)
+			if vids == "" {
+				continue
+			}
+			if vid, err := strconv.ParseUint(vids, 10, 32); err == nil {
+				volumeIDMap[uint32(vid)] = true
+			} else {
+				return fmt.Errorf("invalid volume ID %q", vids)
+			}
+		}
+	}
+
+	dataNodes, err := collectDataNodes(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	return ec.CheckEcVolumeReplication(writer, dataNodes, volumeIDMap, *showDetails)
+}
+
+// ecCheckReplicationRunner holds the state for a single ec.check.replication invocation.

weed/shell/command_ec_common.go

@@ -0,0 +1,154 @@
+package shell
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+)
+
+// The EC orchestration logic lives in weed/ec so the shell commands and the
+// maintenance workers share it. The aliases below keep the shell-internal
+// names working for the command files that still use them.
+type DataCenterId = ec.DataCenterId
+type RackId = ec.RackId
+type EcNode = ec.EcNode
+
+var (
+	ecBalanceAlgorithmDescription = ec.BalanceAlgorithmDescription
+	eachDataNode                  = ec.EachDataNode
+	collectEcVolumeServersByDc    = ec.CollectEcVolumeServersByDc
+	pickBestDiskOnNode            = ec.PickBestDiskOnNode
+	collectVolumeIdToCollection   = ec.CollectVolumeIdToCollection
+	parseVolumeIdsFlag            = ec.ParseVolumeIdsFlag
+	chunkVolumeIds                = ec.ChunkVolumeIds
+)
+
+// ecEnv adapts a CommandEnv to the cluster access hooks the ec package needs.
+func (ce *CommandEnv) ecEnv() *ec.Env {
+	return &ec.Env{
+		GrpcDialOption: ce.option.GrpcDialOption,
+		FetchTopology: func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) {
+			return collectTopologyInfo(ce, delay)
+		},
+		GetVolumeLocations: func(vid uint32) ([]wdclient.Location, bool) {
+			return ce.MasterClient.GetLocationsClone(vid)
+		},
+		IsLocked: ce.isLocked,
+	}
+}
+
+// Overridable functions for testing.
+var getDefaultReplicaPlacement = _getDefaultReplicaPlacement
+
+func _getDefaultReplicaPlacement(commandEnv *CommandEnv) (*super_block.ReplicaPlacement, error) {
+	var resp *master_pb.GetMasterConfigurationResponse
+	var err error
+
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err = client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return super_block.NewReplicaPlacementFromString(resp.DefaultReplication)
+}
+
+func parseReplicaPlacementArg(commandEnv *CommandEnv, replicaStr string) (*super_block.ReplicaPlacement, error) {
+	var rp *super_block.ReplicaPlacement
+	var err error
+
+	if replicaStr != "" {
+		rp, err = super_block.NewReplicaPlacementFromString(replicaStr)
+		if err != nil {
+			return rp, err
+		}
+		glog.V(1).Infof("using replica placement %q for EC volumes\n", rp.String())
+	} else {
+		// No replica placement argument provided, resolve from master default settings.
+		rp, err = getDefaultReplicaPlacement(commandEnv)
+		if err != nil {
+			return rp, err
+		}
+		glog.V(1).Infof("using master default replica placement %q for EC volumes\n", rp.String())
+	}
+
+	return rp, nil
+}
+
+func collectTopologyInfo(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) {
+	if delayBeforeCollecting > 0 {
+		time.Sleep(delayBeforeCollecting)
+	}
+
+	var resp *master_pb.VolumeListResponse
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err = pb.CollectVolumeList(context.Background(), client, &master_pb.VolumeListRequest{})
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	return resp.TopologyInfo, resp.VolumeSizeLimitMb, nil
+}
+
+func collectDataNodes(commandEnv *CommandEnv, delayBeforeCollecting time.Duration) ([]*master_pb.DataNodeInfo, error) {
+	dataNodes := []*master_pb.DataNodeInfo{}
+
+	topo, _, err := collectTopologyInfo(commandEnv, delayBeforeCollecting)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, dci := range topo.GetDataCenterInfos() {
+		for _, r := range dci.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				dataNodes = append(dataNodes, dn)
+			}
+		}
+	}
+
+	return dataNodes, nil
+}
+
+func collectEcNodesForDC(commandEnv *CommandEnv, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) {
+	return ec.CollectEcNodesForDC(commandEnv.ecEnv(), selectedDataCenter, diskType)
+}
+
+func collectEcNodes(commandEnv *CommandEnv, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) {
+	return ec.CollectEcNodes(commandEnv.ecEnv(), diskType)
+}
+
+func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, destDiskId uint32, applyBalancing bool, diskType types.DiskType) (err error) {
+	return ec.MoveMountedShardToEcNode(commandEnv.ecEnv(), existingLocation, collection, vid, shardId, destinationEcNode, destDiskId, applyBalancing, diskType)
+}
+
+// EcBalance balances EC shards across the cluster; see ec.EcBalance for the
+// excludeNodes and volumeIds semantics.
+func EcBalance(commandEnv *CommandEnv, collections []string, dc string, ecReplicaPlacement *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, ioBytePerSecond int64, applyBalancing bool, excludeNodes map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId) (err error) {
+	return ec.EcBalance(commandEnv.ecEnv(), collections, dc, ecReplicaPlacement, diskType, maxParallelization, ioBytePerSecond, applyBalancing, excludeNodes, volumeIds, nil)
+}
+
+// compileCollectionPattern compiles the -collection value shared by the ec and
+// tier commands: a comma-separated list of collection names, wildcards, and
+// regex patterns. An empty value matches the empty-named collection only, the
+// same as CollectionDefault.
+func compileCollectionPattern(pattern string) (*wildcard.CollectionMatcher, error) {
+	if strings.TrimSpace(pattern) == "" {
+		pattern = CollectionDefault
+	}
+	return wildcard.CompileCollectionMatcher(pattern)
+}

weed/shell/command_ec_common_test.go

@@ -0,0 +1,39 @@
+package shell
+
+import "testing"
+
+// The ec and tier commands read an empty -collection as the collection with no
+// name, unlike -collectionPattern, where empty means every collection.
+func TestCompileCollectionPattern(t *testing.T) {
+	tests := []struct {
+		pattern string
+		matches []string
+		misses  []string
+	}{
+		{pattern: "", matches: []string{""}, misses: []string{"pictures"}},
+		{pattern: CollectionDefault, matches: []string{""}, misses: []string{"pictures"}},
+		{pattern: "pictures", matches: []string{"pictures"}, misses: []string{"", "pictures-backup"}},
+		{pattern: "pictures,videos", matches: []string{"pictures", "videos"}, misses: []string{"", "clips"}},
+		{pattern: "pictures,_default", matches: []string{"pictures", ""}, misses: []string{"clips"}},
+		{pattern: "pictures*", matches: []string{"pictures", "pictures-backup"}, misses: []string{"clips"}},
+		{pattern: "^pictures", matches: []string{"pictures", "pictures-backup"}, misses: []string{"clips"}},
+		{pattern: "*", matches: []string{"", "pictures"}},
+	}
+
+	for _, tt := range tests {
+		matcher, err := compileCollectionPattern(tt.pattern)
+		if err != nil {
+			t.Fatalf("compileCollectionPattern(%q): %v", tt.pattern, err)
+		}
+		for _, collection := range tt.matches {
+			if !matcher.Matches(collection) {
+				t.Errorf("pattern %q should match collection %q", tt.pattern, collection)
+			}
+		}
+		for _, collection := range tt.misses {
+			if matcher.Matches(collection) {
+				t.Errorf("pattern %q should not match collection %q", tt.pattern, collection)
+			}
+		}
+	}
+}

weed/shell/command_ec_decode.go

@@ -0,0 +1,135 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcDecode{})
+}
+
+type commandEcDecode struct {
+}
+
+func (c *commandEcDecode) Name() string {
+	return "ec.decode"
+}
+
+func (c *commandEcDecode) Help() string {
+	return `decode a erasure coded volume into a normal volume
+
+	ec.decode [-collection=""] [-volumeId=<volume_id>] [-batchSize=10] [-diskType=<disk_type>] [-checkMinFreeSpace]
+
+	The -collection parameter supports regular expressions for pattern matching:
+	  - Use exact match: ec.decode -collection="^mybucket$"
+	  - Match multiple buckets: ec.decode -collection="bucket.*"
+	  - Match all collections: ec.decode -collection=".*"
+
+	Options:
+	  -diskType: source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)
+	  -checkMinFreeSpace: check min free space when selecting the decode target (default true)
+	  -batchSize: decode this many volumes per topology refresh (default 10; 0 = one snapshot for all volumes)
+
+	Examples:
+	  # Decode EC shards from HDD (default)
+	  ec.decode -collection=mybucket
+
+	  # Decode EC shards from SSD
+	  ec.decode -collection=mybucket -diskType=ssd
+
+`
+}
+
+func (c *commandEcDecode) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	decodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	volumeId := decodeCommand.Int("volumeId", 0, "the volume id")
+	collection := decodeCommand.String("collection", "", "comma-separated collection names, wildcards, or regex patterns; empty matches the collection with no name")
+	diskTypeStr := decodeCommand.String("diskType", "", "source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)")
+	checkMinFreeSpace := decodeCommand.Bool("checkMinFreeSpace", true, "check min free space when selecting the decode target")
+	batchSize := decodeCommand.Int("batchSize", DefaultEcBatchSize, "decode up to this many volumes per topology refresh (0 = one snapshot for all volumes)")
+	if err = decodeCommand.Parse(args); err != nil {
+		return nil
+	}
+	if *batchSize < 0 {
+		return fmt.Errorf("-batchSize must be >= 0")
+	}
+
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	vid := needle.VolumeId(*volumeId)
+	diskType := types.ToDiskType(*diskTypeStr)
+	env := commandEnv.ecEnv()
+
+	// collect topology information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	var diskUsageState *ec.DecodeDiskUsageState
+	if *checkMinFreeSpace {
+		diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
+	}
+
+	// volumeId is provided
+	if vid != 0 {
+		return ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
+	}
+
+	// apply to all volumes in the collection
+	collectionMatcher, err := compileCollectionPattern(*collection)
+	if err != nil {
+		return fmt.Errorf("invalid collection pattern '%s': %v", *collection, err)
+	}
+	volumeIds := ec.CollectEcShardIds(topologyInfo, collectionMatcher, diskType)
+	fmt.Printf("ec decode volumes: %v\n", volumeIds)
+	batches := chunkVolumeIds(volumeIds, *batchSize)
+	for i, batch := range batches {
+		if i > 0 {
+			// earlier batches moved shards and created volumes; re-snapshot so
+			// shard locations and free-space accounting stay accurate
+			topologyInfo, _, err = collectTopologyInfo(commandEnv, 0)
+			if err != nil {
+				return err
+			}
+			if *checkMinFreeSpace {
+				diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
+			}
+		}
+		if len(batches) > 1 {
+			fmt.Printf("ec decode batch %d/%d: %v\n", i+1, len(batches), batch)
+		}
+		for _, vid := range batch {
+			if err = ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) {
+	var resp *master_pb.LookupVolumeResponse
+	err = commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		resp, err = client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: volumeIds})
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	return resp.VolumeIdLocations, nil
+}

weed/shell/command_ec_encode.go

@@ -0,0 +1,210 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcEncode{})
+}
+
+type commandEcEncode struct {
+}
+
+func (c *commandEcEncode) Name() string {
+	return "ec.encode"
+}
+
+func (c *commandEcEncode) Help() string {
+	return `apply erasure coding to a volume
+
+	ec.encode [-collection=""] [-fullPercent=95 -quietFor=1h] [-batchSize=10] [-verbose] [-sourceDiskType=<disk_type>] [-diskType=<disk_type>]
+	ec.encode [-volumeId=<volume_id>|-volumeIds=<volume_id>,...] [-batchSize=10] [-verbose] [-diskType=<disk_type>]
+
+	This command will:
+	1. freeze one volume
+	2. apply erasure coding to the volume
+	3. (optionally) re-balance encoded shards across multiple volume servers
+
+	The erasure coding is 10.4. So ideally you have more than 14 volume servers, and you can afford
+	to lose 4 volume servers.
+
+	If the number of volumes are not high, the worst case is that you only have 4 volume servers,
+	and the shards are spread as 4,4,3,3, respectively. You can afford to lose one volume server.
+
+	If you only have less than 4 volume servers, with erasure coding, at least you can afford to
+	have 4 corrupted shard files.
+
+	The -collection parameter is a comma-separated list of collection names, with
+	"*" and "?" wildcards, and regex patterns:
+	  - One collection: ec.encode -collection="mybucket"
+	  - Several collections: ec.encode -collection="mybucket,otherbucket"
+	  - Match by prefix: ec.encode -collection="bucket*"
+	  - Match all collections: ec.encode -collection="*"
+	  - The empty-named collection: ec.encode -collection="_default"
+
+	Options:
+	  -verbose: show detailed reasons why volumes are not selected for encoding
+	  -sourceDiskType: filter source volumes by disk type (hdd, ssd, or empty for all)
+	  -diskType: target disk type for EC shards (hdd, ssd, or empty for default hdd)
+	  -batchSize: encode/rebalance/verify/delete this many volumes at a time (default 10; 0 = all in one batch)
+	  -volumeIds: comma-separated volume IDs to encode
+
+	Each batch is committed independently. If a later batch fails, earlier batches
+	are already encoded and their original volumes deleted.
+
+	Examples:
+	  # Encode SSD volumes to SSD EC shards (same tier)
+	  ec.encode -collection=mybucket -sourceDiskType=ssd -diskType=ssd
+
+	  # Encode SSD volumes to HDD EC shards (tier migration to cheaper storage)
+	  ec.encode -collection=mybucket -sourceDiskType=ssd -diskType=hdd
+
+	  # Encode all volumes to SSD EC shards
+	  ec.encode -collection=mybucket -diskType=ssd
+
+	  # Encode selected volume IDs and delete originals after each batch
+	  ec.encode -volumeIds=101,102,103 -batchSize=2
+
+	Re-balancing algorithm:
+	` + ecBalanceAlgorithmDescription
+}
+
+func (c *commandEcEncode) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
+	volumeIdsStr := encodeCommand.String("volumeIds", "", "comma-separated volume ids")
+	collection := encodeCommand.String("collection", "", "comma-separated collection names, wildcards, or regex patterns; empty matches the collection with no name")
+	fullPercentage := encodeCommand.Float64("fullPercent", 95, "the volume reaches the percentage of max volume size")
+	quietPeriod := encodeCommand.Duration("quietFor", time.Hour, "select volumes without no writes for this period")
+	maxParallelization := encodeCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
+	batchSize := encodeCommand.Int("batchSize", DefaultEcBatchSize, "encode/re-balance/verify/delete up to this many volumes at a time (0 = all in one batch)")
+	forceChanges := encodeCommand.Bool("force", false, "force the encoding even if the cluster has less than recommended 4 nodes")
+	shardReplicaPlacement := encodeCommand.String("shardReplicaPlacement", "", "replica placement for EC shards, or master default if empty")
+	sourceDiskTypeStr := encodeCommand.String("sourceDiskType", "", "filter source volumes by disk type (hdd, ssd, or empty for all)")
+	diskTypeStr := encodeCommand.String("diskType", "", "target disk type for EC shards (hdd, ssd, or empty for default hdd)")
+	applyBalancing := encodeCommand.Bool("rebalance", true, "re-balance EC shards after creation (default: true)")
+	verbose := encodeCommand.Bool("verbose", false, "show detailed reasons why volumes are not selected for encoding")
+
+	if err = encodeCommand.Parse(args); err != nil {
+		return nil
+	}
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+	rp, err := parseReplicaPlacementArg(commandEnv, *shardReplicaPlacement)
+	if err != nil {
+		return err
+	}
+
+	// Parse source disk type filter (optional)
+	var sourceDiskType *types.DiskType
+	if *sourceDiskTypeStr != "" {
+		sdt := types.ToDiskType(*sourceDiskTypeStr)
+		sourceDiskType = &sdt
+	}
+
+	// Parse target disk type for EC shards
+	diskType := types.ToDiskType(*diskTypeStr)
+
+	// collect topology information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	if !*forceChanges {
+		var nodeCount int
+		eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+			nodeCount++
+		})
+		if nodeCount < erasure_coding.ParityShardsCount {
+			glog.V(0).Infof("skip erasure coding with %d nodes, less than recommended %d nodes", nodeCount, erasure_coding.ParityShardsCount)
+			return nil
+		}
+	}
+
+	var volumeIds []needle.VolumeId
+	if *volumeId != 0 || strings.TrimSpace(*volumeIdsStr) != "" {
+		if *volumeId != 0 && strings.TrimSpace(*volumeIdsStr) != "" {
+			return fmt.Errorf("-volumeId and -volumeIds are mutually exclusive")
+		}
+		if *volumeId != 0 {
+			volumeIds = append(volumeIds, needle.VolumeId(*volumeId))
+		} else {
+			volumeIds, err = parseVolumeIdsFlag(*volumeIdsStr)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		// apply to all volumes for the given collection pattern (regex)
+		volumeIds, _, err = collectVolumeIdsForEcEncode(commandEnv, *collection, sourceDiskType, *fullPercentage, *quietPeriod, *verbose)
+		if err != nil {
+			return err
+		}
+	}
+	if len(volumeIds) == 0 {
+		fmt.Println("No volumes, nothing to do.")
+		return nil
+	}
+	if *batchSize < 0 {
+		return fmt.Errorf("-batchSize must be >= 0")
+	}
+
+	batches := ec.ChunkVolumeIds(volumeIds, *batchSize)
+	if len(batches) > 1 {
+		fmt.Printf("Processing %d volumes in %d batch(es), batchSize=%d\n", len(volumeIds), len(batches), *batchSize)
+	}
+	for i, batchVolumeIds := range batches {
+		if len(batches) > 1 {
+			fmt.Printf("Starting EC encoding batch %d/%d with %d volumes: %v\n", i+1, len(batches), len(batchVolumeIds), batchVolumeIds)
+		}
+		if err := ec.ProcessEcEncodeBatch(commandEnv.ecEnv(), writer, batchVolumeIds, rp, diskType, *maxParallelization, *applyBalancing, *collection); err != nil {
+			return fmt.Errorf("ec encode batch %d/%d for volumes %v: %w", i+1, len(batches), batchVolumeIds, err)
+		}
+	}
+	if len(batches) > 1 {
+		fmt.Printf("Successfully completed EC encoding for %d volumes in %d batch(es)\n", len(volumeIds), len(batches))
+	}
+
+	return nil
+}
+func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, collectionPattern string, sourceDiskType *types.DiskType, fullPercentage float64, quietPeriod time.Duration, verbose bool) (vids []needle.VolumeId, matchedCollections []string, err error) {
+	// compile regex pattern for collection matching
+	collectionMatcher, err := compileCollectionPattern(collectionPattern)
+	if err != nil {
+		return nil, nil, fmt.Errorf("invalid collection pattern '%s': %v", collectionPattern, err)
+	}
+
+	// collect topology information
+	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return
+	}
+
+	quietSeconds := int64(quietPeriod / time.Second)
+	nowUnixSeconds := time.Now().Unix()
+
+	fmt.Printf("collect volumes with collection pattern '%s', quiet for: %d seconds and %.1f%% full\n", collectionPattern, quietSeconds, fullPercentage)
+
+	vids, matchedCollections = ec.SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionMatcher, sourceDiskType, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
+	return
+}

weed/shell/command_ec_rebuild.go

@@ -0,0 +1,127 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcRebuild{})
+}
+
+type commandEcRebuild struct {
+}
+
+func (c *commandEcRebuild) Name() string {
+	return "ec.rebuild"
+}
+
+func (c *commandEcRebuild) Help() string {
+	return `find and rebuild missing ec shards among volume servers
+
+	ec.rebuild [-c EACH_COLLECTION|<collection_name>] [-apply] [-maxParallelization N] [-diskType=<disk_type>]
+
+	Before rebuilding, asks volume servers to recover any shards left unmounted by
+	a missing .ecx index (the index resides only on a peer server). Such shards are
+	invisible to the master, so recovering them first avoids regenerating data that
+	is actually present (issue #10104).
+
+	Options:
+	  -collection: specify a collection name, or "EACH_COLLECTION" to process all collections
+	  -apply: actually perform the rebuild operations (default is dry-run mode)
+	  -maxParallelization: number of volumes to rebuild concurrently (default: 10)
+	                       Increase for faster rebuilds with more system resources.
+	                       Decrease if experiencing resource contention or instability.
+	  -diskType: disk type for EC shards (hdd, ssd, or empty for default hdd)
+
+	Algorithm:
+
+	For each type of volume server (different max volume count limit){
+		for each collection {
+			rebuildEcVolumes()
+		}
+	}
+
+	func rebuildEcVolumes(){
+		idealWritableVolumes = totalWritableVolumes / numVolumeServers
+		for {
+			sort all volume servers ordered by the number of local writable volumes
+			pick the volume server A with the lowest number of writable volumes x
+			pick the volume server B with the highest number of writable volumes y
+			if y > idealWritableVolumes and x +1 <= idealWritableVolumes {
+				if B has a writable volume id v that A does not have {
+					move writable volume v from A to B
+				}
+			}
+		}
+	}
+
+`
+}
+
+func (c *commandEcRebuild) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	fixCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	collection := fixCommand.String("collection", "EACH_COLLECTION", "collection name, or \"EACH_COLLECTION\" for each collection")
+	volumeIdsStr := fixCommand.String("volumeIds", "", "optional comma-separated list of volume ID to process; defaults to all volumes in the collection")
+	maxParallelization := fixCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
+	applyChanges := fixCommand.Bool("apply", false, "apply the changes")
+	diskTypeStr := fixCommand.String("diskType", "", "disk type for EC shards (hdd, ssd, or empty for default hdd)")
+	// TODO: remove this alias
+	applyChangesAlias := fixCommand.Bool("force", false, "apply the changes (alias for -apply)")
+	if err = fixCommand.Parse(args); err != nil {
+		return nil
+	}
+	handleDeprecatedForceFlag(writer, fixCommand, applyChangesAlias, applyChanges)
+	infoAboutSimulationMode(writer, *applyChanges, "-apply")
+
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	diskType := types.ToDiskType(*diskTypeStr)
+
+	// collect all ec nodes
+	allEcNodes, _, err := collectEcNodes(commandEnv, diskType)
+	if err != nil {
+		return err
+	}
+
+	var collections []string
+	if *collection == "EACH_COLLECTION" {
+		collections, err = ListCollectionNames(commandEnv, false, true)
+		if err != nil {
+			return err
+		}
+	} else {
+		collections = []string{*collection}
+	}
+
+	var volumeIds []needle.VolumeId
+	if *volumeIdsStr != "" {
+		for _, vidStr := range strings.Split(*volumeIdsStr, ",") {
+			vidStr = strings.TrimSpace(vidStr)
+			if len(vidStr) == 0 {
+				continue
+			}
+			if vid, err := strconv.ParseUint(vidStr, 10, 32); err == nil {
+				volumeIds = append(volumeIds, needle.VolumeId(vid))
+			} else {
+				return fmt.Errorf("invalid volume ID %q", vidStr)
+			}
+		}
+	}
+
+	return ec.RebuildEcVolumes(commandEnv.ecEnv(), allEcNodes, writer, collections, volumeIds, diskType, *maxParallelization, *applyChanges)
+}

weed/shell/command_ec_scrub.go

@@ -0,0 +1,112 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcVolumeScrub{})
+}
+
+type commandEcVolumeScrub struct {
+}
+
+func (c *commandEcVolumeScrub) Name() string {
+	return "ec.scrub"
+}
+
+func (c *commandEcVolumeScrub) Help() string {
+	return `scrubs EC volume contents on volume servers.
+
+	Supports either scrubbing only needle data, or deep scrubbing file contents as well.
+
+	Scrubbing can be limited to specific EC volume IDs for specific volume servers.
+	By default, all volume IDs across all servers are processed.
+`
+}
+
+func (c *commandEcVolumeScrub) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	volScrubCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	nodesStr := volScrubCommand.String("node", "", "comma-separated list of volume server <host>:<port> (optional)")
+	volumeIDsStr := volScrubCommand.String("volumeId", "", "comma-separated EC volume IDs to process (optional)")
+	mode := volScrubCommand.String("mode", "local", "scrubbing mode (index/local/full/reads/checksum)")
+	maxParallelization := volScrubCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
+	showDetails := volScrubCommand.Bool("details", false, "display scrub result details, if available")
+	forceDeletedNeedlesCheck := volScrubCommand.Bool("forceDeletedNeedlesCheck", false, "force strict verification of deleted needles (full and reads modes only); may report false positives when EC indexes disagree")
+
+	if err = volScrubCommand.Parse(args); err != nil {
+		return err
+	}
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	volumeServerAddrs := []pb.ServerAddress{}
+	if *nodesStr != "" {
+		for _, addr := range strings.Split(*nodesStr, ",") {
+			addr = strings.TrimSpace(addr)
+			if addr == "" {
+				continue
+			}
+			volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr))
+		}
+	} else {
+		dns, err := collectDataNodes(commandEnv, 0)
+		if err != nil {
+			return err
+		}
+		for _, dn := range dns {
+			volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address))
+		}
+	}
+
+	volumeIDs := []uint32{}
+	if *volumeIDsStr != "" {
+		for _, vids := range strings.Split(*volumeIDsStr, ",") {
+			vids = strings.TrimSpace(vids)
+			if vids == "" {
+				continue
+			}
+			if vid, err := strconv.ParseUint(vids, 10, 32); err == nil {
+				volumeIDs = append(volumeIDs, uint32(vid))
+			} else {
+				return fmt.Errorf("invalid volume ID %q", vids)
+			}
+		}
+	}
+
+	var scrubMode volume_server_pb.VolumeScrubMode
+	switch strings.ToUpper(*mode) {
+	case "INDEX":
+		scrubMode = volume_server_pb.VolumeScrubMode_INDEX
+	case "LOCAL":
+		scrubMode = volume_server_pb.VolumeScrubMode_LOCAL
+	case "FULL":
+		scrubMode = volume_server_pb.VolumeScrubMode_FULL
+	case "CHECKSUM":
+		scrubMode = volume_server_pb.VolumeScrubMode_CHECKSUM
+	case "READS":
+		scrubMode = volume_server_pb.VolumeScrubMode_READS
+	default:
+		return fmt.Errorf("unsupported scrubbing mode %q", *mode)
+	}
+	fmt.Fprintf(writer, "using %s mode\n", scrubMode.String())
+	if *forceDeletedNeedlesCheck &&
+		scrubMode != volume_server_pb.VolumeScrubMode_FULL && scrubMode != volume_server_pb.VolumeScrubMode_READS {
+		return fmt.Errorf("deleted needle checks are only supported for FULL and READS scrubs")
+	}
+
+	return ec.ScrubEcVolumes(commandEnv.ecEnv(), writer, volumeServerAddrs, volumeIDs, scrubMode, *forceDeletedNeedlesCheck, *maxParallelization, *showDetails)
+}

weed/shell/command_ec_shard_unmount.go

@@ -0,0 +1,97 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/seaweedfs/seaweedfs/weed/ec"
+)
+
+func init() {
+	Commands = append(Commands, &commandEcShardUnmount{})
+}
+
+type commandEcShardUnmount struct {
+}
+
+func (c *commandEcShardUnmount) Name() string {
+	return "ec.shard.unmount"
+}
+
+func (c *commandEcShardUnmount) Help() string {
+	return `Unmounts, and optionally deletes, EC volume shards.
+
+	ec.shard.unmount --volumeId=<volume id> --shardId=<comma-separated list of shard IDs> [--delete]
+
+	Shard IDs can be specified either as a single numeric ID (f.ex. "2") or with a
+	node address (<shard_id>@<address>, f.ex. "3@10.200.18.88:9007") to pick one
+	copy when shards are repeated and/or co-located on the same node. The live
+	topology is printed in the same form, so its entries can be copied verbatim.
+
+	Unmounting only takes shards offline on their volume servers; pass --delete to
+	also remove the shard files. This is useful to clean up over-replicated EC
+	volumes without a full rebalance.
+
+	Shards are processed one by one, best-effort: on error the command stops and
+	leaves already-processed shards as they are.
+
+	This command can, and will, irrevocably delete data if used incorrectly.
+	Tread carefully.
+`
+}
+
+func (c *commandEcShardUnmount) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+	ecShardUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	volumeID := ecShardUnmountCommand.Uint("volumeId", 0, "volume ID for the shards to process")
+	shardIDsStr := ecShardUnmountCommand.String("shardId", "", "comma-separated EC shard IDs for the volume")
+	delete := ecShardUnmountCommand.Bool("delete", false, "also delete the shard files after unmounting them")
+	ignoreInvalid := ecShardUnmountCommand.Bool("ignoreInvalid", false, "ignore user-provided shards that match nothing live, and try to proceed anyway")
+	apply := ecShardUnmountCommand.Bool("apply", false, "Execute actions on the cluster. By default only lists what actions would be executed.")
+	applyAlias := ecShardUnmountCommand.Bool("force", false, "Execute actions on the cluster (alias for -apply).")
+
+	if err = ecShardUnmountCommand.Parse(args); err != nil {
+		return err
+	}
+	if err = commandEnv.confirmIsLocked(args); err != nil {
+		return
+	}
+
+	if *volumeID == 0 {
+		return fmt.Errorf("no volume ID")
+	}
+	if *volumeID > math.MaxUint32 {
+		return fmt.Errorf("invalid volume ID %d", *volumeID)
+	}
+
+	if *shardIDsStr == "" {
+		return fmt.Errorf("missing shardId")
+	}
+	shards, err := ec.ShardRefsFromString(*shardIDsStr)
+	if err != nil {
+		return err
+	}
+
+	// collect topology information
+	topology, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+
+	return ec.UnmountShards(commandEnv.ecEnv(), writer, ec.ShardUnmountRequest{
+		Topology:      topology,
+		VolumeID:      uint32(*volumeID),
+		Shards:        shards,
+		Delete:        *delete,
+		IgnoreInvalid: *ignoreInvalid,
+		Apply:         *apply || *applyAlias,
+	})
+}

weed/shell/command_fs_cat.go

@@ -0,0 +1,72 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsCat{})
+}
+
+type commandFsCat struct {
+}
+
+func (c *commandFsCat) Name() string {
+	return "fs.cat"
+}
+
+func (c *commandFsCat) Help() string {
+	return `stream the file content on to the screen
+
+	fs.cat /dir/file_name
+`
+}
+
+func (c *commandFsCat) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	if commandEnv.isDirectory(path) {
+		return fmt.Errorf("%s is a directory", path)
+	}
+
+	dir, name := util.FullPath(path).DirAndName()
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Name:      name,
+			Directory: dir,
+		}
+		respLookupEntry, err := filer_pb.LookupEntry(context.Background(), client, request)
+		if err != nil {
+			return err
+		}
+
+		if len(respLookupEntry.Entry.Content) > 0 {
+			_, err = writer.Write(respLookupEntry.Entry.Content)
+			return err
+		}
+
+		return filer.StreamContent(commandEnv.MasterClient, writer, respLookupEntry.Entry.GetChunks(), 0, int64(filer.FileSize(respLookupEntry.Entry)))
+
+	})
+
+}

weed/shell/command_fs_cd.go

@@ -0,0 +1,58 @@
+package shell
+
+import (
+	"io"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsCd{})
+}
+
+type commandFsCd struct {
+}
+
+func (c *commandFsCd) Name() string {
+	return "fs.cd"
+}
+
+func (c *commandFsCd) Help() string {
+	return `change directory to a directory /path/to/dir
+
+	The full path can be too long to type. For example,
+		fs.ls /some/path/to/file_name
+
+	can be simplified as
+
+		fs.cd /some/path
+		fs.ls to/file_name
+`
+}
+
+func (c *commandFsCd) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsCd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	if path == "/" {
+		commandEnv.option.Directory = "/"
+		return nil
+	}
+
+	err = commandEnv.checkDirectory(path)
+
+	if err == nil {
+		commandEnv.option.Directory = path
+	}
+
+	return err
+}

weed/shell/command_fs_configure.go

@@ -0,0 +1,216 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsConfigure{})
+}
+
+type commandFsConfigure struct {
+}
+
+func (c *commandFsConfigure) Name() string {
+	return "fs.configure"
+}
+
+func (c *commandFsConfigure) Help() string {
+	return `configure and apply storage options for each location
+
+	# see the current configuration file content
+	fs.configure
+
+	# trying the changes and see the possible configuration file content
+	fs.configure -locationPrefix=/my/folder -collection=abc
+	fs.configure -locationPrefix=/my/folder -collection=abc -ttl=7d
+
+	# example: configure adding only 1 physical volume for each bucket collection
+	fs.configure -locationPrefix=/buckets/ -volumeGrowthCount=1
+
+	# apply the changes
+	fs.configure -locationPrefix=/my/folder -collection=abc -apply
+
+	# example: unlock a bucket that quota enforcement made read-only
+	fs.configure -locationPrefix=/buckets/my_bucket/ -readOnly=false -apply
+
+	# example: keep one directory writable under a worm-protected tree
+	fs.configure -locationPrefix=/buckets/my_bucket/ -worm -apply
+	fs.configure -locationPrefix=/buckets/my_bucket/scratch/ -worm=false -apply
+
+	# delete the changes
+	fs.configure -locationPrefix=/my/folder -delete -apply
+
+`
+}
+
+func (c *commandFsConfigure) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	fsConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	locationPrefix := fsConfigureCommand.String("locationPrefix", "", "path prefix, required to update the path-specific configuration")
+	collection := fsConfigureCommand.String("collection", "", "assign writes to this collection")
+	replication := fsConfigureCommand.String("replication", "", "assign writes with this replication")
+	ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl (e.g., 1m, 1h, 1d, 1w, 1y)")
+	diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes")
+	isReadOnly := fsConfigureCommand.Bool("readOnly", false, "disable writes")
+	worm := fsConfigureCommand.Bool("worm", false, "write-once-read-many, written files are readonly; unset inherits from the parent path")
+	wormGracePeriod := fsConfigureCommand.Uint64("wormGracePeriod", 0, "grace period before worm is enforced, in seconds")
+	wormRetentionTime := fsConfigureCommand.Uint64("wormRetentionTime", 0, "retention time for a worm enforced file, in seconds")
+	maxFileNameLength := fsConfigureCommand.Uint("maxFileNameLength", 0, "file name length limits in bytes for compatibility with Unix-based systems")
+	dataCenter := fsConfigureCommand.String("dataCenter", "", "assign writes to this dataCenter")
+	rack := fsConfigureCommand.String("rack", "", "assign writes to this rack")
+	dataNode := fsConfigureCommand.String("dataNode", "", "assign writes to this dataNode")
+	volumeGrowthCount := fsConfigureCommand.Int("volumeGrowthCount", 0, "the number of physical volumes to add if no writable volumes")
+	isDelete := fsConfigureCommand.Bool("delete", false, "delete the configuration by locationPrefix")
+	apply := fsConfigureCommand.Bool("apply", false, "update and apply filer configuration")
+	if err = fsConfigureCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient)
+	if err != nil {
+		return err
+	}
+
+	if *locationPrefix != "" {
+		infoAboutSimulationMode(writer, *apply, "-apply")
+		locConf := &filer_pb.FilerConf_PathConf{
+			LocationPrefix:           *locationPrefix,
+			Collection:               *collection,
+			Replication:              *replication,
+			Ttl:                      *ttl,
+			Fsync:                    *fsync,
+			MaxFileNameLength:        uint32(*maxFileNameLength),
+			DiskType:                 *diskType,
+			VolumeGrowthCount:        uint32(*volumeGrowthCount),
+			ReadOnly:                 *isReadOnly,
+			DataCenter:               *dataCenter,
+			Rack:                     *rack,
+			DataNode:                 *dataNode,
+			WormGracePeriodSeconds:   *wormGracePeriod,
+			WormRetentionTimeSeconds: *wormRetentionTime,
+		}
+
+		// worm is only carried when the flag is passed, so a rule that says nothing
+		// about it keeps inheriting from the enclosing path
+		fsConfigureCommand.Visit(func(f *flag.Flag) {
+			if f.Name == "worm" {
+				locConf.Worm = proto.Bool(*worm)
+			}
+		})
+
+		// check collection
+		if *collection != "" && strings.HasPrefix(*locationPrefix, "/buckets/") {
+			return fmt.Errorf("one s3 bucket goes to one collection and not customizable")
+		}
+
+		// check replication
+		if *replication != "" {
+			rp, err := super_block.NewReplicaPlacementFromString(*replication)
+			if err != nil {
+				return fmt.Errorf("parse replication %s: %v", *replication, err)
+			}
+			if *volumeGrowthCount%rp.GetCopyCount() != 0 {
+				return fmt.Errorf("volumeGrowthCount %d should be divided by replication copy count %d", *volumeGrowthCount, rp.GetCopyCount())
+			}
+		}
+
+		// check ttl
+		if *ttl != "" {
+			regex := "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)[mhdwMy]$"
+			match, _ := regexp.MatchString(regex, *ttl)
+
+			if !match {
+				return fmt.Errorf("ttl should be of the following format [1 to 255][unit] (e.g., 5m, 2h, 180d, 1w, 2y)")
+			}
+		}
+
+		// save it
+		if *isDelete {
+			fc.DeleteLocationConf(*locationPrefix)
+		} else {
+			fc.AddLocationConf(locConf)
+			// AddLocationConf merges these boolean fields with OR, which can never
+			// turn a flag off; let an explicitly passed false win, e.g. -readOnly=false
+			// to reopen a bucket that quota enforcement locked. worm does not belong
+			// here: it merges on presence, so the value set above already wins.
+			if mergedConf, found := fc.GetLocationConf(*locationPrefix); found {
+				fsConfigureCommand.Visit(func(f *flag.Flag) {
+					switch f.Name {
+					case "readOnly":
+						mergedConf.ReadOnly = *isReadOnly
+					case "fsync":
+						mergedConf.Fsync = *fsync
+					}
+				})
+			}
+		}
+	}
+
+	var buf2 bytes.Buffer
+	fc.ToText(&buf2)
+
+	fmt.Fprint(writer, buf2.String())
+	fmt.Fprintln(writer)
+
+	if *apply {
+
+		if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			return filer.SaveInsideFiler(context.Background(), client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes())
+		}); err != nil && err != filer_pb.ErrNotFound {
+			return err
+		}
+
+	}
+
+	return nil
+
+}
+
+func infoAboutSimulationMode(writer io.Writer, forceMode bool, forceModeOption string) {
+	if forceMode {
+		return
+	}
+	fmt.Fprintf(writer, "Running in simulation mode. Use \"%s\" option to apply the changes.\n", forceModeOption)
+}
+
+// handleDeprecatedForceFlag handles the deprecated -force flag by checking if it was
+// explicitly provided, printing a deprecation warning, and copying its
+// value to the new flag. This ensures that explicit -force=false takes precedence.
+func handleDeprecatedForceFlag(writer io.Writer, fs *flag.FlagSet, forceAlias *bool, applyFlag *bool) {
+	forceIsSet := false
+	applyIsSet := false
+	fs.Visit(func(f *flag.Flag) {
+		switch f.Name {
+		case "force":
+			forceIsSet = true
+		case "apply":
+			applyIsSet = true
+		}
+	})
+
+	if forceIsSet {
+		if applyIsSet {
+			fmt.Fprintf(writer, "WARNING: both -force and -apply are set. -force is deprecated and takes precedence. Please use only -apply.\n")
+		} else {
+			fmt.Fprintf(writer, "WARNING: -force is deprecated, please use -apply instead.\n")
+		}
+		*applyFlag = *forceAlias
+	}
+}

weed/shell/command_fs_distribute_chunks.go

@@ -0,0 +1,905 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"maps"
+	"math"
+	"net/http"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsDistributeChunks{})
+}
+
+type commandFsDistributeChunks struct{}
+
+func (c *commandFsDistributeChunks) Name() string {
+	return "fs.distributeChunks"
+}
+
+func (c *commandFsDistributeChunks) Help() string {
+	return `redistribute file chunks evenly across volume server nodes
+
+	Modes:
+	  primary      (default) balance chunk count per node using topology-based ownership.
+	               Note: with replication, ownership is derived from volume ID hashing
+	               (vid %% node count), which may not reflect the actual Assign target.
+	               Results can be approximate in replicated environments.
+
+	  replica      balance both ownership and replica copies across nodes.
+	               Step 1: owner-based balancing (same as primary mode).
+	               Step 2: additional moves to balance total copies (including replicas).
+	               Same ownership limitation as primary mode applies.
+
+	  round-robin  assign chunks to nodes by file offset order (chunk[0]->A, chunk[1]->B,
+	               chunk[2]->C, ...). Best mode for sequential read performance — ensures
+	               consecutive chunks are on different nodes for I/O pipelining.
+	               Recommended for replication environments as it does not depend on
+	               ownership calculation.
+
+	Files using chunk manifests (very large files) are handled by resolving manifests
+	to the underlying data chunks, redistributing those, then re-manifestizing before
+	updating the filer. Old manifest + data chunks are garbage-collected by the filer.
+
+	# analyze current distribution (dry-run, primary mode)
+	fs.distributeChunks -path=/buckets/my-bucket/large-file.dat
+
+	# apply redistribution
+	fs.distributeChunks -path=/buckets/my-bucket/large-file.dat -apply
+
+	# distribute across 5 nodes (instead of all)
+	fs.distributeChunks -path=/buckets/my-bucket/large-file.dat -nodes=5 -apply
+
+	# balance including replica copies
+	fs.distributeChunks -path=/buckets/my-bucket/large-file.dat -mode=replica -apply
+
+	# round-robin for sequential read performance (recommended with replication)
+	fs.distributeChunks -path=/buckets/my-bucket/large-file.dat -mode=round-robin -apply
+`
+}
+
+func (c *commandFsDistributeChunks) HasTag(CommandTag) bool {
+	return false
+}
+
+type chunkMove struct {
+	chunkIndex int
+	chunk      *filer_pb.FileChunk
+	fromNode   string
+	toNode     string
+}
+
+func (c *commandFsDistributeChunks) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	fsDistributeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	filePath := fsDistributeCommand.String("path", "", "file path to redistribute chunks")
+	apply := fsDistributeCommand.Bool("apply", false, "apply the redistribution")
+	targetNodeCount := fsDistributeCommand.Int("nodes", 0, "number of nodes to distribute across (0 = all nodes)")
+	mode := fsDistributeCommand.String("mode", "primary", "distribution mode: primary, replica, round-robin")
+	if err = fsDistributeCommand.Parse(args); err != nil {
+		return err
+	}
+	if *filePath == "" {
+		return fmt.Errorf("-path is required")
+	}
+	if *mode != "primary" && *mode != "replica" && *mode != "round-robin" {
+		return fmt.Errorf("-mode must be one of: primary, replica, round-robin")
+	}
+
+	infoAboutSimulationMode(writer, *apply, "-apply")
+
+	path, err := commandEnv.parseUrl(*filePath)
+	if err != nil {
+		return err
+	}
+	dir, name := util.FullPath(path).DirAndName()
+
+	entry, err := lookupFileEntry(commandEnv, dir, name)
+	if err != nil {
+		return fmt.Errorf("lookup %s: %v", path, err)
+	}
+
+	chunks, hadManifest, err := resolveEntryDataChunks(commandEnv, entry)
+	if err != nil {
+		return err
+	}
+	if len(chunks) == 0 {
+		fmt.Fprintf(writer, "File has no chunks (possibly stored inline)\n")
+		return nil
+	}
+	if hadManifest {
+		fmt.Fprintf(writer, "Resolved chunk manifests: %d top-level chunk(s) -> %d data chunks.\n", len(entry.GetChunks()), len(chunks))
+	}
+
+	volumeInfoMap, volumeNodesList, volumeToOwner, nodeList, err := collectVolumeTopology(commandEnv)
+	if err != nil {
+		return err
+	}
+
+	if len(nodeList) < 2 {
+		return fmt.Errorf("need at least 2 volume nodes, found %d", len(nodeList))
+	}
+	if *targetNodeCount < 0 {
+		return fmt.Errorf("-nodes must be >= 0 (0 = all nodes)")
+	}
+	if *targetNodeCount > len(nodeList) {
+		return fmt.Errorf("-nodes=%d exceeds available nodes (%d)", *targetNodeCount, len(nodeList))
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	if err != nil {
+		return err
+	}
+
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, *targetNodeCount)
+
+	totalChunks := len(chunks)
+	totalNodes := len(activeNodeList)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	fmt.Fprintf(writer, "\nFile: %s\n", path)
+	fmt.Fprintf(writer, "Total chunks: %d, Total nodes: %d, Mode: %s\n", totalChunks, totalNodes, *mode)
+	printCurrentDistribution(writer, nodeList, ownerCount, copiesCount, activeSet, *targetNodeCount, *mode, totalChunks, totalNodes, totalCopies)
+
+	moves, ownerTarget := planDistribution(*mode, writer, activeNodeList, nodeList, activeSet, ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList, totalChunks, totalNodes, totalCopies)
+
+	if len(moves) == 0 {
+		fmt.Fprintf(writer, "\nAlready well distributed. No moves needed.\n")
+		return nil
+	}
+
+	allRelevantNodes := relevantNodes(nodeList, activeSet, ownerCount)
+	printRedistributionPlan(writer, allRelevantNodes, ownerCount, ownerTarget, len(moves))
+
+	if !*apply {
+		fmt.Fprintf(writer, "\nTo apply, add -apply flag\n")
+		return nil
+	}
+
+	fmt.Fprintf(writer, "\nExecuting redistribution...\n")
+	defer util_http.GetGlobalHttpClient().CloseIdleConnections()
+
+	movedCount, err := executeChunkMoves(commandEnv, writer, moves, volumeInfoMap)
+	if err != nil {
+		return err
+	}
+	if movedCount == 0 {
+		fmt.Fprintf(writer, "\nNo chunks were moved successfully.\n")
+		return nil
+	}
+
+	finalChunks := chunks
+	if hadManifest {
+		// no chunk deleter here: a failed fold reports the blobs it saved
+		remanifested, manErr := filer.MaybeManifestize(newShellSaveAsChunk(commandEnv), nil, chunks)
+		if manErr != nil {
+			fmt.Fprintf(writer, "WARNING: re-manifestize failed: %v. Writing flat chunk list.\n", manErr)
+		} else {
+			finalChunks = remanifested
+			fmt.Fprintf(writer, "Re-manifestized %d data chunks into %d top-level chunk(s).\n", len(chunks), len(finalChunks))
+		}
+	}
+	entry.Chunks = finalChunks
+
+	fmt.Fprintf(writer, "\nUpdating filer metadata (%d chunks moved)...\n", movedCount)
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
+			Directory: dir,
+			Entry:     entry,
+		})
+	})
+	if err != nil {
+		fmt.Fprintf(writer, "FAILED to update filer metadata: %v\n", err)
+		fmt.Fprintf(writer, "WARNING: New chunks were uploaded but metadata was not updated.\n")
+		fmt.Fprintf(writer, "The original file is still intact. New chunks are orphaned.\n")
+		return err
+	}
+	fmt.Fprintf(writer, "Filer metadata updated successfully.\n")
+
+	fmt.Fprintf(writer, "\nRedistribution complete. %d chunks moved.\n", movedCount)
+	return nil
+}
+
+// lookupFileEntry fetches the filer entry for the given directory and name.
+func lookupFileEntry(commandEnv *CommandEnv, dir, name string) (*filer_pb.Entry, error) {
+	var entry *filer_pb.Entry
+	err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: dir,
+			Name:      name,
+		})
+		if err != nil {
+			return err
+		}
+		entry = resp.Entry
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	if entry == nil {
+		return nil, fmt.Errorf("lookup for %s/%s returned no entry", dir, name)
+	}
+	if entry.IsDirectory {
+		return nil, fmt.Errorf("%s/%s is a directory", dir, name)
+	}
+	return entry, nil
+}
+
+// resolveEntryDataChunks returns the flattened data chunks of an entry. If the
+// entry contains manifest chunks, their contents are resolved. hadManifest
+// signals whether any manifest chunks were resolved, so the caller knows to
+// re-manifestize after redistribution.
+func resolveEntryDataChunks(commandEnv *CommandEnv, entry *filer_pb.Entry) (dataChunks []*filer_pb.FileChunk, hadManifest bool, err error) {
+	chunks := entry.GetChunks()
+	if !filer.HasChunkManifest(chunks) {
+		return chunks, false, nil
+	}
+	dataChunks, _, err = filer.ResolveChunkManifest(context.Background(), filer.LookupFn(commandEnv), chunks, 0, math.MaxInt64, nil)
+	if err != nil {
+		return nil, true, fmt.Errorf("resolve chunk manifest: %v", err)
+	}
+	return dataChunks, true, nil
+}
+
+// collectVolumeTopology queries the master for cluster topology and returns per-volume
+// metadata, per-volume node lists, per-volume ownership, and the sorted node list.
+func collectVolumeTopology(commandEnv *CommandEnv) (
+	volumeInfoMap map[uint32]*master_pb.VolumeInformationMessage,
+	volumeNodesList map[uint32][]string,
+	volumeToOwner map[uint32]string,
+	nodeList []string,
+	err error,
+) {
+	topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return nil, nil, nil, nil, fmt.Errorf("collect topology: %v", err)
+	}
+
+	volumeInfoMap = make(map[uint32]*master_pb.VolumeInformationMessage)
+	volumeNodesList = make(map[uint32][]string)
+	allNodes := make(map[string]bool)
+
+	for _, dc := range topoInfo.GetDataCenterInfos() {
+		for _, rack := range dc.GetRackInfos() {
+			for _, dn := range rack.GetDataNodeInfos() {
+				nodeId := dn.GetId()
+				allNodes[nodeId] = true
+				for _, disk := range dn.GetDiskInfos() {
+					for _, vol := range disk.GetVolumeInfos() {
+						volumeNodesList[vol.GetId()] = append(volumeNodesList[vol.GetId()], nodeId)
+						if _, exists := volumeInfoMap[vol.GetId()]; !exists {
+							volumeInfoMap[vol.GetId()] = vol
+						}
+					}
+				}
+			}
+		}
+	}
+
+	volumeToOwner = make(map[uint32]string)
+	for vid, nodes := range volumeNodesList {
+		sort.Strings(nodes)
+		volumeToOwner[vid] = nodes[int(vid)%len(nodes)]
+	}
+
+	nodeList = make([]string, 0, len(allNodes))
+	for node := range allNodes {
+		nodeList = append(nodeList, node)
+	}
+	sort.Strings(nodeList)
+
+	return volumeInfoMap, volumeNodesList, volumeToOwner, nodeList, nil
+}
+
+// buildDistributionCounts maps each chunk to its owning node and tallies owner/copy counts.
+func buildDistributionCounts(
+	chunks []*filer_pb.FileChunk,
+	volumeToOwner map[uint32]string,
+	volumeNodesList map[uint32][]string,
+) (chunkToNode map[int]string, ownerCount map[string]int, copiesCount map[string]int, err error) {
+	chunkToNode = make(map[int]string)
+	ownerCount = make(map[string]int)
+	copiesCount = make(map[string]int)
+
+	for i, chunk := range chunks {
+		var vid uint32
+		if chunk.Fid == nil {
+			fileId, parseErr := needle.ParseFileIdFromString(chunk.GetFileIdString())
+			if parseErr != nil {
+				return nil, nil, nil, fmt.Errorf("failed to parse file id for chunk %d: %v", i, parseErr)
+			}
+			vid = uint32(fileId.VolumeId)
+		} else {
+			vid = chunk.Fid.GetVolumeId()
+		}
+		owner, ok := volumeToOwner[vid]
+		if !ok {
+			return nil, nil, nil, fmt.Errorf("volume %d not found in topology", vid)
+		}
+		chunkToNode[i] = owner
+		ownerCount[owner]++
+		for _, node := range volumeNodesList[vid] {
+			copiesCount[node]++
+		}
+	}
+	return chunkToNode, ownerCount, copiesCount, nil
+}
+
+// selectActiveNodes picks up to targetNodeCount nodes to participate in redistribution.
+// When targetNodeCount is 0 all nodes are active.
+func selectActiveNodes(nodeList []string, ownerCount map[string]int, targetNodeCount int) (activeNodeList []string, activeSet map[string]bool) {
+	activeNodeList = nodeList
+	if targetNodeCount > 0 && targetNodeCount < len(nodeList) {
+		var withChunks, withoutChunks []string
+		for _, node := range nodeList {
+			if ownerCount[node] > 0 {
+				withChunks = append(withChunks, node)
+			} else {
+				withoutChunks = append(withoutChunks, node)
+			}
+		}
+		n := targetNodeCount
+		if len(withChunks) >= n {
+			sort.Slice(withChunks, func(i, j int) bool {
+				return ownerCount[withChunks[i]] > ownerCount[withChunks[j]]
+			})
+			activeNodeList = withChunks[:n]
+		} else {
+			activeNodeList = append(withChunks, withoutChunks[:n-len(withChunks)]...)
+		}
+		sort.Strings(activeNodeList)
+	}
+
+	activeSet = make(map[string]bool, len(activeNodeList))
+	for _, node := range activeNodeList {
+		activeSet[node] = true
+	}
+	return activeNodeList, activeSet
+}
+
+// printCurrentDistribution writes the per-node chunk/copy table to writer.
+func printCurrentDistribution(
+	writer io.Writer,
+	nodeList []string,
+	ownerCount, copiesCount map[string]int,
+	activeSet map[string]bool,
+	targetNodeCount int,
+	mode string,
+	totalChunks, totalNodes, totalCopies int,
+) {
+	maxOwner := 0
+	for _, cnt := range ownerCount {
+		if cnt > maxOwner {
+			maxOwner = cnt
+		}
+	}
+
+	fmt.Fprintf(writer, "\nCurrent distribution:\n")
+	for _, node := range nodeList {
+		oc := ownerCount[node]
+		cc := copiesCount[node]
+		bar := strings.Repeat("#", int(math.Ceil(float64(oc)*30/math.Max(float64(maxOwner), 1))))
+		marker := " "
+		if targetNodeCount > 0 && !activeSet[node] {
+			marker = "-"
+		}
+		if mode == "replica" {
+			fmt.Fprintf(writer, "  %s %-20s %4d chunks  %4d copies  %-30s\n", marker, shortName(node), oc, cc, bar)
+		} else {
+			fmt.Fprintf(writer, "  %s %-20s %4d chunks  %-30s\n", marker, shortName(node), oc, bar)
+		}
+	}
+	fmt.Fprintf(writer, "  Ideal: ~%.0f chunks per node", math.Ceil(float64(totalChunks)/float64(totalNodes)))
+	if mode == "replica" {
+		fmt.Fprintf(writer, ", ~%.0f copies per node", math.Ceil(float64(totalCopies)/float64(totalNodes)))
+	}
+	fmt.Fprintln(writer)
+}
+
+// planDistribution computes the set of chunk moves required for the chosen mode.
+func planDistribution(
+	mode string,
+	writer io.Writer,
+	activeNodeList, nodeList []string,
+	activeSet map[string]bool,
+	ownerCount, copiesCount map[string]int,
+	chunkToNode map[int]string,
+	chunks []*filer_pb.FileChunk,
+	volumeNodesList map[uint32][]string,
+	totalChunks, totalNodes, totalCopies int,
+) (moves []chunkMove, ownerTarget map[string]int) {
+	type nodeExcess struct {
+		node   string
+		excess int
+	}
+
+	allRelevantNodes := relevantNodes(nodeList, activeSet, ownerCount)
+	ownerTarget = make(map[string]int)
+
+	switch mode {
+	case "primary":
+		ownerTarget = computeOwnerTarget(activeNodeList, nodeList, activeSet, totalChunks, totalNodes)
+		moves = planOwnerMoves(ownerCount, ownerTarget, chunkToNode, chunks, allRelevantNodes)
+
+	case "replica":
+		ownerTarget = computeOwnerTarget(activeNodeList, nodeList, activeSet, totalChunks, totalNodes)
+		moves = planOwnerMoves(ownerCount, ownerTarget, chunkToNode, chunks, allRelevantNodes)
+
+		copiesMin := totalCopies / totalNodes
+		copiesRemainder := totalCopies % totalNodes
+		copiesTarget := make(map[string]int)
+		for i, node := range activeNodeList {
+			if i < copiesRemainder {
+				copiesTarget[node] = copiesMin + 1
+			} else {
+				copiesTarget[node] = copiesMin
+			}
+		}
+
+		alreadyMoving := make(map[int]bool)
+		for _, mv := range moves {
+			alreadyMoving[mv.chunkIndex] = true
+		}
+
+		// local copy so the planner does not mutate the caller's copiesCount
+		localCopiesCount := maps.Clone(copiesCount)
+
+		var copiesOver, copiesUnder []nodeExcess
+		for _, node := range activeNodeList {
+			diff := localCopiesCount[node] - copiesTarget[node]
+			if diff > 0 {
+				copiesOver = append(copiesOver, nodeExcess{node, diff})
+			} else if diff < 0 {
+				copiesUnder = append(copiesUnder, nodeExcess{node, -diff})
+			}
+		}
+		sort.Slice(copiesOver, func(i, j int) bool {
+			return copiesOver[i].excess > copiesOver[j].excess
+		})
+
+		for oi := range copiesOver {
+			over := &copiesOver[oi]
+			if over.excess <= 0 {
+				continue
+			}
+			for idx, chunk := range chunks {
+				if over.excess <= 0 {
+					break
+				}
+				if alreadyMoving[idx] {
+					continue
+				}
+				vid, vidErr := chunkVolumeId(chunk)
+				if vidErr != nil {
+					continue
+				}
+				if !slices.Contains(volumeNodesList[vid], over.node) {
+					continue
+				}
+				sort.Slice(copiesUnder, func(i, j int) bool {
+					return copiesUnder[i].excess > copiesUnder[j].excess
+				})
+				for j := range copiesUnder {
+					if copiesUnder[j].excess > 0 {
+						moves = append(moves, chunkMove{
+							chunkIndex: idx,
+							chunk:      chunk,
+							fromNode:   chunkToNode[idx],
+							toNode:     copiesUnder[j].node,
+						})
+						alreadyMoving[idx] = true
+						for _, srcNode := range volumeNodesList[vid] {
+							localCopiesCount[srcNode]--
+							for k := range copiesOver {
+								if copiesOver[k].node == srcNode {
+									copiesOver[k].excess--
+								}
+							}
+						}
+						copiesUnder[j].excess--
+						localCopiesCount[copiesUnder[j].node]++
+						over.excess--
+						break
+					}
+				}
+			}
+		}
+
+	case "round-robin":
+		type chunkWithIndex struct {
+			index  int
+			offset int64
+		}
+		sortedChunks := make([]chunkWithIndex, totalChunks)
+		for i, chunk := range chunks {
+			sortedChunks[i] = chunkWithIndex{index: i, offset: chunk.GetOffset()}
+		}
+		sort.Slice(sortedChunks, func(i, j int) bool {
+			return sortedChunks[i].offset < sortedChunks[j].offset
+		})
+
+		for i, sc := range sortedChunks {
+			targetNode := activeNodeList[i%totalNodes]
+			currentOwner := chunkToNode[sc.index]
+			ownerTarget[targetNode]++
+			if currentOwner != targetNode {
+				moves = append(moves, chunkMove{
+					chunkIndex: sc.index,
+					chunk:      chunks[sc.index],
+					fromNode:   currentOwner,
+					toNode:     targetNode,
+				})
+			}
+		}
+
+		fmt.Fprintf(writer, "\nRound-robin assignment (sequential read optimized):\n")
+		fmt.Fprintf(writer, "  First assignments: ")
+		for pos := range sortedChunks {
+			if pos >= 12 {
+				fmt.Fprintf(writer, "... ")
+				break
+			}
+			fmt.Fprintf(writer, "%s ", shortName(activeNodeList[pos%totalNodes]))
+		}
+		fmt.Fprintf(writer, "\n  Pattern: ")
+		for i := 0; i < totalNodes && i < 6; i++ {
+			fmt.Fprintf(writer, "%s ", shortName(activeNodeList[i]))
+		}
+		if totalNodes > 6 {
+			fmt.Fprintf(writer, "...")
+		}
+		fmt.Fprintf(writer, "(repeating)\n")
+	}
+
+	return moves, ownerTarget
+}
+
+// printRedistributionPlan writes the before/after chunk count table for each node.
+func printRedistributionPlan(writer io.Writer, allRelevantNodes []string, ownerCount, ownerTarget map[string]int, numMoves int) {
+	fmt.Fprintf(writer, "\nRedistribution plan: %d chunks to move\n", numMoves)
+	for _, node := range allRelevantNodes {
+		current := ownerCount[node]
+		target := ownerTarget[node]
+		diff := target - current
+		sign := ""
+		if diff > 0 {
+			sign = "+"
+		}
+		fmt.Fprintf(writer, "  %-25s %4d -> %4d  (%s%d)\n", shortName(node), current, target, sign, diff)
+	}
+}
+
+// relevantNodes returns nodes that are either active or currently hold chunks.
+func relevantNodes(nodeList []string, activeSet map[string]bool, ownerCount map[string]int) []string {
+	relevant := make([]string, 0, len(nodeList))
+	for _, node := range nodeList {
+		if activeSet[node] || ownerCount[node] > 0 {
+			relevant = append(relevant, node)
+		}
+	}
+	return relevant
+}
+
+// chunkVolumeId extracts the volume ID from a chunk, falling back to parsing
+// the legacy FileId string when Fid is nil.
+func chunkVolumeId(chunk *filer_pb.FileChunk) (uint32, error) {
+	if chunk.Fid != nil {
+		return chunk.Fid.GetVolumeId(), nil
+	}
+	fileId, err := needle.ParseFileIdFromString(chunk.GetFileIdString())
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse file id %q: %v", chunk.GetFileIdString(), err)
+	}
+	return uint32(fileId.VolumeId), nil
+}
+
+// shortName returns the hostname portion (before the first ".") of a node ID.
+func shortName(nodeID string) string {
+	return strings.Split(nodeID, ".")[0]
+}
+
+// computeOwnerTarget returns the target chunk count per node for owner-based balancing.
+func computeOwnerTarget(activeNodeList, nodeList []string, activeSet map[string]bool, totalChunks, totalNodes int) map[string]int {
+	target := make(map[string]int)
+	perNode := totalChunks / totalNodes
+	remainder := totalChunks % totalNodes
+	for i, node := range activeNodeList {
+		if i < remainder {
+			target[node] = perNode + 1
+		} else {
+			target[node] = perNode
+		}
+	}
+	for _, node := range nodeList {
+		if !activeSet[node] {
+			target[node] = 0
+		}
+	}
+	return target
+}
+
+// executeChunkMoves runs all moves concurrently and returns the number of successfully moved chunks.
+// Old chunk cleanup is handled automatically by the filer: filer.UpdateEntry calls
+// deleteChunksIfNotNew which removes chunks present in the old entry but not the new one.
+func executeChunkMoves(
+	commandEnv *CommandEnv,
+	writer io.Writer,
+	moves []chunkMove,
+	volumeInfoMap map[uint32]*master_pb.VolumeInformationMessage,
+) (int, error) {
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, fmt.Errorf("create uploader: %v", err)
+	}
+
+	var mu sync.Mutex
+	movedCount := 0
+	ewg := NewErrorWaitGroup(DefaultMaxParallelization)
+
+	for i, mv := range moves {
+		ewg.Add(func() error {
+			chunk := mv.chunk
+			oldFidStr := chunk.GetFileIdString()
+			prefix := fmt.Sprintf("  [%d/%d] Moving %s from %s to %s",
+				i+1, len(moves), oldFidStr,
+				shortName(mv.fromNode), shortName(mv.toNode),
+			)
+
+			fail := func(msg string) {
+				mu.Lock()
+				fmt.Fprintf(writer, "%s ... FAILED (%s)\n", prefix, msg)
+				mu.Unlock()
+			}
+
+			oldFid, parseErr := needle.ParseFileIdFromString(oldFidStr)
+			if parseErr != nil {
+				fail(fmt.Sprintf("parse fid: %v", parseErr))
+				return nil
+			}
+
+			downloadURLs, lookupErr := commandEnv.MasterClient.LookupVolumeServerUrl(oldFid.VolumeId.String())
+			if lookupErr != nil || len(downloadURLs) == 0 {
+				fail(fmt.Sprintf("lookup source: %v", lookupErr))
+				return nil
+			}
+
+			// dlCancel must be called after reader.Close() — the response body is
+			// streamed into the uploader, so cancelling the context before the
+			// body is fully consumed causes "context canceled" upload failures.
+			dlCtx, dlCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+			var resp *http.Response
+			var reader io.ReadCloser
+			var readErr error
+			readJwt := filer.JwtForVolumeServer(oldFidStr)
+			for _, serverURL := range downloadURLs {
+				var dlReq *http.Request
+				dlReq, readErr = http.NewRequestWithContext(dlCtx, http.MethodGet, fmt.Sprintf("http://%s/%s", serverURL, oldFidStr), nil)
+				if readErr != nil {
+					continue
+				}
+				dlReq.Header.Add("Accept-Encoding", "gzip")
+				if readJwt != "" {
+					dlReq.Header.Set("Authorization", security.BearerPrefix+readJwt)
+				}
+				resp, readErr = util_http.GetGlobalHttpClient().Do(dlReq)
+				if readErr == nil && resp.StatusCode >= 400 {
+					util_http.CloseResponse(resp)
+					readErr = fmt.Errorf("download %s: %s", serverURL, resp.Status)
+				}
+				if readErr == nil {
+					reader = resp.Body
+					break
+				}
+			}
+			if readErr != nil {
+				dlCancel()
+				fail(fmt.Sprintf("download: %v", readErr))
+				return nil
+			}
+
+			contentType := resp.Header.Get("Content-Type")
+			isCompressed := resp.Header.Get("Content-Encoding") == "gzip"
+			md5Hash := resp.Header.Get("Content-MD5")
+			var filename string
+			if cd := resp.Header.Get("Content-Disposition"); len(cd) > 0 {
+				if _, after, found := strings.Cut(cd, "filename="); found {
+					filename = strings.Trim(after, "\"")
+				}
+			}
+
+			// Buffer the full response body so the download context can be
+			// cancelled before the upload starts — otherwise the 5-minute
+			// dlCtx timeout would also cancel the (potentially slow) upload.
+			bodyBytes, readBodyErr := io.ReadAll(reader)
+			reader.Close()
+			util_http.CloseResponse(resp)
+			dlCancel()
+			if readBodyErr != nil {
+				fail(fmt.Sprintf("read body: %v", readBodyErr))
+				return nil
+			}
+
+			var replication, collection string
+			if vInfo, ok := volumeInfoMap[uint32(oldFid.VolumeId)]; ok {
+				replication = fmt.Sprintf("%03d", vInfo.GetReplicaPlacement())
+				collection = vInfo.GetCollection()
+			}
+
+			assignResult, assignErr := operation.Assign(context.Background(), commandEnv.MasterClient.GetMaster, commandEnv.option.GrpcDialOption,
+				&operation.VolumeAssignRequest{
+					Count:       1,
+					Replication: replication,
+					Collection:  collection,
+					DataNode:    mv.toNode,
+				})
+			if assignErr != nil {
+				fail(fmt.Sprintf("assign: %v", assignErr))
+				return nil
+			}
+
+			uploadURL := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
+
+			var jwt security.EncodedJwt
+			if assignResult.Auth != "" {
+				jwt = assignResult.Auth
+			} else {
+				v := util.GetViper()
+				signingKey := v.GetString("jwt.signing.key")
+				if signingKey != "" {
+					expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")
+					jwt = security.GenJwtForVolumeServer(security.SigningKey(signingKey), expiresAfterSec, assignResult.Fid)
+				}
+			}
+
+			_, uploadErr, _ := uploader.Upload(context.Background(), bytes.NewReader(bodyBytes), &operation.UploadOption{
+				UploadUrl:         uploadURL,
+				Filename:          filename,
+				IsInputCompressed: isCompressed,
+				Cipher:            false,
+				MimeType:          contentType,
+				Md5:               md5Hash,
+				Jwt:               jwt,
+			})
+
+			if uploadErr != nil {
+				fail(fmt.Sprintf("upload: %v", uploadErr))
+				return nil
+			}
+
+			newFid, fidParseErr := needle.ParseFileIdFromString(assignResult.Fid)
+			if fidParseErr != nil {
+				fail(fmt.Sprintf("parse new fid: %v", fidParseErr))
+				return nil
+			}
+
+			mu.Lock()
+			movedCount++
+			chunk.Fid = &filer_pb.FileId{
+				VolumeId: uint32(newFid.VolumeId),
+				FileKey:  uint64(newFid.Key),
+				Cookie:   uint32(newFid.Cookie),
+			}
+			chunk.FileId = ""
+			fmt.Fprintf(writer, "%s ... OK -> %s\n", prefix, assignResult.Fid)
+			mu.Unlock()
+
+			return nil
+		})
+	}
+	_ = ewg.Wait()
+	return movedCount, nil
+}
+
+// newShellSaveAsChunk returns a SaveDataAsChunk function for use with
+// filer.MaybeManifestize from shell context. The master selects placement —
+// manifest chunks are metadata and don't need fan-out distribution.
+func newShellSaveAsChunk(commandEnv *CommandEnv) filer.SaveDataAsChunkFunctionType {
+	return filer.SaveDataAsChunkFunctionType(func(reader io.Reader, name string, offset int64, tsNs int64, _ uint64) (*filer_pb.FileChunk, error) {
+		uploader, err := operation.NewUploader()
+		if err != nil {
+			return nil, fmt.Errorf("create uploader: %v", err)
+		}
+		data, err := io.ReadAll(reader)
+		if err != nil {
+			return nil, fmt.Errorf("read manifest data: %v", err)
+		}
+		assignResult, err := operation.Assign(context.Background(), commandEnv.MasterClient.GetMaster, commandEnv.option.GrpcDialOption,
+			&operation.VolumeAssignRequest{Count: 1})
+		if err != nil {
+			return nil, fmt.Errorf("assign manifest chunk: %v", err)
+		}
+		var jwt security.EncodedJwt
+		if assignResult.Auth != "" {
+			jwt = assignResult.Auth
+		} else {
+			v := util.GetViper()
+			signingKey := v.GetString("jwt.signing.key")
+			if signingKey != "" {
+				expiresAfterSec := v.GetInt("jwt.signing.expires_after_seconds")
+				jwt = security.GenJwtForVolumeServer(security.SigningKey(signingKey), expiresAfterSec, assignResult.Fid)
+			}
+		}
+		uploadResult, uploadErr, _ := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+			UploadUrl: fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid),
+			Filename:  name,
+			Jwt:       jwt,
+		})
+		if uploadErr != nil {
+			return nil, fmt.Errorf("upload manifest chunk: %v", uploadErr)
+		}
+		if uploadResult.Error != "" {
+			return nil, fmt.Errorf("upload manifest chunk: %s", uploadResult.Error)
+		}
+		return uploadResult.ToPbFileChunk(assignResult.Fid, offset, tsNs), nil
+	})
+}
+
+// planOwnerMoves generates moves to balance owner-based chunk distribution.
+func planOwnerMoves(ownerCount, ownerTarget map[string]int, chunkToNode map[int]string, chunks []*filer_pb.FileChunk, allRelevantNodes []string) []chunkMove {
+	type nodeExcess struct {
+		node   string
+		excess int
+	}
+	var overNodes, underNodes []nodeExcess
+	for _, node := range allRelevantNodes {
+		diff := ownerCount[node] - ownerTarget[node]
+		if diff > 0 {
+			overNodes = append(overNodes, nodeExcess{node, diff})
+		} else if diff < 0 {
+			underNodes = append(underNodes, nodeExcess{node, -diff})
+		}
+	}
+
+	var moves []chunkMove
+	for _, over := range overNodes {
+		toMove := over.excess
+		var nodeChunks []int
+		for i, node := range chunkToNode {
+			if node == over.node {
+				nodeChunks = append(nodeChunks, i)
+			}
+		}
+		moved := 0
+		for _, idx := range nodeChunks {
+			if moved >= toMove {
+				break
+			}
+			for j := range underNodes {
+				if underNodes[j].excess > 0 {
+					moves = append(moves, chunkMove{
+						chunkIndex: idx,
+						chunk:      chunks[idx],
+						fromNode:   over.node,
+						toNode:     underNodes[j].node,
+					})
+					underNodes[j].excess--
+					moved++
+					break
+				}
+			}
+		}
+	}
+	return moves
+}

weed/shell/command_fs_distribute_chunks_test.go

@@ -0,0 +1,426 @@
+package shell
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// makeChunk builds a FileChunk with the given volume ID, file key, and byte offset.
+func makeChunk(vid uint32, key uint64, offset int64) *filer_pb.FileChunk {
+	return &filer_pb.FileChunk{
+		Fid:    &filer_pb.FileId{VolumeId: vid, FileKey: key, Cookie: 1},
+		Offset: offset,
+		Size:   1024 * 1024 * 16,
+	}
+}
+
+// threeNodeTopology returns a minimal topology for 3 nodes with 3 volumes (one per node).
+//
+//	node A owns vol 1, node B owns vol 2, node C owns vol 3
+//	no replication: each volume exists on exactly one node.
+func threeNodeTopology() (volumeToOwner map[uint32]string, volumeNodesList map[uint32][]string) {
+	volumeToOwner = map[uint32]string{
+		1: "nodeA",
+		2: "nodeB",
+		3: "nodeC",
+	}
+	volumeNodesList = map[uint32][]string{
+		1: {"nodeA"},
+		2: {"nodeB"},
+		3: {"nodeC"},
+	}
+	return volumeToOwner, volumeNodesList
+}
+
+// ── shortName ────────────────────────────────────────────────────────────────
+
+func TestShortName(t *testing.T) {
+	tests := []struct {
+		input string
+		want  string
+	}{
+		{"nodeA.example.com", "nodeA"},
+		{"nodeA", "nodeA"},
+		{"10.0.0.1", "10"},
+		{"", ""},
+	}
+	for _, tt := range tests {
+		assert.Equal(t, tt.want, shortName(tt.input), "input=%q", tt.input)
+	}
+}
+
+// ── relevantNodes ────────────────────────────────────────────────────────────
+
+func TestRelevantNodes(t *testing.T) {
+	nodeList := []string{"nodeA", "nodeB", "nodeC", "nodeD"}
+	activeSet := map[string]bool{"nodeA": true, "nodeB": true}
+	ownerCount := map[string]int{"nodeA": 3, "nodeC": 1} // nodeC has chunks but isn't active
+
+	got := relevantNodes(nodeList, activeSet, ownerCount)
+
+	// nodeA (active+chunks), nodeB (active), nodeC (inactive but has chunks)
+	// nodeD (inactive, no chunks) → excluded
+	assert.ElementsMatch(t, []string{"nodeA", "nodeB", "nodeC"}, got)
+}
+
+// ── computeOwnerTarget ───────────────────────────────────────────────────────
+
+func TestComputeOwnerTarget_EvenDivision(t *testing.T) {
+	nodes := []string{"nodeA", "nodeB", "nodeC"}
+	activeSet := map[string]bool{"nodeA": true, "nodeB": true, "nodeC": true}
+
+	target := computeOwnerTarget(nodes, nodes, activeSet, 9, 3)
+
+	assert.Equal(t, 3, target["nodeA"])
+	assert.Equal(t, 3, target["nodeB"])
+	assert.Equal(t, 3, target["nodeC"])
+}
+
+func TestComputeOwnerTarget_UnevenDivision(t *testing.T) {
+	nodes := []string{"nodeA", "nodeB", "nodeC"}
+	activeSet := map[string]bool{"nodeA": true, "nodeB": true, "nodeC": true}
+
+	// 10 chunks / 3 nodes → 3,3,4 (remainder goes to first nodes)
+	target := computeOwnerTarget(nodes, nodes, activeSet, 10, 3)
+
+	total := target["nodeA"] + target["nodeB"] + target["nodeC"]
+	assert.Equal(t, 10, total)
+	for _, node := range nodes {
+		assert.True(t, target[node] == 3 || target[node] == 4,
+			"node %s has unexpected target %d", node, target[node])
+	}
+}
+
+func TestComputeOwnerTarget_InactiveNodesDrainToZero(t *testing.T) {
+	activeNodes := []string{"nodeA", "nodeB"}
+	allNodes := []string{"nodeA", "nodeB", "nodeC"}
+	activeSet := map[string]bool{"nodeA": true, "nodeB": true}
+
+	target := computeOwnerTarget(activeNodes, allNodes, activeSet, 4, 2)
+
+	assert.Equal(t, 0, target["nodeC"], "inactive node should have target 0")
+	assert.Equal(t, 2, target["nodeA"])
+	assert.Equal(t, 2, target["nodeB"])
+}
+
+// ── buildDistributionCounts ──────────────────────────────────────────────────
+
+func TestBuildDistributionCounts(t *testing.T) {
+	volumeToOwner, volumeNodesList := threeNodeTopology()
+
+	// 4 chunks on vol1 (nodeA), 2 on vol2 (nodeB), 0 on nodeC
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(1, 1, 0),
+		makeChunk(1, 2, 1),
+		makeChunk(1, 3, 2),
+		makeChunk(1, 4, 3),
+		makeChunk(2, 5, 4),
+		makeChunk(2, 6, 5),
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+
+	require.NoError(t, err)
+	assert.Equal(t, 4, ownerCount["nodeA"])
+	assert.Equal(t, 2, ownerCount["nodeB"])
+	assert.Equal(t, 0, ownerCount["nodeC"])
+
+	// each chunk maps to its owner
+	assert.Equal(t, "nodeA", chunkToNode[0])
+	assert.Equal(t, "nodeB", chunkToNode[4])
+
+	// no replication → copies == owner counts
+	assert.Equal(t, 4, copiesCount["nodeA"])
+	assert.Equal(t, 2, copiesCount["nodeB"])
+}
+
+func TestBuildDistributionCounts_NilFidFallback(t *testing.T) {
+	volumeToOwner := map[uint32]string{3: "nodeA"}
+	volumeNodesList := map[uint32][]string{3: {"nodeA"}}
+
+	// chunk with nil Fid but a valid legacy FileId string "3,0123456789abcdef01"
+	chunks := []*filer_pb.FileChunk{
+		{Fid: nil, FileId: "3,0123456789abcdef01"},
+	}
+
+	_, ownerCount, _, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+
+	require.NoError(t, err)
+	assert.Equal(t, 1, ownerCount["nodeA"])
+}
+
+// ── selectActiveNodes ────────────────────────────────────────────────────────
+
+func TestSelectActiveNodes_AllNodes(t *testing.T) {
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	ownerCount := map[string]int{"nodeA": 5, "nodeB": 3, "nodeC": 2}
+
+	active, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+
+	assert.Equal(t, nodeList, active)
+	assert.Len(t, activeSet, 3)
+}
+
+func TestSelectActiveNodes_LimitToTwo(t *testing.T) {
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	ownerCount := map[string]int{"nodeA": 5, "nodeB": 3, "nodeC": 0}
+
+	active, activeSet := selectActiveNodes(nodeList, ownerCount, 2)
+
+	assert.Len(t, active, 2)
+	// nodeA and nodeB have chunks and are the top-2 heaviest
+	assert.True(t, activeSet["nodeA"])
+	assert.True(t, activeSet["nodeB"])
+	assert.False(t, activeSet["nodeC"])
+}
+
+// ── planOwnerMoves ───────────────────────────────────────────────────────────
+
+func TestPlanOwnerMoves_BasicBalance(t *testing.T) {
+	// nodeA has 5 chunks, nodeB/C have 0; target is 2,2,1 (or similar even split of 5)
+	ownerCount := map[string]int{"nodeA": 5, "nodeB": 0, "nodeC": 0}
+	ownerTarget := map[string]int{"nodeA": 1, "nodeB": 2, "nodeC": 2}
+	nodes := []string{"nodeA", "nodeB", "nodeC"}
+
+	chunkToNode := map[int]string{0: "nodeA", 1: "nodeA", 2: "nodeA", 3: "nodeA", 4: "nodeA"}
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(1, 1, 0),
+		makeChunk(1, 2, 1),
+		makeChunk(1, 3, 2),
+		makeChunk(1, 4, 3),
+		makeChunk(1, 5, 4),
+	}
+
+	moves := planOwnerMoves(ownerCount, ownerTarget, chunkToNode, chunks, nodes)
+
+	// 4 chunks must leave nodeA
+	assert.Len(t, moves, 4)
+	for _, mv := range moves {
+		assert.Equal(t, "nodeA", mv.fromNode)
+		assert.NotEqual(t, "nodeA", mv.toNode)
+	}
+}
+
+func TestPlanOwnerMoves_AlreadyBalanced(t *testing.T) {
+	ownerCount := map[string]int{"nodeA": 2, "nodeB": 2, "nodeC": 2}
+	ownerTarget := map[string]int{"nodeA": 2, "nodeB": 2, "nodeC": 2}
+	nodes := []string{"nodeA", "nodeB", "nodeC"}
+	chunkToNode := map[int]string{0: "nodeA", 1: "nodeA", 2: "nodeB", 3: "nodeB", 4: "nodeC", 5: "nodeC"}
+	chunks := make([]*filer_pb.FileChunk, 6)
+	for i := range chunks {
+		chunks[i] = makeChunk(1, uint64(i+1), int64(i))
+	}
+
+	moves := planOwnerMoves(ownerCount, ownerTarget, chunkToNode, chunks, nodes)
+
+	assert.Empty(t, moves)
+}
+
+// ── planDistribution: primary ─────────────────────────────────────────────────
+
+func TestPlanDistribution_Primary_BalancesChunks(t *testing.T) {
+	// 9 chunks, all on nodeA; 3 nodes; primary should balance to 3 each
+	volumeToOwner, volumeNodesList := threeNodeTopology()
+	chunks := make([]*filer_pb.FileChunk, 9)
+	for i := range chunks {
+		chunks[i] = makeChunk(1, uint64(i+1), int64(i)) // all on vol1 → nodeA
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	require.NoError(t, err)
+
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+	totalChunks := len(chunks)
+	totalNodes := len(activeNodeList)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	moves, ownerTarget := planDistribution("primary", &bytes.Buffer{},
+		activeNodeList, nodeList, activeSet,
+		ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList,
+		totalChunks, totalNodes, totalCopies)
+
+	assert.Equal(t, 3, ownerTarget["nodeA"])
+	assert.Equal(t, 3, ownerTarget["nodeB"])
+	assert.Equal(t, 3, ownerTarget["nodeC"])
+	// 6 chunks must be moved off nodeA
+	assert.Len(t, moves, 6)
+	for _, mv := range moves {
+		assert.Equal(t, "nodeA", mv.fromNode)
+	}
+}
+
+func TestPlanDistribution_Primary_NoMovesWhenBalanced(t *testing.T) {
+	volumeToOwner, volumeNodesList := threeNodeTopology()
+	// 3 chunks evenly spread: one per volume/node
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(1, 1, 0),
+		makeChunk(2, 2, 1),
+		makeChunk(3, 3, 2),
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	require.NoError(t, err)
+
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	moves, _ := planDistribution("primary", &bytes.Buffer{},
+		activeNodeList, nodeList, activeSet,
+		ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList,
+		len(chunks), len(activeNodeList), totalCopies)
+
+	assert.Empty(t, moves)
+}
+
+// ── planDistribution: round-robin ─────────────────────────────────────────────
+
+func TestPlanDistribution_RoundRobin_AssignsByOffset(t *testing.T) {
+	// 6 chunks all on nodeA; round-robin over 3 nodes should spread them A,B,C,A,B,C
+	volumeToOwner, volumeNodesList := threeNodeTopology()
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(1, 1, 0),
+		makeChunk(1, 2, 1),
+		makeChunk(1, 3, 2),
+		makeChunk(1, 4, 3),
+		makeChunk(1, 5, 4),
+		makeChunk(1, 6, 5),
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	require.NoError(t, err)
+
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	var buf bytes.Buffer
+	moves, ownerTarget := planDistribution("round-robin", &buf,
+		activeNodeList, nodeList, activeSet,
+		ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList,
+		len(chunks), len(activeNodeList), totalCopies)
+
+	// 2 chunks stay on nodeA (positions 0,3), 3 moves to nodeB and 3 to nodeC
+	assert.Equal(t, 2, ownerTarget["nodeA"])
+	assert.Equal(t, 2, ownerTarget["nodeB"])
+	assert.Equal(t, 2, ownerTarget["nodeC"])
+	// 4 chunks must move (nodeB and nodeC positions)
+	assert.Len(t, moves, 4)
+
+	// output should contain the round-robin preview
+	assert.Contains(t, buf.String(), "Round-robin assignment")
+}
+
+func TestPlanDistribution_RoundRobin_OffsetOrdering(t *testing.T) {
+	// chunks deliberately added in reverse offset order to verify sort
+	volumeToOwner := map[uint32]string{1: "nodeA", 2: "nodeB", 3: "nodeC"}
+	volumeNodesList := map[uint32][]string{1: {"nodeA"}, 2: {"nodeB"}, 3: {"nodeC"}}
+
+	// 3 chunks already balanced but in non-sequential order
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(3, 3, 200), // highest offset → should be assigned last
+		makeChunk(1, 1, 0),   // lowest offset  → assigned to nodeList[0]
+		makeChunk(2, 2, 100),
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	require.NoError(t, err)
+
+	nodeList := []string{"nodeA", "nodeB", "nodeC"}
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	_, ownerTarget := planDistribution("round-robin", &bytes.Buffer{},
+		activeNodeList, nodeList, activeSet,
+		ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList,
+		len(chunks), len(activeNodeList), totalCopies)
+
+	// every node gets exactly 1 chunk
+	assert.Equal(t, 1, ownerTarget["nodeA"])
+	assert.Equal(t, 1, ownerTarget["nodeB"])
+	assert.Equal(t, 1, ownerTarget["nodeC"])
+}
+
+// ── planDistribution: replica ─────────────────────────────────────────────────
+
+func TestPlanDistribution_Replica_BalancesOwnerAndCopies(t *testing.T) {
+	// 2-node replica topology: each volume has copies on both nodeA and nodeB
+	volumeToOwner := map[uint32]string{
+		1: "nodeA", // vid%2=1 → nodeA
+		2: "nodeB", // vid%2=0 → nodeB (sorted: [nodeA,nodeB], idx 2%2=0 → nodeA... let's be explicit)
+	}
+	// explicit: vol1 primary=nodeA, vol2 primary=nodeB; both replicated on each other
+	volumeNodesList := map[uint32][]string{
+		1: {"nodeA", "nodeB"},
+		2: {"nodeA", "nodeB"},
+	}
+
+	// 6 chunks: 5 on vol1 (nodeA), 1 on vol2 (nodeB)
+	chunks := []*filer_pb.FileChunk{
+		makeChunk(1, 1, 0),
+		makeChunk(1, 2, 1),
+		makeChunk(1, 3, 2),
+		makeChunk(1, 4, 3),
+		makeChunk(1, 5, 4),
+		makeChunk(2, 6, 5),
+	}
+
+	chunkToNode, ownerCount, copiesCount, err := buildDistributionCounts(chunks, volumeToOwner, volumeNodesList)
+	require.NoError(t, err)
+
+	nodeList := []string{"nodeA", "nodeB"}
+	activeNodeList, activeSet := selectActiveNodes(nodeList, ownerCount, 0)
+	totalCopies := 0
+	for _, cnt := range copiesCount {
+		totalCopies += cnt
+	}
+
+	moves, ownerTarget := planDistribution("replica", &bytes.Buffer{},
+		activeNodeList, nodeList, activeSet,
+		ownerCount, copiesCount, chunkToNode, chunks, volumeNodesList,
+		len(chunks), len(activeNodeList), totalCopies)
+
+	// after balancing, nodeA should have 3 and nodeB should have 3
+	assert.Equal(t, 3, ownerTarget["nodeA"])
+	assert.Equal(t, 3, ownerTarget["nodeB"])
+	assert.NotEmpty(t, moves)
+	for _, mv := range moves {
+		assert.Equal(t, "nodeA", mv.fromNode)
+		assert.Equal(t, "nodeB", mv.toNode)
+	}
+}
+
+// ── printRedistributionPlan ──────────────────────────────────────────────────
+
+func TestPrintRedistributionPlan_Output(t *testing.T) {
+	var buf bytes.Buffer
+	nodes := []string{"nodeA.local", "nodeB.local"}
+	ownerCount := map[string]int{"nodeA.local": 5, "nodeB.local": 1}
+	ownerTarget := map[string]int{"nodeA.local": 3, "nodeB.local": 3}
+
+	printRedistributionPlan(&buf, nodes, ownerCount, ownerTarget, 2)
+
+	out := buf.String()
+	assert.Contains(t, out, "2 chunks to move")
+	assert.Contains(t, out, "nodeA")
+	assert.Contains(t, out, "nodeB")
+	assert.Contains(t, out, "5 ->")
+	assert.Contains(t, out, "1 ->")
+}

weed/shell/command_fs_du.go

@@ -0,0 +1,93 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsDu{})
+}
+
+type commandFsDu struct {
+}
+
+func (c *commandFsDu) Name() string {
+	return "fs.du"
+}
+
+func (c *commandFsDu) Help() string {
+	return `show disk usage
+
+	fs.du /dir
+	fs.du /dir/file_name
+	fs.du /dir/file_prefix
+`
+}
+
+func (c *commandFsDu) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsDu) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	if commandEnv.isDirectory(path) {
+		path = path + "/"
+	}
+
+	var blockCount, byteCount uint64
+	dir, name := util.FullPath(path).DirAndName()
+	blockCount, byteCount, err = duTraverseDirectory(writer, commandEnv, dir, name)
+
+	if name == "" && err == nil {
+		fmt.Fprintf(writer, "block:%4d\tlogical size:%10d\t%s\n", blockCount, byteCount, dir)
+	}
+
+	return
+
+}
+
+func duTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir, name string) (blockCount, byteCount uint64, err error) {
+
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) error {
+
+		var fileBlockCount, fileByteCount uint64
+
+		if entry.IsDirectory {
+			subDir := fmt.Sprintf("%s/%s", dir, entry.Name)
+			if dir == "/" {
+				subDir = "/" + entry.Name
+			}
+			numBlock, numByte, err := duTraverseDirectory(writer, filerClient, subDir, "")
+			if err == nil {
+				blockCount += numBlock
+				byteCount += numByte
+			}
+		} else {
+			fileBlockCount = uint64(len(entry.GetChunks()))
+			fileByteCount = filer.FileSize(entry)
+			blockCount += fileBlockCount
+			byteCount += fileByteCount
+		}
+
+		if name != "" && !entry.IsDirectory {
+			fmt.Fprintf(writer, "block:%4d\tlogical size:%10d\t%s/%s\n", fileBlockCount, fileByteCount, dir, entry.Name)
+		}
+		return nil
+	})
+	return
+}

weed/shell/command_fs_log.go

@@ -0,0 +1,59 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsLogPurge{})
+}
+
+type commandFsLogPurge struct {
+}
+
+func (c *commandFsLogPurge) Name() string {
+	return "fs.log.purge"
+}
+
+func (c *commandFsLogPurge) Help() string {
+	return `purge filer logs
+
+	fs.log.purge [-v] [-daysAgo 365]
+`
+}
+
+func (c *commandFsLogPurge) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsLogPurge) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	fsLogPurgeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	daysAgo := fsLogPurgeCommand.Uint("daysAgo", 365, "purge logs older than N days")
+	verbose := fsLogPurgeCommand.Bool("v", false, "verbose mode")
+
+	if err = fsLogPurgeCommand.Parse(args); err != nil {
+		return err
+	}
+
+	modificationTimeAgo := time.Now().Add(-time.Hour * 24 * time.Duration(*daysAgo)).Unix()
+	err = filer_pb.ReadDirAllEntries(context.Background(), commandEnv, filer.SystemLogDir, "", func(entry *filer_pb.Entry, isLast bool) error {
+		if entry.Attributes.Mtime > modificationTimeAgo {
+			return nil
+		}
+		if errDel := filer_pb.Remove(context.Background(), commandEnv, filer.SystemLogDir, entry.Name, true, true, true, false, nil); errDel != nil {
+			return errDel
+		}
+		if *verbose {
+			fmt.Fprintf(writer, "delete %s\n", entry.Name)
+		}
+		return nil
+	})
+	return err
+}

weed/shell/command_fs_ls.go

@@ -0,0 +1,120 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/user"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsLs{})
+}
+
+type commandFsLs struct {
+}
+
+func (c *commandFsLs) Name() string {
+	return "fs.ls"
+}
+
+func (c *commandFsLs) Help() string {
+	return `list all files under a directory
+
+	fs.ls [-l] [-a] /dir/
+	fs.ls [-l] [-a] /dir/file_name
+	fs.ls [-l] [-a] /dir/file_prefix
+`
+}
+
+func (c *commandFsLs) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsLs) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	var isLongFormat, showHidden bool
+	for _, arg := range args {
+		if !strings.HasPrefix(arg, "-") {
+			break
+		}
+		for _, t := range arg {
+			switch t {
+			case 'a':
+				showHidden = true
+			case 'l':
+				isLongFormat = true
+			}
+		}
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	if commandEnv.isDirectory(path) {
+		path = path + "/"
+	}
+
+	dir, name := util.FullPath(path).DirAndName()
+	entryCount := 0
+
+	err = filer_pb.ReadDirAllEntries(context.Background(), commandEnv, util.FullPath(dir), name, func(entry *filer_pb.Entry, isLast bool) error {
+
+		if !showHidden && strings.HasPrefix(entry.Name, ".") {
+			return nil
+		}
+
+		entryCount++
+
+		if isLongFormat {
+			fileMode := os.FileMode(entry.Attributes.FileMode)
+			userName, groupNames := entry.Attributes.UserName, entry.Attributes.GroupName
+			if userName == "" {
+				if user, userErr := user.LookupId(strconv.Itoa(int(entry.Attributes.Uid))); userErr == nil {
+					userName = user.Username
+				}
+			}
+			groupName := ""
+			if len(groupNames) > 0 {
+				groupName = groupNames[0]
+			}
+			if groupName == "" {
+				if group, groupErr := user.LookupGroupId(strconv.Itoa(int(entry.Attributes.Gid))); groupErr == nil {
+					groupName = group.Name
+				}
+			}
+
+			if strings.HasSuffix(dir, "/") {
+				// just for printing
+				dir = dir[:len(dir)-1]
+			}
+			fmt.Fprintf(writer, "%s %3d %s %s %6d %s/%s\n",
+				fileMode, len(entry.GetChunks()),
+				userName, groupName,
+				filer.FileSize(entry), dir, entry.Name)
+		} else {
+			fmt.Fprintf(writer, "%s\n", entry.Name)
+		}
+
+		return nil
+	})
+
+	if isLongFormat && err == nil {
+		fmt.Fprintf(writer, "total %d\n", entryCount)
+	}
+
+	return
+}

weed/shell/command_fs_merge_volumes_test.go

@@ -0,0 +1,348 @@
+package shell
+
+import (
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func newMergeCmd(limitMB uint64, vols ...*master_pb.VolumeInformationMessage) *commandFsMergeVolumes {
+	c := &commandFsMergeVolumes{
+		volumes:         make(map[needle.VolumeId]*master_pb.VolumeInformationMessage),
+		volumeSizeLimit: limitMB * 1024 * 1024,
+	}
+	for _, v := range vols {
+		c.volumes[needle.VolumeId(v.Id)] = v
+	}
+	return c
+}
+
+// An explicit -fromVolumeId/-toVolumeId pair must be honored as given, even when
+// the source is larger than the target. The heuristic planner only ever merges
+// a smaller volume into a larger one, which used to make this request a silent
+// no-op.
+func TestCreateMergePlan_HonorsExplicitDirection(t *testing.T) {
+	larger := &master_pb.VolumeInformationMessage{Id: 87, Size: 7192590976}
+	smaller := &master_pb.VolumeInformationMessage{Id: 83, Size: 7088822248}
+	c := newMergeCmd(250000, larger, smaller)
+
+	plan, err := c.createMergePlan("*", []needle.VolumeId{83}, needle.VolumeId(87))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := plan.targets[needle.VolumeId(87)]; len(got) != 1 || got[0] != needle.VolumeId(83) {
+		t.Fatalf("expected 87->83, got plan=%v", plan.targets)
+	}
+
+	// The reverse direction keeps working too.
+	plan, err = c.createMergePlan("*", []needle.VolumeId{87}, needle.VolumeId(83))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := plan.targets[needle.VolumeId(83)]; len(got) != 1 || got[0] != needle.VolumeId(87) {
+		t.Fatalf("expected 83->87, got plan=%v", plan.targets)
+	}
+}
+
+// Merging into an empty target is valid (e.g. consolidating into a freshly
+// vacuumed volume); only an empty source should be rejected.
+func TestCreateMergePlan_DirectedAllowsEmptyTarget(t *testing.T) {
+	from := &master_pb.VolumeInformationMessage{Id: 87, Size: 100}
+	emptyTo := &master_pb.VolumeInformationMessage{Id: 83, Size: 0}
+	c := newMergeCmd(250000, from, emptyTo)
+
+	plan, err := c.createMergePlan("*", []needle.VolumeId{83}, needle.VolumeId(87))
+	if err != nil {
+		t.Fatalf("unexpected error merging into empty target: %v", err)
+	}
+	if got := plan.targets[needle.VolumeId(87)]; len(got) != 1 || got[0] != needle.VolumeId(83) {
+		t.Fatalf("expected 87->83, got plan=%v", plan.targets)
+	}
+}
+
+// The directed planner rejects a self-map on its own, not just via Do, since it
+// is exercised directly.
+func TestCreateMergePlan_DirectedRejectsSelfMap(t *testing.T) {
+	v := &master_pb.VolumeInformationMessage{Id: 87, Size: 100}
+	c := newMergeCmd(250000, v)
+
+	_, err := c.createMergePlan("*", []needle.VolumeId{87}, needle.VolumeId(87))
+	if err == nil || !strings.Contains(err.Error(), "no volume id changes") {
+		t.Fatalf("expected self-map rejection, got %v", err)
+	}
+}
+
+func TestCreateMergePlan_DirectedRejectsIneligible(t *testing.T) {
+	cases := []struct {
+		name     string
+		from, to *master_pb.VolumeInformationMessage
+		coll     string
+		wantErr  string
+	}{
+		{
+			name:    "readonly source",
+			from:    &master_pb.VolumeInformationMessage{Id: 87, Size: 100, ReadOnly: true},
+			to:      &master_pb.VolumeInformationMessage{Id: 83, Size: 100},
+			coll:    "*",
+			wantErr: "volume 87 is readonly",
+		},
+		{
+			name:    "readonly target",
+			from:    &master_pb.VolumeInformationMessage{Id: 87, Size: 100},
+			to:      &master_pb.VolumeInformationMessage{Id: 83, Size: 100, ReadOnly: true},
+			coll:    "*",
+			wantErr: "volume 83 is readonly",
+		},
+		{
+			name:    "empty source",
+			from:    &master_pb.VolumeInformationMessage{Id: 87, Size: 0},
+			to:      &master_pb.VolumeInformationMessage{Id: 83, Size: 100},
+			coll:    "*",
+			wantErr: "volume 87 is empty",
+		},
+		{
+			name:    "wrong collection",
+			from:    &master_pb.VolumeInformationMessage{Id: 87, Size: 100, Collection: "other"},
+			to:      &master_pb.VolumeInformationMessage{Id: 83, Size: 100, Collection: "other"},
+			coll:    "wanted",
+			wantErr: "volume 87 is not in collection",
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := newMergeCmd(250000, tc.from, tc.to)
+			_, err := c.createMergePlan(tc.coll, []needle.VolumeId{needle.VolumeId(tc.to.Id)}, needle.VolumeId(tc.from.Id))
+			if err == nil {
+				t.Fatalf("expected error %q, got nil", tc.wantErr)
+			}
+			if got := err.Error(); !strings.Contains(got, tc.wantErr) {
+				t.Fatalf("expected error containing %q, got %q", tc.wantErr, got)
+			}
+		})
+	}
+}
+
+// A source that fits into no single target distributes across several, each
+// chunk going to the target with the most remaining capacity.
+func TestCreateMergePlan_DistributesAcrossMultipleTargets(t *testing.T) {
+	mb := uint64(1024 * 1024)
+	from := &master_pb.VolumeInformationMessage{Id: 112, Size: 60 * mb}
+	to1 := &master_pb.VolumeInformationMessage{Id: 111, Size: 60 * mb}
+	to2 := &master_pb.VolumeInformationMessage{Id: 107, Size: 70 * mb}
+	c := newMergeCmd(100, from, to1, to2)
+
+	plan, err := c.createMergePlan("*", []needle.VolumeId{111, 107}, needle.VolumeId(112))
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := plan.targets[needle.VolumeId(112)]; len(got) != 2 {
+		t.Fatalf("expected two targets, got %v", got)
+	}
+
+	// 111 has 40 MB free, 107 has 30 MB free. The first chunk goes to the
+	// emptier 111; allocations keep balancing remaining capacity after that.
+	vid, ok := plan.allocate(needle.VolumeId(112), 5*mb)
+	if !ok || vid != needle.VolumeId(111) {
+		t.Fatalf("expected first chunk on 111, got %v ok=%v", vid, ok)
+	}
+	vid, ok = plan.allocate(needle.VolumeId(112), 10*mb)
+	if !ok || vid != needle.VolumeId(111) {
+		t.Fatalf("expected second chunk on 111 (35 MB free vs 30), got %v ok=%v", vid, ok)
+	}
+	vid, ok = plan.allocate(needle.VolumeId(112), 5*mb)
+	if !ok || vid != needle.VolumeId(107) {
+		t.Fatalf("expected third chunk on 107 (30 MB free vs 25), got %v ok=%v", vid, ok)
+	}
+	// A chunk larger than any remaining capacity is refused.
+	if vid, ok = plan.allocate(needle.VolumeId(112), 45*mb); ok {
+		t.Fatalf("expected no room for oversized chunk, got %v", vid)
+	}
+	// Smaller chunks still fit afterwards.
+	if _, ok = plan.allocate(needle.VolumeId(112), 20*mb); !ok {
+		t.Fatal("expected room for 20 MB chunk")
+	}
+
+	// A failed move's release makes the reservation reusable.
+	vid2, ok := plan.allocate(needle.VolumeId(112), 20*mb)
+	if !ok {
+		t.Fatal("expected room for second 20 MB chunk")
+	}
+	plan.release(needle.VolumeId(112), vid2, 20*mb)
+	if vid3, ok := plan.allocate(needle.VolumeId(112), 20*mb); !ok || vid3 != vid2 {
+		t.Fatalf("released capacity must be reusable, got %v ok=%v", vid3, ok)
+	}
+}
+
+// The combined free capacity of all targets must cover the source's live data.
+func TestCreateMergePlan_MultiTargetRejectsInsufficientCapacity(t *testing.T) {
+	mb := uint64(1024 * 1024)
+	from := &master_pb.VolumeInformationMessage{Id: 112, Size: 90 * mb}
+	to1 := &master_pb.VolumeInformationMessage{Id: 111, Size: 60 * mb}
+	to2 := &master_pb.VolumeInformationMessage{Id: 107, Size: 60 * mb}
+	c := newMergeCmd(100, from, to1, to2)
+
+	_, err := c.createMergePlan("*", []needle.VolumeId{111, 107}, needle.VolumeId(112))
+	if err == nil || !strings.Contains(err.Error(), "cannot merge into volumes") {
+		t.Fatalf("expected capacity rejection, got %v", err)
+	}
+}
+
+// Without -fromVolumeId, a -toVolumeId list restricts the heuristic planner's
+// target candidates to the listed volumes.
+func TestCreateMergePlan_TargetListRestrictsHeuristic(t *testing.T) {
+	mb := uint64(1024 * 1024)
+	big := &master_pb.VolumeInformationMessage{Id: 1, Size: 80 * mb}
+	mid := &master_pb.VolumeInformationMessage{Id: 2, Size: 50 * mb}
+	small := &master_pb.VolumeInformationMessage{Id: 3, Size: 10 * mb}
+	c := newMergeCmd(100, big, mid, small)
+
+	plan, err := c.createMergePlan("*", []needle.VolumeId{2}, 0)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if got := plan.targets[needle.VolumeId(3)]; len(got) != 1 || got[0] != needle.VolumeId(2) {
+		t.Fatalf("expected 3->2, got %v", plan.targets)
+	}
+	if plan.isSource(needle.VolumeId(1)) {
+		t.Fatalf("volume 1 must not merge anywhere, got %v", plan.targets)
+	}
+}
+
+func TestParseTargetVolumeIds(t *testing.T) {
+	if ids, err := parseTargetVolumeIds(""); err != nil || ids != nil {
+		t.Fatalf("empty arg: got %v, %v", ids, err)
+	}
+	if ids, err := parseTargetVolumeIds("0"); err != nil || ids != nil {
+		t.Fatalf("zero arg: got %v, %v", ids, err)
+	}
+	ids, err := parseTargetVolumeIds("111, 107")
+	if err != nil || len(ids) != 2 || ids[0] != needle.VolumeId(111) || ids[1] != needle.VolumeId(107) {
+		t.Fatalf("list arg: got %v, %v", ids, err)
+	}
+	if _, err = parseTargetVolumeIds("111,111"); err == nil {
+		t.Fatal("expected duplicate rejection")
+	}
+	if _, err = parseTargetVolumeIds("111,x"); err == nil {
+		t.Fatal("expected invalid id rejection")
+	}
+	if _, err = parseTargetVolumeIds("4294967297"); err == nil {
+		t.Fatal("expected uint32 overflow rejection")
+	}
+}
+
+// A volume reporting more deleted bytes than it holds must read as empty.
+func TestGetVolumeSize_ClampsDeletedOverSize(t *testing.T) {
+	c := newMergeCmd(250000)
+
+	overDeleted := &master_pb.VolumeInformationMessage{Id: 1, Size: 1000, DeletedByteCount: 4000}
+	if got := c.getVolumeSize(overDeleted); got != 0 {
+		t.Errorf("expected 0 for a volume with more deleted than stored, got %d", got)
+	}
+
+	healthy := &master_pb.VolumeInformationMessage{Id: 2, Size: 3000, DeletedByteCount: 1000}
+	if got := c.getVolumeSize(healthy); got != 2000 {
+		t.Errorf("expected 2000, got %d", got)
+	}
+}
+
+// Manifests on volumes in a foreign collection cannot reference plan volumes
+// and must be skipped instead of downloaded; plan sources and unknown volumes
+// must still be resolved.
+func TestManifestMayReferencePlan(t *testing.T) {
+	src := &master_pb.VolumeInformationMessage{Id: 1, Size: 100, Collection: "x"}
+	same := &master_pb.VolumeInformationMessage{Id: 2, Size: 100, Collection: "x"}
+	foreign := &master_pb.VolumeInformationMessage{Id: 3, Size: 100, Collection: "y"}
+	c := newMergeCmd(250000, src, same, foreign)
+
+	plan := newMergePlan(c.volumeSizeLimit)
+	plan.targets[needle.VolumeId(1)] = []needle.VolumeId{2}
+	planCollections := map[string]bool{"x": true}
+
+	for vid, want := range map[needle.VolumeId]bool{
+		1: true,  // plan source
+		2: true,  // same collection
+		3: false, // foreign collection
+		9: true,  // unknown volume, resolve conservatively
+	} {
+		if got := c.manifestMayReferencePlan(plan, planCollections, vid); got != want {
+			t.Fatalf("volume %d: got %v, want %v", vid, got, want)
+		}
+	}
+}
+
+// A source whose index still holds needles but that the traversal never saw is
+// made entirely of orphan needles — the merge moves nothing, but the operator
+// must be told the real cleanup is fsck.
+func TestWarnUnreferencedSources(t *testing.T) {
+	vol := &master_pb.VolumeInformationMessage{Id: 203, FileCount: 18}
+	c := newMergeCmd(250000, vol)
+	plan := &mergePlan{targets: map[needle.VolumeId][]needle.VolumeId{
+		needle.VolumeId(203): {needle.VolumeId(187)},
+	}}
+
+	var sb strings.Builder
+	c.warnUnreferencedSources(&sb, plan, newSourceNeedleCounter(), "/buckets/test")
+	out := sb.String()
+	if !strings.Contains(out, "volume 203") || !strings.Contains(out, "orphan needles") {
+		t.Fatalf("expected orphan warning, got %q", out)
+	}
+
+	// Needles seen during traversal: no warning.
+	seen := newSourceNeedleCounter()
+	for i := 0; i < 7; i++ {
+		seen.record(needle.VolumeId(203))
+	}
+	sb.Reset()
+	c.warnUnreferencedSources(&sb, plan, seen, "/buckets/test")
+	if sb.Len() != 0 {
+		t.Fatalf("expected no warning when needles were seen, got %q", sb.String())
+	}
+
+	// Zero FileCount (truly empty volume): nothing to report.
+	emptyVol := &master_pb.VolumeInformationMessage{Id: 204}
+	c2 := newMergeCmd(250000, emptyVol)
+	plan2 := &mergePlan{targets: map[needle.VolumeId][]needle.VolumeId{
+		needle.VolumeId(204): {needle.VolumeId(187)},
+	}}
+	sb.Reset()
+	c2.warnUnreferencedSources(&sb, plan2, newSourceNeedleCounter(), "/buckets/test")
+	if sb.Len() != 0 {
+		t.Fatalf("expected no warning for empty volume, got %q", sb.String())
+	}
+}
+
+// Concurrent BFS workers increment the production counter through record — the
+// count must stay correct under -race with source-heavy inputs.
+func TestWarnUnreferencedSources_ConcurrentRecording(t *testing.T) {
+	vol := &master_pb.VolumeInformationMessage{Id: 203, FileCount: 1000}
+	c := newMergeCmd(250000, vol)
+	plan := &mergePlan{targets: map[needle.VolumeId][]needle.VolumeId{
+		needle.VolumeId(203): {needle.VolumeId(187)},
+	}}
+
+	seen := newSourceNeedleCounter()
+	var wg sync.WaitGroup
+	for w := 0; w < 5; w++ { // TraverseBfs worker count
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 200; i++ {
+				seen.record(needle.VolumeId(203))
+			}
+		}()
+	}
+	wg.Wait()
+
+	if got := seen.count(needle.VolumeId(203)); got != 1000 {
+		t.Fatalf("lost increments under concurrency: %d", got)
+	}
+
+	var sb strings.Builder
+	c.warnUnreferencedSources(&sb, plan, seen, "/buckets/test")
+	if sb.Len() != 0 {
+		t.Fatalf("expected no warning when all needles were seen, got %q", sb.String())
+	}
+}

weed/shell/command_fs_meta_cat.go

@@ -0,0 +1,80 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMetaCat{})
+}
+
+type commandFsMetaCat struct {
+}
+
+func (c *commandFsMetaCat) Name() string {
+	return "fs.meta.cat"
+}
+
+func (c *commandFsMetaCat) Help() string {
+	return `print out the meta data content for a file or directory
+
+	fs.meta.cat /dir/
+	fs.meta.cat /dir/file_name
+`
+}
+
+func (c *commandFsMetaCat) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMetaCat) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	dir, name := util.FullPath(path).DirAndName()
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.LookupDirectoryEntryRequest{
+			Name:      name,
+			Directory: dir,
+		}
+		respLookupEntry, err := filer_pb.LookupEntry(context.Background(), client, request)
+		if err != nil {
+			return err
+		}
+
+		chunks := respLookupEntry.Entry.Chunks
+		sort.Slice(chunks, func(i, j int) bool {
+			return chunks[i].Offset < chunks[j].Offset
+		})
+
+		filer.ProtoToText(writer, respLookupEntry.Entry)
+
+		bytes, _ := proto.Marshal(respLookupEntry.Entry)
+		gzippedBytes, _ := util.GzipData(bytes)
+		// zstdBytes, _ := util.ZstdData(bytes)
+		// fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d zstd:%d\n", len(respLookupEntry.Entry.GetChunks()), len(bytes), len(gzippedBytes), len(zstdBytes))
+		fmt.Fprintf(writer, "chunks %d meta size: %d gzip:%d\n", len(respLookupEntry.Entry.GetChunks()), len(bytes), len(gzippedBytes))
+
+		return nil
+
+	})
+
+}

weed/shell/command_fs_meta_change_volume_id.go

@@ -0,0 +1,137 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMetaChangeVolumeId{})
+}
+
+type commandFsMetaChangeVolumeId struct {
+}
+
+func (c *commandFsMetaChangeVolumeId) Name() string {
+	return "fs.meta.changeVolumeId"
+}
+
+func (c *commandFsMetaChangeVolumeId) Help() string {
+	return `change volume id in existing metadata.
+
+	fs.meta.changeVolumeId -dir=/path/to/a/dir -fromVolumeId=x -toVolumeId=y -apply
+	fs.meta.changeVolumeId -dir=/path/to/a/dir -mapping=/path/to/mapping/file -apply
+
+	The mapping file should have these lines, each line is: [fromVolumeId]=>[toVolumeId]
+	e.g.
+		1 => 2
+		3 => 4
+
+`
+}
+
+func (c *commandFsMetaChangeVolumeId) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMetaChangeVolumeId) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	fsMetaChangeVolumeIdCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	dir := fsMetaChangeVolumeIdCommand.String("dir", "/", "fix all metadata under this folder")
+	mappingFileName := fsMetaChangeVolumeIdCommand.String("mapping", "", "a file with multiple volume id changes, with each line as x=>y")
+	fromVolumeId := fsMetaChangeVolumeIdCommand.Uint("fromVolumeId", 0, "change metadata with this volume id")
+	toVolumeId := fsMetaChangeVolumeIdCommand.Uint("toVolumeId", 0, "change metadata to this volume id")
+	applyChanges := fsMetaChangeVolumeIdCommand.Bool("apply", false, "apply the metadata changes")
+	// TODO: remove this alias
+	applyChangesAlias := fsMetaChangeVolumeIdCommand.Bool("force", false, "apply the metadata changes (alias for -apply)")
+	if err = fsMetaChangeVolumeIdCommand.Parse(args); err != nil {
+		return err
+	}
+
+	handleDeprecatedForceFlag(writer, fsMetaChangeVolumeIdCommand, applyChangesAlias, applyChanges)
+	infoAboutSimulationMode(writer, *applyChanges, "-apply")
+
+	// load the mapping
+	mapping := make(map[needle.VolumeId]needle.VolumeId)
+	if *mappingFileName != "" {
+		readMappingFromFile(util.ResolvePath(*mappingFileName), mapping)
+	} else {
+		if *fromVolumeId == *toVolumeId {
+			return fmt.Errorf("no volume id changes")
+		}
+		if *fromVolumeId == 0 || *toVolumeId == 0 {
+			return fmt.Errorf("volume id can not be zero")
+		}
+		mapping[needle.VolumeId(*fromVolumeId)] = needle.VolumeId(*toVolumeId)
+	}
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer_pb.TraverseBfs(context.Background(), commandEnv, util.FullPath(*dir), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
+			if !entry.IsDirectory {
+				var hasChanges bool
+				for _, chunk := range entry.Chunks {
+					if chunk.IsChunkManifest {
+						fmt.Printf("Change volume id for large file is not implemented yet: %s/%s\n", parentPath, entry.Name)
+						return nil
+					}
+					chunkVolumeId := chunk.Fid.VolumeId
+					if toVolumeId, found := mapping[needle.VolumeId(chunkVolumeId)]; found {
+						hasChanges = true
+						chunk.Fid.VolumeId = uint32(toVolumeId)
+						chunk.FileId = ""
+					}
+				}
+				if hasChanges {
+					println("Updating", parentPath, entry.Name)
+					if *applyChanges {
+						if updateErr := filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
+							Directory: string(parentPath),
+							Entry:     entry,
+						}); updateErr != nil {
+							fmt.Printf("failed to update %s/%s: %v\n", parentPath, entry.Name, updateErr)
+						}
+					}
+				}
+			}
+			return nil
+		})
+	})
+}
+
+func readMappingFromFile(filename string, mapping map[needle.VolumeId]needle.VolumeId) error {
+	mappingFile, openErr := os.Open(filename)
+	if openErr != nil {
+		return fmt.Errorf("failed to open file %s: %v", filename, openErr)
+	}
+	defer mappingFile.Close()
+	mappingContent, readErr := io.ReadAll(mappingFile)
+	if readErr != nil {
+		return fmt.Errorf("failed to read file %s: %v", filename, readErr)
+	}
+	for _, line := range strings.Split(string(mappingContent), "\n") {
+		parts := strings.Split(line, "=>")
+		if len(parts) != 2 {
+			println("unrecognized line:", line)
+			continue
+		}
+		x, errX := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 64)
+		if errX != nil {
+			return errX
+		}
+		y, errY := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64)
+		if errY != nil {
+			return errY
+		}
+		mapping[needle.VolumeId(x)] = needle.VolumeId(y)
+	}
+	return nil
+}

weed/shell/command_fs_meta_load.go

@@ -0,0 +1,176 @@
+package shell
+
+import (
+	"compress/gzip"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMetaLoad{})
+}
+
+type commandFsMetaLoad struct {
+	dirPrefix *string
+}
+
+func (c *commandFsMetaLoad) Name() string {
+	return "fs.meta.load"
+}
+
+func (c *commandFsMetaLoad) Help() string {
+	return `load saved filer meta data to restore the directory and file structure
+
+	fs.meta.load <filer_host>-<port>-<time>.meta
+	fs.meta.load -v=false <filer_host>-<port>-<time>.meta // skip printing out the verbose output
+ 	fs.meta.load -concurrency=1 <filer_host>-<port>-<time>.meta // number of parallel meta load to filer
+	fs.meta.load -dirPrefix=/buckets/important <filer_host>.meta // load any dirs with prefix "important"
+
+`
+}
+
+func (c *commandFsMetaLoad) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMetaLoad) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if len(args) == 0 {
+		fmt.Fprintf(writer, "missing a metadata file\n")
+		return nil
+	}
+
+	fileName := util.ResolvePath(args[len(args)-1])
+
+	metaLoadCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	c.dirPrefix = metaLoadCommand.String("dirPrefix", "", "load entries only with directories matching prefix")
+	concurrency := metaLoadCommand.Int("concurrency", 1, "number of parallel meta load to filer")
+	verbose := metaLoadCommand.Bool("v", true, "verbose mode")
+	if err = metaLoadCommand.Parse(args[0 : len(args)-1]); err != nil {
+		return nil
+	}
+
+	var dst io.Reader
+
+	f, err := os.OpenFile(fileName, os.O_RDONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %v", fileName, err)
+	}
+	defer f.Close()
+
+	dst = f
+
+	if strings.HasSuffix(fileName, ".gz") || strings.HasSuffix(fileName, ".gzip") {
+		var gr *gzip.Reader
+		gr, err = gzip.NewReader(dst)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			err1 := gr.Close()
+			if err == nil {
+				err = err1
+			}
+		}()
+
+		dst = gr
+	}
+
+	var dirCount, fileCount uint64
+	lastLogTime := time.Now()
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		sizeBuf := make([]byte, 4)
+		waitChan := make(chan struct{}, *concurrency)
+		defer close(waitChan)
+		var wg sync.WaitGroup
+
+		for {
+			if _, err := io.ReadFull(dst, sizeBuf); err != nil {
+				if err == io.EOF {
+					return nil
+				}
+				return err
+			}
+
+			size := util.BytesToUint32(sizeBuf)
+
+			data := make([]byte, int(size))
+
+			if _, err := io.ReadFull(dst, data); err != nil {
+				return err
+			}
+
+			fullEntry := &filer_pb.FullEntry{}
+			if err = proto.Unmarshal(data, fullEntry); err != nil {
+				return err
+			}
+
+			// check collection name pattern
+			entryFullName := string(util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name))
+			if *c.dirPrefix != "" {
+				if !strings.HasPrefix(fullEntry.Dir, *c.dirPrefix) {
+					if *verbose {
+						fmt.Fprintf(writer, "not match dir prefix %s\n", entryFullName)
+					}
+					continue
+				}
+			}
+
+			if *verbose || lastLogTime.Add(time.Second).Before(time.Now()) {
+				if !*verbose {
+					lastLogTime = time.Now()
+				}
+				fmt.Fprintf(writer, "load %s\n", entryFullName)
+			}
+
+			fullEntry.Entry.Name = strings.ReplaceAll(fullEntry.Entry.Name, "/", "x")
+			if fullEntry.Entry.IsDirectory {
+				wg.Wait()
+				if errEntry := filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
+					Directory: fullEntry.Dir,
+					Entry:     fullEntry.Entry,
+				}); errEntry != nil {
+					return errEntry
+				}
+				dirCount++
+			} else {
+				wg.Add(1)
+				waitChan <- struct{}{}
+				go func(entry *filer_pb.FullEntry) {
+					if errEntry := filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
+						Directory: entry.Dir,
+						Entry:     entry.Entry,
+					}); errEntry != nil {
+						err = errEntry
+					}
+					defer wg.Done()
+					<-waitChan
+				}(fullEntry)
+				if err != nil {
+					return err
+				}
+				fileCount++
+			}
+		}
+	})
+
+	if err == nil {
+		fmt.Fprintf(writer, "\ntotal %d directories, %d files", dirCount, fileCount)
+		fmt.Fprintf(writer, "\n%s is loaded.\n", fileName)
+	}
+
+	return err
+}

weed/shell/command_fs_meta_notify.go

@@ -0,0 +1,82 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/notification"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMetaNotify{})
+}
+
+type commandFsMetaNotify struct {
+}
+
+func (c *commandFsMetaNotify) Name() string {
+	return "fs.meta.notify"
+}
+
+func (c *commandFsMetaNotify) Help() string {
+	return `recursively send directory and file meta data to notification message queue
+
+	fs.meta.notify	# send meta data from current directory to notification message queue
+
+	The message queue will use it to trigger replication from this filer.
+
+`
+}
+
+func (c *commandFsMetaNotify) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMetaNotify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	util.LoadConfiguration("notification", true)
+	v := util.GetViper()
+	notification.LoadConfiguration(v, "notification.")
+
+	var dirCount, fileCount uint64
+
+	err = filer_pb.TraverseBfs(context.Background(), commandEnv, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
+
+		if entry.IsDirectory {
+			dirCount++
+		} else {
+			fileCount++
+		}
+
+		notifyErr := notification.Queue.SendMessage(
+			string(parentPath.Child(entry.Name)),
+			&filer_pb.EventNotification{
+				NewEntry: entry,
+			},
+		)
+
+		if notifyErr != nil {
+			fmt.Fprintf(writer, "fail to notify new entry event for %s: %v\n", parentPath.Child(entry.Name), notifyErr)
+		}
+		return nil
+	})
+
+	if err == nil {
+		fmt.Fprintf(writer, "\ntotal notified %d directories, %d files\n", dirCount, fileCount)
+	}
+
+	return err
+
+}

weed/shell/command_fs_meta_save.go

@@ -0,0 +1,264 @@
+package shell
+
+import (
+	"compress/gzip"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMetaSave{})
+}
+
+type commandFsMetaSave struct {
+}
+
+func (c *commandFsMetaSave) Name() string {
+	return "fs.meta.save"
+}
+
+func (c *commandFsMetaSave) Help() string {
+	return `save all directory and file meta data to a local file for metadata backup.
+
+	fs.meta.save /               # save from the root
+	fs.meta.save -v -o t.meta /  # save from the root, output to t.meta file.
+	fs.meta.save /path/to/save   # save from the directory /path/to/save
+	fs.meta.save .               # save from current directory
+	fs.meta.save                 # save from current directory
+
+	The meta data will be saved into a local <filer_host>-<port>-<time>.meta.gz file.
+	These meta data can be later loaded by fs.meta.load command
+
+`
+}
+
+func (c *commandFsMetaSave) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	fsMetaSaveCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	verbose := fsMetaSaveCommand.Bool("v", false, "print out each processed files")
+	outputFileName := fsMetaSaveCommand.String("o", "", "output the meta data to this file. If file name ends with .gz or .gzip, it will be gzip compressed")
+	isObfuscate := fsMetaSaveCommand.Bool("obfuscate", false, "obfuscate the file names")
+	// chunksFileName := fsMetaSaveCommand.String("chunks", "", "output all the chunks to this file")
+	if err = fsMetaSaveCommand.Parse(args); err != nil {
+		return err
+	}
+
+	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsMetaSaveCommand.Args()))
+	if parseErr != nil {
+		return parseErr
+	}
+
+	fileName := util.ResolvePath(*outputFileName)
+	if fileName == "" {
+		t := time.Now()
+		fileName = fmt.Sprintf("%s-%4d%02d%02d-%02d%02d%02d.meta.gz",
+			commandEnv.option.FilerAddress.ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
+	}
+
+	var dst io.Writer
+
+	f, openErr := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if openErr != nil {
+		return fmt.Errorf("failed to create file %s: %v", fileName, openErr)
+	}
+	defer f.Close()
+
+	dst = f
+
+	if strings.HasSuffix(fileName, ".gz") || strings.HasSuffix(fileName, ".gzip") {
+		gw := gzip.NewWriter(dst)
+		defer func() {
+			err1 := gw.Close()
+			if err == nil {
+				err = err1
+			}
+		}()
+
+		dst = gw
+	}
+
+	var cipherKey util.CipherKey
+	if *isObfuscate {
+		cipherKey = util.GenCipherKey()
+	}
+
+	err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, true, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
+		if !entry.Entry.IsDirectory {
+			ext := filepath.Ext(entry.Entry.Name)
+			if encrypted, encErr := util.Encrypt([]byte(entry.Entry.Name), cipherKey); encErr == nil {
+				entry.Entry.Name = util.Base64Encode(encrypted)[:len(entry.Entry.Name)] + ext
+				entry.Entry.Name = strings.ReplaceAll(entry.Entry.Name, "/", "x")
+			}
+		}
+		bytes, err := proto.Marshal(entry)
+		if err != nil {
+			fmt.Fprintf(writer, "marshall error: %v\n", err)
+			return
+		}
+
+		select {
+		case outputChan <- bytes:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+		return nil
+	}, func(outputChan chan interface{}) error {
+		sizeBuf := make([]byte, 4)
+		for item := range outputChan {
+			b := item.([]byte)
+			util.Uint32toBytes(sizeBuf, uint32(len(b)))
+			_, err := dst.Write(sizeBuf)
+			if err != nil {
+				return err
+			}
+			_, err = dst.Write(b)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	if err == nil {
+		fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.option.FilerAddress.ToHttpAddress(), path, fileName)
+	}
+
+	return err
+
+}
+
+// skipSystemLog excludes the metadata-log subtree (SystemLogDir) from the
+// traversal. fs.meta.save sets it so the export doesn't carry the whole change
+// log; volume.fsck and fs.verify must leave it false so the live log-file
+// chunks are accounted for — otherwise fsck reports them as orphans and could
+// delete referenced needles.
+func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error {
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	outputChan := make(chan interface{}, 1024)
+	saveErrChan := make(chan error, 1)
+	go func() {
+		saveErr := saveFn(outputChan)
+		saveErrChan <- saveErr
+		if saveErr != nil {
+			cancel()
+		}
+		wg.Done()
+	}()
+
+	var dirCount, fileCount uint64
+	var once sync.Once
+	var firstErr error
+	var hasErr atomic.Bool
+
+	// also save the directory itself (path) if it exists in the filer
+	if e, _, _, getErr := filer_pb.GetEntry(ctx, filerClient, util.FullPath(path)); getErr != nil {
+		// Entry not found is expected and can be ignored; log other errors.
+		if !errors.Is(getErr, filer_pb.ErrNotFound) {
+			fmt.Fprintf(writer, "failed to get entry %s: %v\n", path, getErr)
+		}
+	} else if e != nil {
+		parentDir, _ := util.FullPath(path).DirAndName()
+		protoMessage := &filer_pb.FullEntry{
+			Dir:   parentDir,
+			Entry: e,
+		}
+
+		if genErr := genFn(ctx, protoMessage, outputChan); genErr != nil {
+			once.Do(func() {
+				firstErr = genErr
+				hasErr.Store(true)
+			})
+			return genErr
+		} else {
+			if e.IsDirectory {
+				atomic.AddUint64(&dirCount, 1)
+			} else {
+				atomic.AddUint64(&fileCount, 1)
+			}
+		}
+	}
+
+	defer cancel()
+	systemLogChildPrefix := filer.SystemLogDir + "/"
+	err := filer_pb.TraverseBfs(ctx, filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
+
+		if skipSystemLog {
+			parent := string(parentPath)
+			if parent == filer.SystemLogDir || strings.HasPrefix(parent, systemLogChildPrefix) {
+				return nil
+			}
+		}
+
+		protoMessage := &filer_pb.FullEntry{
+			Dir:   string(parentPath),
+			Entry: entry,
+		}
+
+		if hasErr.Load() {
+			// fail-fast: stop traversal once an error is observed.
+			return firstErr
+		}
+		if genErr := genFn(ctx, protoMessage, outputChan); genErr != nil {
+			once.Do(func() {
+				firstErr = genErr
+				hasErr.Store(true)
+				cancel()
+			})
+			return genErr
+		}
+
+		if entry.IsDirectory {
+			atomic.AddUint64(&dirCount, 1)
+		} else {
+			atomic.AddUint64(&fileCount, 1)
+		}
+
+		if verbose {
+			println(parentPath.Child(entry.Name))
+		}
+
+		return nil
+	})
+
+	close(outputChan)
+
+	wg.Wait()
+	saveErr := <-saveErrChan
+
+	if err != nil {
+		return err
+	}
+	if saveErr != nil {
+		return saveErr
+	}
+
+	if writer != nil {
+		fmt.Fprintf(writer, "total %d directories, %d files\n", dirCount, fileCount)
+	}
+	return nil
+}

weed/shell/command_fs_mkdir.go

@@ -0,0 +1,66 @@
+package shell
+
+import (
+	"context"
+	"io"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMkdir{})
+}
+
+type commandFsMkdir struct {
+}
+
+func (c *commandFsMkdir) Name() string {
+	return "fs.mkdir"
+}
+
+func (c *commandFsMkdir) Help() string {
+	return `create a directory
+
+	fs.mkdir path/to/dir
+`
+}
+
+func (c *commandFsMkdir) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMkdir) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	dir, name := util.FullPath(path).DirAndName()
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		_, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+			Directory: dir,
+			Entry: &filer_pb.Entry{
+				Name:        name,
+				IsDirectory: true,
+				Attributes: &filer_pb.FuseAttributes{
+					Mtime:    time.Now().Unix(),
+					Crtime:   time.Now().Unix(),
+					FileMode: uint32(0777 | os.ModeDir),
+				},
+			},
+		})
+		return createErr
+	})
+
+	return
+}

weed/shell/command_fs_mv.go

@@ -0,0 +1,102 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsMv{})
+}
+
+type commandFsMv struct {
+}
+
+func (c *commandFsMv) Name() string {
+	return "fs.mv"
+}
+
+func (c *commandFsMv) Help() string {
+	return `move or rename a file or a folder
+
+	fs.mv  <source entry> <destination entry> 
+
+	fs.mv /dir/file_name /dir2/filename2
+	fs.mv /dir/file_name /dir2
+
+	fs.mv /dir/dir2 /dir3/dir4/
+	fs.mv /dir/dir2 /dir3/new_dir
+
+`
+}
+
+func (c *commandFsMv) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsMv) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	if len(args) != 2 {
+		return fmt.Errorf("need to have 2 arguments")
+	}
+
+	sourcePath, err := commandEnv.parseUrl(args[0])
+	if err != nil {
+		return err
+	}
+
+	destinationPath, err := commandEnv.parseUrl(args[1])
+	if err != nil {
+		return err
+	}
+
+	sourceDir, sourceName := util.FullPath(sourcePath).DirAndName()
+
+	destinationDir, destinationName := util.FullPath(destinationPath).DirAndName()
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		// collect destination entry info
+		destinationRequest := &filer_pb.LookupDirectoryEntryRequest{
+			Directory: destinationDir,
+			Name:      destinationName,
+		}
+		respDestinationLookupEntry, err := filer_pb.LookupEntry(context.Background(), client, destinationRequest)
+
+		var targetDir, targetName string
+
+		// moving a file or folder
+		if err == nil && respDestinationLookupEntry.Entry.IsDirectory {
+			// to a directory
+			targetDir = util.Join(destinationDir, destinationName)
+			targetName = sourceName
+		} else {
+			// to a file or folder
+			targetDir = destinationDir
+			targetName = destinationName
+		}
+
+		request := &filer_pb.AtomicRenameEntryRequest{
+			OldDirectory: sourceDir,
+			OldName:      sourceName,
+			NewDirectory: targetDir,
+			NewName:      targetName,
+		}
+
+		_, err = client.AtomicRenameEntry(context.Background(), request)
+
+		fmt.Fprintf(writer, "move: %s => %s\n", sourcePath, util.NewFullPath(targetDir, targetName))
+
+		return err
+
+	})
+
+}

weed/shell/command_fs_mv_test.go

@@ -0,0 +1,107 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/status"
+)
+
+type fsMvTestFilerServer struct {
+	filer_pb.UnimplementedSeaweedFilerServer
+
+	lookupReq *filer_pb.LookupDirectoryEntryRequest
+	renameReq *filer_pb.AtomicRenameEntryRequest
+}
+
+func (s *fsMvTestFilerServer) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
+	s.lookupReq = req
+	if req.Directory == "/dst" && req.Name == "dir" {
+		return &filer_pb.LookupDirectoryEntryResponse{
+			Entry: &filer_pb.Entry{
+				Name:        "dir",
+				IsDirectory: true,
+				Attributes:  &filer_pb.FuseAttributes{},
+			},
+		}, nil
+	}
+	return nil, status.Error(codes.NotFound, "not found")
+}
+
+func (s *fsMvTestFilerServer) AtomicRenameEntry(_ context.Context, req *filer_pb.AtomicRenameEntryRequest) (*filer_pb.AtomicRenameEntryResponse, error) {
+	s.renameReq = req
+	return &filer_pb.AtomicRenameEntryResponse{}, nil
+}
+
+func TestFsMvMovesIntoExistingDestinationDirectory(t *testing.T) {
+	filerServer := &fsMvTestFilerServer{}
+	commandEnv, cleanup := newFsMvTestCommandEnv(t, filerServer)
+	defer cleanup()
+
+	var output bytes.Buffer
+	err := (&commandFsMv{}).Do([]string{"/src/file", "/dst/dir"}, commandEnv, &output)
+	if err != nil {
+		t.Fatalf("fs.mv returned error: %v", err)
+	}
+
+	if filerServer.lookupReq == nil {
+		t.Fatal("expected fs.mv to look up destination entry")
+	}
+	if filerServer.lookupReq.Directory != "/dst" || filerServer.lookupReq.Name != "dir" {
+		t.Fatalf("destination lookup = directory %q name %q, want /dst dir", filerServer.lookupReq.Directory, filerServer.lookupReq.Name)
+	}
+	if filerServer.renameReq == nil {
+		t.Fatal("expected fs.mv to issue rename")
+	}
+	if filerServer.renameReq.NewDirectory != "/dst/dir" || filerServer.renameReq.NewName != "file" {
+		t.Fatalf("rename target = directory %q name %q, want /dst/dir file", filerServer.renameReq.NewDirectory, filerServer.renameReq.NewName)
+	}
+}
+
+func newFsMvTestCommandEnv(t *testing.T, filerServer filer_pb.SeaweedFilerServer) (*CommandEnv, func()) {
+	t.Helper()
+
+	socketDir, err := os.MkdirTemp("", "swmv-")
+	if err != nil {
+		t.Fatalf("create socket dir: %v", err)
+	}
+	t.Cleanup(func() { _ = os.RemoveAll(socketDir) })
+
+	socketPath := filepath.Join(socketDir, "filer.sock")
+	listener, err := net.Listen("unix", socketPath)
+	if err != nil {
+		t.Fatalf("listen unix socket: %v", err)
+	}
+
+	grpcServer := grpc.NewServer()
+	filer_pb.RegisterSeaweedFilerServer(grpcServer, filerServer)
+	go func() {
+		_ = grpcServer.Serve(listener)
+	}()
+
+	grpcPort := 47000 + os.Getpid()%1000
+	pb.RegisterLocalGrpcSocket("127.0.0.1", grpcPort, socketPath)
+
+	cleanup := func() {
+		grpcServer.Stop()
+		_ = listener.Close()
+	}
+
+	return &CommandEnv{
+		option: &ShellOptions{
+			FilerAddress:   pb.ServerAddress(fmt.Sprintf("127.0.0.1:8888.%d", grpcPort)),
+			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+			Directory:      "/",
+		},
+	}, cleanup
+}

weed/shell/command_fs_pwd.go

@@ -0,0 +1,36 @@
+package shell
+
+import (
+	"fmt"
+	"io"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsPwd{})
+}
+
+type commandFsPwd struct {
+}
+
+func (c *commandFsPwd) Name() string {
+	return "fs.pwd"
+}
+
+func (c *commandFsPwd) Help() string {
+	return `print out current directory`
+}
+
+func (c *commandFsPwd) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsPwd) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	fmt.Fprintf(writer, "%s\n", commandEnv.option.Directory)
+
+	return nil
+}

weed/shell/command_fs_rm.go

@@ -0,0 +1,109 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsRm{})
+}
+
+type commandFsRm struct {
+}
+
+func (c *commandFsRm) Name() string {
+	return "fs.rm"
+}
+
+func (c *commandFsRm) Help() string {
+	return `remove file and directory entries
+
+	fs.rm [-rf] <entry1> <entry2> ...
+
+	fs.rm /dir/file_name1 dir/file_name2
+	fs.rm /dir
+
+	The option "-r" can be recursive.
+	The option "-f" can be ignored by recursive error.
+`
+}
+
+func (c *commandFsRm) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsRm) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	isRecursive := false
+	ignoreRecursiveError := false
+	var entries []string
+	for _, arg := range args {
+		if !strings.HasPrefix(arg, "-") {
+			entries = append(entries, arg)
+			continue
+		}
+		for _, t := range arg {
+			switch t {
+			case 'r':
+				isRecursive = true
+			case 'f':
+				ignoreRecursiveError = true
+			}
+		}
+	}
+	if len(entries) < 1 {
+		return fmt.Errorf("need to have arguments")
+	}
+
+	commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		for _, entry := range entries {
+			targetPath, err := commandEnv.parseUrl(entry)
+			if err != nil {
+				fmt.Fprintf(writer, "rm: %s: %v\n", targetPath, err)
+				continue
+			}
+
+			targetDir, targetName := util.FullPath(targetPath).DirAndName()
+
+			lookupRequest := &filer_pb.LookupDirectoryEntryRequest{
+				Directory: targetDir,
+				Name:      targetName,
+			}
+			_, err = filer_pb.LookupEntry(context.Background(), client, lookupRequest)
+			if err != nil {
+				fmt.Fprintf(writer, "rm: %s: %v\n", targetPath, err)
+				continue
+			}
+
+			request := &filer_pb.DeleteEntryRequest{
+				Directory:            targetDir,
+				Name:                 targetName,
+				IgnoreRecursiveError: ignoreRecursiveError,
+				IsDeleteData:         true,
+				IsRecursive:          isRecursive,
+				IsFromOtherCluster:   false,
+				Signatures:           nil,
+			}
+			if resp, err := client.DeleteEntry(context.Background(), request); err != nil {
+				fmt.Fprintf(writer, "rm: %s: %v\n", targetPath, err)
+			} else {
+				if resp.Error != "" {
+					fmt.Fprintf(writer, "rm: %s: %v\n", targetPath, resp.Error)
+				}
+			}
+		}
+		return nil
+	})
+
+	return
+}

weed/shell/command_fs_tree.go

@@ -0,0 +1,122 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsTree{})
+}
+
+type commandFsTree struct {
+}
+
+func (c *commandFsTree) Name() string {
+	return "fs.tree"
+}
+
+func (c *commandFsTree) Help() string {
+	return `recursively list all files under a directory
+
+	fs.tree /some/dir
+
+`
+}
+
+func (c *commandFsTree) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsTree) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	if handleHelpRequest(c, args, writer) {
+		return nil
+	}
+
+	path, err := commandEnv.parseUrl(findInputDirectory(args))
+	if err != nil {
+		return err
+	}
+
+	dir, name := util.FullPath(path).DirAndName()
+
+	dirCount, fCount, terr := treeTraverseDirectory(writer, commandEnv, util.FullPath(dir), name, newPrefix(), -1)
+
+	if terr == nil {
+		fmt.Fprintf(writer, "%d directories, %d files\n", dirCount, fCount)
+	}
+
+	return terr
+
+}
+
+func treeTraverseDirectory(writer io.Writer, filerClient filer_pb.FilerClient, dir util.FullPath, name string, prefix *Prefix, level int) (directoryCount, fileCount int64, err error) {
+
+	prefix.addMarker(level)
+
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, dir, name, func(entry *filer_pb.Entry, isLast bool) error {
+		if level < 0 && name != "" {
+			if entry.Name != name {
+				return nil
+			}
+		}
+
+		fmt.Fprintf(writer, "%s%s\n", prefix.getPrefix(level, isLast), entry.Name)
+
+		if entry.IsDirectory {
+			directoryCount++
+			subDir := dir.Child(entry.Name)
+			dirCount, fCount, terr := treeTraverseDirectory(writer, filerClient, subDir, "", prefix, level+1)
+			directoryCount += dirCount
+			fileCount += fCount
+			err = terr
+		} else {
+			fileCount++
+		}
+		return nil
+	})
+	return
+}
+
+type Prefix struct {
+	markers map[int]bool
+}
+
+func newPrefix() *Prefix {
+	return &Prefix{
+		markers: make(map[int]bool),
+	}
+}
+func (p *Prefix) addMarker(marker int) {
+	p.markers[marker] = true
+}
+func (p *Prefix) removeMarker(marker int) {
+	delete(p.markers, marker)
+}
+func (p *Prefix) getPrefix(level int, isLastChild bool) string {
+	var sb strings.Builder
+	if level < 0 {
+		return ""
+	}
+	for i := 0; i < level; i++ {
+		if _, ok := p.markers[i]; ok {
+			sb.WriteString("│")
+		} else {
+			sb.WriteString(" ")
+		}
+		sb.WriteString("   ")
+	}
+	if isLastChild {
+		sb.WriteString("└──")
+		p.removeMarker(level)
+	} else {
+		sb.WriteString("├──")
+	}
+	return sb.String()
+}

weed/shell/command_fs_verify.go

@@ -0,0 +1,324 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"sync"
+	"time"
+
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"go.uber.org/atomic"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsVerify{})
+}
+
+type commandFsVerify struct {
+	env                *CommandEnv
+	volumeServers      []pb.ServerAddress
+	volumeIds          map[uint32][]pb.ServerAddress
+	verbose            *bool
+	metadataFromLog    *bool
+	concurrency        *int
+	modifyTimeAgoAtSec int64
+	writer             io.Writer
+	waitChan           map[string]chan struct{}
+	waitChanLock       sync.RWMutex
+}
+
+func (c *commandFsVerify) Name() string {
+	return "fs.verify"
+}
+
+func (c *commandFsVerify) Help() string {
+	return `recursively verify all files under a directory
+
+	fs.verify [-v] [-modifyTimeAgo 1h] /buckets/dir
+
+`
+}
+
+func (c *commandFsVerify) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	c.env = commandEnv
+	c.writer = writer
+	fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
+	modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
+	c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server")
+	c.metadataFromLog = fsVerifyCommand.Bool("metadataFromLog", false, "Using  filer log to get metadata")
+	if err = fsVerifyCommand.Parse(args); err != nil {
+		return err
+	}
+
+	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsVerifyCommand.Args()))
+	if parseErr != nil {
+		return parseErr
+	}
+
+	c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
+	c.volumeIds = make(map[uint32][]pb.ServerAddress)
+	c.waitChan = make(map[string]chan struct{})
+	c.volumeServers = []pb.ServerAddress{}
+	defer func() {
+		c.modifyTimeAgoAtSec = 0
+		c.volumeIds = nil
+		c.waitChan = nil
+		c.volumeServers = nil
+	}()
+
+	if err := c.collectVolumeIds(); err != nil {
+		return err
+	}
+
+	if *c.concurrency > 0 {
+		for _, volumeServer := range c.volumeServers {
+			volumeServerStr := string(volumeServer)
+			c.waitChan[volumeServerStr] = make(chan struct{}, *c.concurrency)
+			defer close(c.waitChan[volumeServerStr])
+		}
+	}
+	var fCount, eCount uint64
+	if *c.metadataFromLog {
+		var wg sync.WaitGroup
+		fCount, eCount, err = c.verifyProcessMetadata(path, &wg)
+		wg.Wait()
+		if err != nil {
+			return err
+		}
+	} else {
+		fCount, eCount, err = c.verifyTraverseBfs(path)
+	}
+	fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eCount)
+	return err
+}
+
+func (c *commandFsVerify) collectVolumeIds() error {
+	topologyInfo, _, err := collectTopologyInfo(c.env, 0)
+	if err != nil {
+		return err
+	}
+	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
+		for _, diskInfo := range nodeInfo.DiskInfos {
+			for _, vi := range diskInfo.VolumeInfos {
+				volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
+				c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
+				if !slices.Contains(c.volumeServers, volumeServer) {
+					c.volumeServers = append(c.volumeServers, volumeServer)
+				}
+			}
+			for _, vi := range diskInfo.EcShardInfos {
+				volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
+				c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
+				if !slices.Contains(c.volumeServers, volumeServer) {
+					c.volumeServers = append(c.volumeServers, volumeServer)
+				}
+			}
+		}
+	})
+	return nil
+}
+
+func (c *commandFsVerify) verifyChunk(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error {
+	err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			_, err := client.VolumeNeedleStatus(context.Background(),
+				&volume_server_pb.VolumeNeedleStatusRequest{
+					VolumeId: fileId.VolumeId,
+					NeedleId: fileId.FileKey})
+			return err
+		},
+	)
+	if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
+		return err
+	}
+	return nil
+}
+
+type ItemEntry struct {
+	chunks []*filer_pb.FileChunk
+	path   util.FullPath
+}
+
+func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup) (fileCount uint64, errCount uint64, err error) {
+	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
+		message := resp.EventNotification
+		if resp.EventNotification.NewEntry == nil {
+			return nil
+		}
+		chunkCount := len(message.NewEntry.Chunks)
+		if chunkCount == 0 {
+			return nil
+		}
+		entryPath := fmt.Sprintf("%s/%s", message.NewParentPath, message.NewEntry.Name)
+		errorChunksCount := atomic.NewUint64(0)
+		if !c.verifyEntry(entryPath, message.NewEntry.Chunks, errorChunksCount, wg) {
+			if err = c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+				entryResp, errReq := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+					Directory: message.NewParentPath,
+					Name:      message.NewEntry.Name,
+				})
+				if errReq != nil {
+					if strings.HasSuffix(errReq.Error(), "no entry is found in filer store") {
+						return nil
+					}
+					return errReq
+				}
+				if entryResp.Entry.Attributes.Mtime == message.NewEntry.Attributes.Mtime &&
+					bytes.Equal(entryResp.Entry.Attributes.Md5, message.NewEntry.Attributes.Md5) {
+					fmt.Fprintf(c.writer, "file: %s needles:%d failed:%d\n", entryPath, chunkCount, errorChunksCount.Load())
+					errCount++
+				}
+				return nil
+			}); err != nil {
+				return err
+			}
+			return nil
+		}
+		if *c.verbose {
+			fmt.Fprintf(c.writer, "file: %s needles:%d verified\n", entryPath, chunkCount)
+		}
+		fileCount++
+		return nil
+	}
+	metadataFollowOption := &pb.MetadataFollowOption{
+		ClientName:             "shell_verify",
+		ClientId:               util.RandomInt32(),
+		ClientEpoch:            0,
+		SelfSignature:          0,
+		PathPrefix:             path,
+		AdditionalPathPrefixes: nil,
+		DirectoriesToWatch:     nil,
+		StartTsNs:              time.Now().Add(-1 * time.Second * time.Duration(c.modifyTimeAgoAtSec)).UnixNano(),
+		StopTsNs:               time.Now().UnixNano(),
+		EventErrorType:         pb.DontLogError,
+	}
+	return fileCount, errCount, pb.FollowMetadata(c.env.option.FilerAddress, c.env.option.GrpcDialOption, metadataFollowOption, processEventFn)
+}
+
+func (c *commandFsVerify) verifyEntry(path string, chunks []*filer_pb.FileChunk, errorCount *atomic.Uint64, wg *sync.WaitGroup) bool {
+	fileMsg := fmt.Sprintf("file:%s", path)
+	itemIsVerifed := atomic.NewBool(true)
+	for _, chunk := range chunks {
+		if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
+			for _, volumeServer := range volumeIds {
+				if *c.concurrency == 0 {
+					if err := c.verifyChunk(volumeServer, chunk.Fid); err != nil {
+						if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) {
+							fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v, at volume server %v\n",
+								fileMsg, chunk.GetFileIdString(), err, volumeServer)
+						}
+						if itemIsVerifed.Load() {
+							itemIsVerifed.Store(false)
+							errorCount.Add(1)
+						}
+					}
+					continue
+				}
+				c.waitChanLock.RLock()
+				waitChan, ok := c.waitChan[string(volumeServer)]
+				c.waitChanLock.RUnlock()
+				if !ok {
+					fmt.Fprintf(c.writer, "%s failed to get channel for %s fileId: %s\n",
+						string(volumeServer), fileMsg, chunk.GetFileIdString())
+					if itemIsVerifed.Load() {
+						itemIsVerifed.Store(false)
+						errorCount.Add(1)
+					}
+					continue
+				}
+				wg.Add(1)
+				waitChan <- struct{}{}
+				go func(fChunk *filer_pb.FileChunk, path string, volumeServer pb.ServerAddress, msg string) {
+					defer wg.Done()
+					if err := c.verifyChunk(volumeServer, fChunk.Fid); err != nil {
+						if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) {
+							fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v, at volume server %v\n",
+								msg, fChunk.GetFileIdString(), err, volumeServer)
+						}
+						if itemIsVerifed.Load() {
+							itemIsVerifed.Store(false)
+							errorCount.Add(1)
+						}
+					}
+					<-waitChan
+				}(chunk, path, volumeServer, fileMsg)
+			}
+		} else {
+			if !*c.metadataFromLog {
+				err := fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
+				fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n",
+					fileMsg, chunk.GetFileIdString(), err)
+			}
+			if itemIsVerifed.Load() {
+				itemIsVerifed.Store(false)
+				errorCount.Add(1)
+			}
+			break
+		}
+	}
+	return itemIsVerifed.Load()
+}
+
+func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errCount uint64, err error) {
+	timeNowAtSec := time.Now().Unix()
+	return fileCount, errCount, doTraverseBfsAndSaving(c.env, c.writer, path, false, false,
+		func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
+			if c.modifyTimeAgoAtSec > 0 {
+				if entry.Entry.Attributes != nil && c.modifyTimeAgoAtSec < timeNowAtSec-entry.Entry.Attributes.Mtime {
+					return nil
+				}
+			}
+			dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64, nil)
+			if resolveErr != nil {
+				return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr)
+			}
+			dataChunks = append(dataChunks, manifestChunks...)
+			if len(dataChunks) > 0 {
+				select {
+				case outputChan <- &ItemEntry{
+					chunks: dataChunks,
+					path:   util.NewFullPath(entry.Dir, entry.Entry.Name),
+				}:
+				case <-ctx.Done():
+					return ctx.Err()
+				}
+			}
+			return nil
+		},
+		func(outputChan chan interface{}) error {
+			var wg sync.WaitGroup
+			itemErrCount := atomic.NewUint64(0)
+			for itemEntry := range outputChan {
+				i := itemEntry.(*ItemEntry)
+				itemPath := string(i.path)
+				if c.verifyEntry(itemPath, i.chunks, itemErrCount, &wg) {
+					if *c.verbose {
+						fmt.Fprintf(c.writer, "file: %s needles:%d verified\n", itemPath, len(i.chunks))
+					}
+					fileCount++
+				}
+			}
+			wg.Wait()
+			errCount = itemErrCount.Load()
+			return nil
+		})
+}

weed/shell/command_lock_unlock.go

@@ -0,0 +1,84 @@
+package shell
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandUnlock{})
+	Commands = append(Commands, &commandLock{})
+}
+
+// =========== Lock ==============
+type commandLock struct {
+}
+
+func (c *commandLock) Name() string {
+	return "lock"
+}
+
+func (c *commandLock) Help() string {
+	return `lock in order to exclusively manage the cluster
+
+	This is a blocking operation if there is already another lock.
+`
+}
+
+func (c *commandLock) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandLock) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	waited := false
+	if !commandEnv.locker.IsLocked() {
+		if holder, message, held := commandEnv.shellLockHolder(); held {
+			waited = true
+			if holder == "" {
+				holder = "another client"
+			}
+			if message != "" {
+				fmt.Fprintf(writer, "waiting for lock held by %s: %s\n", holder, message)
+			} else {
+				fmt.Fprintf(writer, "waiting for lock held by %s\n", holder)
+			}
+		}
+	}
+
+	commandEnv.locker.RequestLock(util.DetectedHostAddress())
+
+	if waited {
+		fmt.Fprintln(writer, "lock acquired")
+	}
+
+	return nil
+}
+
+// =========== Unlock ==============
+
+type commandUnlock struct {
+}
+
+func (c *commandUnlock) Name() string {
+	return "unlock"
+}
+
+func (c *commandUnlock) Help() string {
+	return `unlock the cluster-wide lock
+
+`
+}
+
+func (c *commandUnlock) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandUnlock) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	commandEnv.locker.ReleaseLock()
+
+	return nil
+}

weed/shell/command_mount_configure.go

@@ -0,0 +1,69 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	_ "google.golang.org/grpc/resolver/passthrough"
+)
+
+func init() {
+	Commands = append(Commands, &commandMountConfigure{})
+}
+
+type commandMountConfigure struct {
+}
+
+func (c *commandMountConfigure) Name() string {
+	return "mount.configure"
+}
+
+func (c *commandMountConfigure) Help() string {
+	return `configure the mount on current server
+
+	mount.configure -dir=<mount_directory>
+
+	This command connects with local mount via unix socket, so it can only run locally.
+	The "mount_directory" value needs to be exactly the same as how mount was started in "weed mount -dir=<mount_directory>"
+
+`
+}
+
+func (c *commandMountConfigure) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMountConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	mountConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	mountDir := mountConfigureCommand.String("dir", "", "the mount directory same as how \"weed mount -dir=<mount_directory>\" was started")
+	mountQuota := mountConfigureCommand.Int("quotaMB", 0, "the quota in MB")
+	if err = mountConfigureCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	mountDirHash := util.HashToInt32([]byte(*mountDir))
+	if mountDirHash < 0 {
+		mountDirHash = -mountDirHash
+	}
+	localSocket := fmt.Sprintf("/tmp/seaweedfs-mount-%d.sock", mountDirHash)
+
+	clientConn, err := grpc.NewClient("passthrough:///unix://"+localSocket, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return
+	}
+	defer clientConn.Close()
+
+	client := mount_pb.NewSeaweedMountClient(clientConn)
+	_, err = client.Configure(context.Background(), &mount_pb.ConfigureRequest{
+		CollectionCapacity: int64(*mountQuota) * 1024 * 1024,
+	})
+
+	return
+}

weed/shell/command_mq_balance.go

@@ -0,0 +1,51 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqBalanceTopics{})
+}
+
+type commandMqBalanceTopics struct {
+}
+
+func (c *commandMqBalanceTopics) Name() string {
+	return "mq.balance"
+}
+
+func (c *commandMqBalanceTopics) Help() string {
+	return `balance topic partitions
+
+`
+}
+
+func (c *commandMqBalanceTopics) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMqBalanceTopics) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+
+	// find the broker balancer
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
+
+	// balance topics
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		_, err := client.BalanceTopics(context.Background(), &mq_pb.BalanceTopicsRequest{})
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+}

weed/shell/command_mq_topic_compact.go

@@ -0,0 +1,95 @@
+package shell
+
+import (
+	"flag"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer_client"
+	"github.com/seaweedfs/seaweedfs/weed/mq/logstore"
+	"github.com/seaweedfs/seaweedfs/weed/mq/schema"
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicCompact{})
+}
+
+type commandMqTopicCompact struct {
+}
+
+func (c *commandMqTopicCompact) Name() string {
+	return "mq.topic.compact"
+}
+
+func (c *commandMqTopicCompact) Help() string {
+	return `compact the topic storage into parquet format
+
+	Example:
+		mq.topic.compact -namespace <namespace> -topic <topic_name> -timeAgo <time_ago>
+
+`
+}
+
+func (c *commandMqTopicCompact) HasTag(tag CommandTag) bool {
+	return ResourceHeavy == tag
+}
+
+func (c *commandMqTopicCompact) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+
+	// parse parameters
+	mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	namespace := mqCommand.String("namespace", "", "namespace name")
+	topicName := mqCommand.String("topic", "", "topic name")
+	timeAgo := mqCommand.Duration("timeAgo", 2*time.Minute, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	replication := mqCommand.String("replication", "", "replication type")
+	collection := mqCommand.String("collection", "", "optional collection name")
+	dataCenter := mqCommand.String("dataCenter", "", "optional data center name")
+	diskType := mqCommand.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	maxMB := mqCommand.Int("maxMB", 4, "split files larger than the limit")
+
+	if err := mqCommand.Parse(args); err != nil {
+		return err
+	}
+
+	storagePreference := &operation.StoragePreference{
+		Replication: *replication,
+		Collection:  *collection,
+		DataCenter:  *dataCenter,
+		DiskType:    *diskType,
+		MaxMB:       *maxMB,
+	}
+
+	// read topic configuration
+	fca := filer_client.NewFilerClientAccessor(
+		[]pb.ServerAddress{commandEnv.option.FilerAddress},
+		commandEnv.option.GrpcDialOption,
+	)
+	t := topic.NewTopic(*namespace, *topicName)
+	topicConf, err := fca.ReadTopicConfFromFiler(t)
+	if err != nil {
+		return err
+	}
+
+	// get record type - prefer flat schema if available
+	var recordType *schema_pb.RecordType
+	if topicConf.GetMessageRecordType() != nil {
+		// New flat schema format - use directly
+		recordType = topicConf.GetMessageRecordType()
+	}
+	recordType = schema.NewRecordTypeBuilder(recordType).
+		WithField(logstore.SW_COLUMN_NAME_TS, schema.TypeInt64).
+		WithField(logstore.SW_COLUMN_NAME_KEY, schema.TypeBytes).
+		RecordTypeEnd()
+
+	// compact the topic partition versions
+	if err = logstore.CompactTopicPartitions(commandEnv, t, *timeAgo, recordType, storagePreference); err != nil {
+		return err
+	}
+
+	return nil
+
+}

weed/shell/command_mq_topic_configure.go

@@ -0,0 +1,164 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicConfigure{})
+}
+
+type commandMqTopicConfigure struct {
+}
+
+func (c *commandMqTopicConfigure) Name() string {
+	return "mq.topic.configure"
+}
+
+func (c *commandMqTopicConfigure) Help() string {
+	return `configure a topic with a given name
+
+	Example:
+		mq.topic.configure -namespace <namespace> -topic <topic_name> -partitionCount <partition_count>
+
+	Retention (delete messages older than the configured duration):
+		mq.topic.configure -namespace <namespace> -topic <topic_name> \
+			-retention 168h -retentionEnabled
+
+		# disable retention on an existing topic
+		mq.topic.configure -namespace <namespace> -topic <topic_name> \
+			-retentionEnabled=false
+
+	-retention accepts any Go duration string ("24h", "168h", "30m"). Use
+	-retentionSeconds for raw seconds when scripting. Specifying both is an
+	error.
+
+	When you set only some retention flags (for example, -retentionEnabled
+	without -retention), the unspecified field is read from the current
+	server-side configuration so it isn't accidentally zeroed. Omitting all
+	retention flags leaves the existing retention configuration alone.
+`
+}
+
+func (c *commandMqTopicConfigure) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMqTopicConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+
+	// parse parameters
+	mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	namespace := mqCommand.String("namespace", "", "namespace name")
+	topicName := mqCommand.String("topic", "", "topic name")
+	partitionCount := mqCommand.Int("partitionCount", 6, "partition count")
+	retention := mqCommand.Duration("retention", 0, "retention duration (Go duration string, e.g. 168h). Mutually exclusive with -retentionSeconds.")
+	retentionSeconds := mqCommand.Int64("retentionSeconds", 0, "retention duration in seconds. Mutually exclusive with -retention.")
+	retentionEnabled := mqCommand.Bool("retentionEnabled", false, "enable retention enforcement on the topic")
+	if err := mqCommand.Parse(args); err != nil {
+		return err
+	}
+
+	// Detect which retention flags the user actually provided. Using Visit
+	// (rather than value comparison) means an explicit `-retention=0` is
+	// treated as "user provided" and still triggers the mutual-exclusion
+	// check or partial-merge with current state.
+	var userSetRetention, userSetSeconds, userSetEnabled bool
+	mqCommand.Visit(func(f *flag.Flag) {
+		switch f.Name {
+		case "retention":
+			userSetRetention = true
+		case "retentionSeconds":
+			userSetSeconds = true
+		case "retentionEnabled":
+			userSetEnabled = true
+		}
+	})
+
+	if userSetRetention && userSetSeconds {
+		return fmt.Errorf("-retention and -retentionSeconds are mutually exclusive")
+	}
+	if *retention < 0 || *retentionSeconds < 0 {
+		return fmt.Errorf("retention duration must be >= 0")
+	}
+
+	retentionTouched := userSetRetention || userSetSeconds || userSetEnabled
+
+	// find the broker balancer
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
+
+	// Build the retention proto. When the user touches any retention flag we
+	// must send a fully-populated TopicRetention so partial flags don't zero
+	// the other field server-side: fetch the current configuration and use
+	// its values for whatever the user didn't specify.
+	var retentionProto *mq_pb.TopicRetention
+	if retentionTouched {
+		var currentRetention *mq_pb.TopicRetention
+		if err := pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+			cur, getErr := client.GetTopicConfiguration(context.Background(), &mq_pb.GetTopicConfigurationRequest{
+				Topic: &schema_pb.Topic{Namespace: *namespace, Name: *topicName},
+			})
+			if getErr != nil {
+				// Topic may not exist yet — that's fine, we'll create it with
+				// the user-supplied retention only.
+				return nil
+			}
+			if cur != nil {
+				currentRetention = cur.Retention
+			}
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		var seconds int64
+		var enabled bool
+		if currentRetention != nil {
+			seconds = currentRetention.RetentionSeconds
+			enabled = currentRetention.Enabled
+		}
+		if userSetRetention {
+			seconds = int64((*retention) / time.Second)
+		} else if userSetSeconds {
+			seconds = *retentionSeconds
+		}
+		if userSetEnabled {
+			enabled = *retentionEnabled
+		}
+		retentionProto = &mq_pb.TopicRetention{
+			RetentionSeconds: seconds,
+			Enabled:          enabled,
+		}
+	}
+
+	// create / update topic
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		resp, err := client.ConfigureTopic(context.Background(), &mq_pb.ConfigureTopicRequest{
+			Topic: &schema_pb.Topic{
+				Namespace: *namespace,
+				Name:      *topicName,
+			},
+			PartitionCount: int32(*partitionCount),
+			Retention:      retentionProto,
+		})
+		if err != nil {
+			return err
+		}
+		output, _ := json.MarshalIndent(resp, "", "  ")
+		fmt.Fprintf(writer, "response:\n%+v\n", string(output))
+		return nil
+	})
+
+}

weed/shell/command_mq_topic_desc.go

@@ -0,0 +1,64 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicDescribe{})
+}
+
+type commandMqTopicDescribe struct {
+}
+
+func (c *commandMqTopicDescribe) Name() string {
+	return "mq.topic.describe"
+}
+
+func (c *commandMqTopicDescribe) Help() string {
+	return `describe a topic`
+}
+
+func (c *commandMqTopicDescribe) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMqTopicDescribe) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	// parse parameters
+	mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	namespace := mqCommand.String("namespace", "", "namespace name")
+	topicName := mqCommand.String("topic", "", "topic name")
+	if err := mqCommand.Parse(args); err != nil {
+		return err
+	}
+
+	// find the broker balancer
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
+
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		resp, err := client.LookupTopicBrokers(context.Background(), &mq_pb.LookupTopicBrokersRequest{
+			Topic: &schema_pb.Topic{
+				Namespace: *namespace,
+				Name:      *topicName,
+			},
+		})
+		if err != nil {
+			return err
+		}
+		for _, assignment := range resp.BrokerPartitionAssignments {
+			fmt.Fprintf(writer, "  %+v\n", assignment)
+		}
+		return nil
+	})
+}

weed/shell/command_mq_topic_list.go

@@ -0,0 +1,69 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicList{})
+}
+
+type commandMqTopicList struct {
+}
+
+func (c *commandMqTopicList) Name() string {
+	return "mq.topic.list"
+}
+
+func (c *commandMqTopicList) Help() string {
+	return `print out all topics`
+}
+
+func (c *commandMqTopicList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMqTopicList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+
+	brokerBalancer, err := findBrokerBalancer(commandEnv)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(writer, "current balancer: %s\n", brokerBalancer)
+
+	return pb.WithBrokerGrpcClient(false, brokerBalancer, commandEnv.option.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
+		resp, err := client.ListTopics(context.Background(), &mq_pb.ListTopicsRequest{})
+		if err != nil {
+			return err
+		}
+		if len(resp.Topics) == 0 {
+			fmt.Fprintf(writer, "no topics found\n")
+			return nil
+		}
+		for _, topic := range resp.Topics {
+			fmt.Fprintf(writer, "  %+v\n", topic)
+		}
+		return nil
+	})
+}
+
+func findBrokerBalancer(commandEnv *CommandEnv) (brokerBalancer string, err error) {
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
+			Name: pub_balancer.LockBrokerBalancer,
+		})
+		if err != nil {
+			return fmt.Errorf("FindLockOwner: %w", err)
+		}
+		brokerBalancer = resp.Owner
+		return nil
+	})
+	return
+}

weed/shell/command_mq_topic_truncate.go

@@ -0,0 +1,140 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandMqTopicTruncate{})
+}
+
+type commandMqTopicTruncate struct {
+}
+
+func (c *commandMqTopicTruncate) Name() string {
+	return "mq.topic.truncate"
+}
+
+func (c *commandMqTopicTruncate) Help() string {
+	return `clear all data from a topic while preserving topic structure
+
+	Example:
+		mq.topic.truncate -namespace <namespace> -topic <topic_name>
+
+	This command removes all log files and parquet files from all partitions
+	of the specified topic, while keeping the topic configuration intact.
+`
+}
+
+func (c *commandMqTopicTruncate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandMqTopicTruncate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	// parse parameters
+	mqCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	namespace := mqCommand.String("namespace", "", "namespace name")
+	topicName := mqCommand.String("topic", "", "topic name")
+	if err := mqCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *namespace == "" {
+		return fmt.Errorf("namespace is required")
+	}
+	if *topicName == "" {
+		return fmt.Errorf("topic name is required")
+	}
+
+	// Verify topic exists by trying to read its configuration
+	t := topic.NewTopic(*namespace, *topicName)
+
+	err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		_, err := t.ReadConfFile(client)
+		if err != nil {
+			return fmt.Errorf("topic %s.%s does not exist or cannot be read: %v", *namespace, *topicName, err)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Truncating topic %s.%s...\n", *namespace, *topicName)
+
+	// Discover and clear all partitions using centralized logic
+	partitions, err := t.DiscoverPartitions(context.Background(), commandEnv)
+	if err != nil {
+		return fmt.Errorf("failed to discover topic partitions: %v", err)
+	}
+
+	if len(partitions) == 0 {
+		fmt.Fprintf(writer, "No partitions found for topic %s.%s\n", *namespace, *topicName)
+		return nil
+	}
+
+	fmt.Fprintf(writer, "Found %d partitions, clearing data...\n", len(partitions))
+
+	// Clear data from each partition
+	totalFilesDeleted := 0
+	for _, partitionPath := range partitions {
+		filesDeleted, err := c.clearPartitionData(commandEnv, partitionPath, writer)
+		if err != nil {
+			fmt.Fprintf(writer, "Warning: failed to clear partition %s: %v\n", partitionPath, err)
+			continue
+		}
+		totalFilesDeleted += filesDeleted
+		fmt.Fprintf(writer, "Cleared partition: %s (%d files)\n", partitionPath, filesDeleted)
+	}
+
+	fmt.Fprintf(writer, "Successfully truncated topic %s.%s - deleted %d files from %d partitions\n",
+		*namespace, *topicName, totalFilesDeleted, len(partitions))
+
+	return nil
+}
+
+// clearPartitionData deletes all data files (log files, parquet files) from a partition directory
+// Returns the number of files deleted
+func (c *commandMqTopicTruncate) clearPartitionData(commandEnv *CommandEnv, partitionPath string, writer io.Writer) (int, error) {
+	filesDeleted := 0
+
+	err := filer_pb.ReadDirAllEntries(context.Background(), commandEnv, util.FullPath(partitionPath), "", func(entry *filer_pb.Entry, isLast bool) error {
+		if entry.IsDirectory {
+			return nil // Skip subdirectories
+		}
+
+		fileName := entry.Name
+
+		// Preserve configuration files
+		if strings.HasSuffix(fileName, ".conf") ||
+			strings.HasSuffix(fileName, ".config") ||
+			fileName == "topic.conf" ||
+			fileName == "partition.conf" {
+			fmt.Fprintf(writer, "  Preserving config file: %s\n", fileName)
+			return nil
+		}
+
+		// Delete all data files (log files, parquet files, offset files, etc.)
+		deleteErr := filer_pb.Remove(context.Background(), commandEnv, partitionPath, fileName, false, true, true, false, nil)
+
+		if deleteErr != nil {
+			fmt.Fprintf(writer, "  Warning: failed to delete %s/%s: %v\n", partitionPath, fileName, deleteErr)
+			// Continue with other files rather than failing entirely
+		} else {
+			fmt.Fprintf(writer, "  Deleted: %s\n", fileName)
+			filesDeleted++
+		}
+
+		return nil
+	})
+
+	return filesDeleted, err
+}

weed/shell/command_remote_cache.go

@@ -0,0 +1,424 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteCache{})
+}
+
+type commandRemoteCache struct {
+}
+
+func (c *commandRemoteCache) Name() string {
+	return "remote.cache"
+}
+
+func (c *commandRemoteCache) Help() string {
+	return `comprehensive synchronization and caching between local and remote storage
+
+	# assume a remote storage is configured to name "cloud1"
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+	# mount and pull one bucket
+	remote.mount -dir=/xxx -remote=cloud1/bucket
+
+	# comprehensive sync and cache: update metadata, cache content, and remove deleted files
+	remote.cache -dir=/xxx                                # sync metadata, cache content, and remove deleted files (default)
+	remote.cache -dir=/xxx -cacheContent=false            # sync metadata and cleanup only, no caching
+	remote.cache -dir=/xxx -deleteLocalExtra=false        # skip removal of local files missing from remote
+	remote.cache -dir=/xxx -concurrent=32                 # with custom file-level concurrency
+	remote.cache -dir=/xxx -chunkConcurrency=16           # parallel chunk downloads per file (0 = server default 8)
+	remote.cache -dir=/xxx -downloadConcurrency=10        # S3 multipart download concurrency per chunk (0 = server default 5)
+	remote.cache -dir=/xxx -include=*.pdf                 # only sync PDF files
+	remote.cache -dir=/xxx -exclude=*.tmp                 # exclude temporary files
+	remote.cache -dir=/xxx -dryRun=true                   # show what would be done without making changes
+
+	This command will:
+	1. Synchronize metadata from remote storage
+	2. Cache file content from remote by default
+	3. Remove local files that no longer exist on remote by default (use -deleteLocalExtra=false to disable)
+
+	This is designed to run regularly. So you can add it to some cronjob.
+
+`
+}
+
+func (c *commandRemoteCache) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteCache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteCacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteCacheCommand.String("dir", "", "a directory in filer")
+	cache := remoteCacheCommand.Bool("cacheContent", true, "cache file content from remote")
+	deleteLocalExtra := remoteCacheCommand.Bool("deleteLocalExtra", true, "delete local files that no longer exist on remote")
+	concurrency := remoteCacheCommand.Int("concurrent", 16, "concurrent file operations")
+	chunkConcurrency := remoteCacheCommand.Int("chunkConcurrency", 0, "parallel chunk downloads per file (0 = server default 8)")
+	downloadConcurrency := remoteCacheCommand.Int("downloadConcurrency", 0, "S3 multipart download concurrency per chunk (0 = server default 5)")
+	dryRun := remoteCacheCommand.Bool("dryRun", false, "show what would be done without making changes")
+	fileFiler := newFileFilter(remoteCacheCommand)
+
+	if err = remoteCacheCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *dir == "" {
+		return fmt.Errorf("need to specify -dir option")
+	}
+	if *chunkConcurrency < 0 || *chunkConcurrency > math.MaxInt32 {
+		return fmt.Errorf("chunkConcurrency must be between 0 and %d", math.MaxInt32)
+	}
+	if *downloadConcurrency < 0 || *downloadConcurrency > math.MaxInt32 {
+		return fmt.Errorf("downloadConcurrency must be between 0 and %d", math.MaxInt32)
+	}
+
+	mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
+	if detectErr != nil {
+		jsonPrintln(writer, mappings)
+		return detectErr
+	}
+
+	// perform comprehensive sync
+	return c.doComprehensiveSync(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *cache, *deleteLocalExtra, *concurrency, int32(*chunkConcurrency), int32(*downloadConcurrency), *dryRun, fileFiler)
+}
+
+func (c *commandRemoteCache) doComprehensiveSync(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToSync util.FullPath, remoteConf *remote_pb.RemoteConf, shouldCache bool, deleteLocalExtra bool, concurrency int, chunkConcurrency int32, downloadConcurrency int32, dryRun bool, fileFilter *FileFilter) error {
+
+	// visit remote storage
+	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
+	if err != nil {
+		return err
+	}
+
+	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToSync)
+
+	// Step 1: Collect all remote files
+	remoteFiles := make(map[string]*filer_pb.RemoteEntry)
+	err = remoteStorage.Traverse(remote, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
+		localDir := filer.MapRemoteStorageLocationPathToFullPath(localMountedDir, remoteMountedLocation, remoteDir)
+		fullPath := string(localDir.Child(name))
+		remoteFiles[fullPath] = remoteEntry
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to traverse remote storage: %w", err)
+	}
+
+	fmt.Fprintf(writer, "Found %d files/directories in remote storage\n", len(remoteFiles))
+
+	// Step 2: Collect all local files (only if we need to delete local extra files)
+	localFiles := make(map[string]*filer_pb.Entry)
+	if deleteLocalExtra {
+		err = recursivelyTraverseDirectory(commandEnv, dirToSync, func(dir util.FullPath, entry *filer_pb.Entry) bool {
+			if entry.RemoteEntry != nil { // only consider files that are part of remote mount
+				fullPath := string(dir.Child(entry.Name))
+				localFiles[fullPath] = entry
+			}
+			return true
+		})
+		if err != nil {
+			return fmt.Errorf("failed to traverse local directory: %w", err)
+		}
+		fmt.Fprintf(writer, "Found %d files/directories in local storage\n", len(localFiles))
+	} else {
+		fmt.Fprintf(writer, "Skipping local file collection (deleteLocalExtra=false)\n")
+	}
+
+	// Step 3: Determine actions needed
+	var filesToDelete []string
+	var filesToUpdate []string
+	var filesToCache []string
+
+	// Find files to delete (exist locally but not remotely) - only if deleteLocalExtra is enabled
+	if deleteLocalExtra {
+		for localPath := range localFiles {
+			if _, exists := remoteFiles[localPath]; !exists {
+				filesToDelete = append(filesToDelete, localPath)
+			}
+		}
+	}
+
+	// Find files to update/cache (exist remotely)
+	for remotePath, remoteEntry := range remoteFiles {
+		if deleteLocalExtra {
+			// When deleteLocalExtra is enabled, we have localFiles to compare with
+			if localEntry, exists := localFiles[remotePath]; exists {
+				// File exists locally, check if it needs updating
+				if localEntry.RemoteEntry == nil ||
+					localEntry.RemoteEntry.RemoteETag != remoteEntry.RemoteETag ||
+					localEntry.RemoteEntry.RemoteMtime < remoteEntry.RemoteMtime ||
+					(remoteEntry.RemoteContentEncoding != nil &&
+						localEntry.RemoteEntry.GetRemoteContentEncoding() != remoteEntry.GetRemoteContentEncoding()) {
+					filesToUpdate = append(filesToUpdate, remotePath)
+				}
+				// Check if it needs caching
+				if shouldCache && shouldCacheToLocal(localEntry) && fileFilter.matches(localEntry) {
+					filesToCache = append(filesToCache, remotePath)
+				}
+			} else {
+				// File doesn't exist locally, needs to be created
+				filesToUpdate = append(filesToUpdate, remotePath)
+			}
+		} else {
+			// When deleteLocalExtra is disabled, we check each file individually
+			// All remote files are candidates for update/creation
+			filesToUpdate = append(filesToUpdate, remotePath)
+
+			// For caching, we need to check if the local file exists and needs caching
+			if shouldCache {
+				// We need to look up the local file to check if it needs caching
+				localDir, name := util.FullPath(remotePath).DirAndName()
+				err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+					lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+						Directory: localDir,
+						Name:      name,
+					})
+					if lookupErr == nil {
+						localEntry := lookupResp.Entry
+						if shouldCacheToLocal(localEntry) && fileFilter.matches(localEntry) {
+							filesToCache = append(filesToCache, remotePath)
+						}
+					}
+					return nil // Don't propagate lookup errors here
+				})
+				if err != nil {
+					// Log error but continue
+					fmt.Fprintf(writer, "Warning: failed to lookup local file %s for caching check: %v\n", remotePath, err)
+				}
+			}
+		}
+	}
+
+	fmt.Fprintf(writer, "Actions needed: %d files to delete, %d files to update, %d files to cache\n",
+		len(filesToDelete), len(filesToUpdate), len(filesToCache))
+
+	if dryRun {
+		fmt.Fprintf(writer, "DRY RUN - showing what would be done:\n")
+		for _, path := range filesToDelete {
+			fmt.Fprintf(writer, "DELETE: %s\n", path)
+		}
+		for _, path := range filesToUpdate {
+			fmt.Fprintf(writer, "UPDATE: %s\n", path)
+		}
+		for _, path := range filesToCache {
+			fmt.Fprintf(writer, "CACHE: %s\n", path)
+		}
+		return nil
+	}
+
+	// Step 4: Execute actions
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		ctx := context.Background()
+
+		// Delete files that no longer exist on remote (only if deleteLocalExtra is enabled)
+		if deleteLocalExtra {
+			for _, pathToDelete := range filesToDelete {
+				fmt.Fprintf(writer, "Deleting %s... ", pathToDelete)
+
+				dir, name := util.FullPath(pathToDelete).DirAndName()
+				err := filer_pb.DoRemove(ctx, client, dir, name, true, false, false, false, nil)
+				if err != nil {
+					fmt.Fprintf(writer, "failed: %v\n", err)
+					return err
+				}
+				fmt.Fprintf(writer, "done\n")
+			}
+		}
+
+		// Update metadata for files that exist on remote
+		for _, pathToUpdate := range filesToUpdate {
+			remoteEntry := remoteFiles[pathToUpdate]
+			localDir, name := util.FullPath(pathToUpdate).DirAndName()
+
+			fmt.Fprintf(writer, "Updating metadata for %s... ", pathToUpdate)
+
+			// Check if file exists locally
+			lookupResp, lookupErr := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+				Directory: string(localDir),
+				Name:      name,
+			})
+
+			if lookupErr != nil && lookupErr != filer_pb.ErrNotFound {
+				fmt.Fprintf(writer, "failed to lookup: %v\n", lookupErr)
+				continue
+			}
+
+			isDirectory := remoteEntry.RemoteSize == 0 && remoteEntry.RemoteMtime == 0
+			if lookupErr == filer_pb.ErrNotFound {
+				// Create new entry
+				_, createErr := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+					Directory: string(localDir),
+					Entry: &filer_pb.Entry{
+						Name:        name,
+						IsDirectory: isDirectory,
+						Attributes: &filer_pb.FuseAttributes{
+							FileSize: uint64(remoteEntry.RemoteSize),
+							Mtime:    remoteEntry.RemoteMtime,
+							FileMode: remoteEntryFileMode(isDirectory),
+						},
+						Extended:    filer.MergeRemoteContentEncoding(remoteEntry, nil),
+						RemoteEntry: remoteEntry,
+					},
+				})
+				if createErr != nil {
+					fmt.Fprintf(writer, "failed to create: %v\n", createErr)
+					continue
+				}
+			} else {
+				// Update existing entry
+				existingEntry := lookupResp.Entry
+				if existingEntry.RemoteEntry == nil {
+					// This is a local file, skip to avoid overwriting
+					fmt.Fprintf(writer, "skipped (local file)\n")
+					continue
+				}
+
+				existingEntry.RemoteEntry = remoteEntry
+				existingEntry.Attributes.FileSize = uint64(remoteEntry.RemoteSize)
+				existingEntry.Attributes.Mtime = remoteEntry.RemoteMtime
+				existingEntry.Attributes.Md5 = nil
+				existingEntry.Extended = filer.MergeRemoteContentEncoding(remoteEntry, existingEntry.Extended)
+				existingEntry.Chunks = nil
+				existingEntry.Content = nil
+
+				_, updateErr := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
+					Directory: string(localDir),
+					Entry:     existingEntry,
+				})
+				if updateErr != nil {
+					fmt.Fprintf(writer, "failed to update: %v\n", updateErr)
+					continue
+				}
+			}
+			fmt.Fprintf(writer, "done\n")
+		}
+
+		// Cache file content if requested
+		if shouldCache && len(filesToCache) > 0 {
+			fmt.Fprintf(writer, "Caching file content...\n")
+
+			var wg sync.WaitGroup
+			limitedConcurrentExecutor := util.NewLimitedConcurrentExecutor(concurrency)
+			var executionErr error
+			var execErrMu sync.Mutex
+
+			for _, pathToCache := range filesToCache {
+				wg.Add(1)
+				pathToCacheCopy := pathToCache // Capture for closure
+				limitedConcurrentExecutor.Execute(func() {
+					defer wg.Done()
+
+					// Get local entry (either from localFiles map or by lookup)
+					var localEntry *filer_pb.Entry
+					if deleteLocalExtra {
+						localEntry = localFiles[pathToCacheCopy]
+						if localEntry == nil {
+							fmt.Fprintf(writer, "Warning: skipping cache for %s (local entry not found)\n", pathToCacheCopy)
+							return
+						}
+					} else {
+						// Look up the local entry since we don't have it in localFiles
+						localDir, name := util.FullPath(pathToCacheCopy).DirAndName()
+						lookupErr := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+							lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+								Directory: localDir,
+								Name:      name,
+							})
+							if err == nil {
+								localEntry = lookupResp.Entry
+							}
+							return err
+						})
+						if lookupErr != nil {
+							fmt.Fprintf(writer, "Warning: failed to lookup local entry for caching %s: %v\n", pathToCacheCopy, lookupErr)
+							return
+						}
+					}
+
+					dir, _ := util.FullPath(pathToCacheCopy).DirAndName()
+
+					fmt.Fprintf(writer, "Caching %s... ", pathToCacheCopy)
+
+					if _, err := filer.CacheRemoteObjectToLocalCluster(commandEnv, util.FullPath(dir), localEntry, chunkConcurrency, downloadConcurrency); err != nil {
+						fmt.Fprintf(writer, "failed: %v\n", err)
+						execErrMu.Lock()
+						if executionErr == nil {
+							executionErr = err
+						}
+						execErrMu.Unlock()
+						return
+					}
+					fmt.Fprintf(writer, "done\n")
+				})
+			}
+
+			wg.Wait()
+			if executionErr != nil {
+				return executionErr
+			}
+		}
+
+		return nil
+	})
+}
+
+func recursivelyTraverseDirectory(filerClient filer_pb.FilerClient, dirPath util.FullPath, visitEntry func(dir util.FullPath, entry *filer_pb.Entry) bool) (err error) {
+
+	err = filer_pb.ReadDirAllEntries(context.Background(), filerClient, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		if entry.IsDirectory {
+			if !visitEntry(dirPath, entry) {
+				return nil
+			}
+			subDir := dirPath.Child(entry.Name)
+			if err := recursivelyTraverseDirectory(filerClient, subDir, visitEntry); err != nil {
+				return err
+			}
+		} else {
+			if !visitEntry(dirPath, entry) {
+				return nil
+			}
+		}
+		return nil
+	})
+	return
+}
+
+func shouldCacheToLocal(entry *filer_pb.Entry) bool {
+	if entry.IsDirectory {
+		return false
+	}
+	if entry.RemoteEntry == nil {
+		return false
+	}
+	if entry.RemoteEntry.LastLocalSyncTsNs == 0 && entry.RemoteEntry.RemoteSize > 0 {
+		return true
+	}
+	return false
+}
+
+func mayHaveCachedToLocal(entry *filer_pb.Entry) bool {
+	if entry.IsDirectory {
+		return false
+	}
+	if entry.RemoteEntry == nil {
+		return false // should not uncache an entry that is not in remote
+	}
+	if entry.RemoteEntry.LastLocalSyncTsNs > 0 {
+		return true
+	}
+	return false
+}

weed/shell/command_remote_configure.go

@@ -0,0 +1,327 @@
+package shell
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteConfigure{})
+}
+
+type commandRemoteConfigure struct {
+}
+
+func (c *commandRemoteConfigure) Name() string {
+	return "remote.configure"
+}
+
+func (c *commandRemoteConfigure) Help() string {
+	return `remote storage configuration
+
+	# see the current configurations
+	remote.configure
+
+	# set or update a configuration
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2
+	remote.configure -name=cloud1_deep -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2 -s3.storage_class=DEEP_ARCHIVE
+	remote.configure -name=cloud2 -type=gcs -gcs.appCredentialsFile=~/service-account-file.json -gcs.projectId=yyy
+	remote.configure -name=cloud3 -type=azure -azure.account_name=xxx -azure.account_key=yyy
+	remote.configure -name=cloud3 -type=azure -azure.account_name=xxx -azure.client_id=zzz
+	remote.configure -name=cloud4 -type=aliyun -aliyun.access_key=xxx -aliyun.secret_key=yyy -aliyun.endpoint=oss-cn-shenzhen.aliyuncs.com -aliyun.region=cn-sehnzhen
+	remote.configure -name=cloud5 -type=tencent -tencent.secret_id=xxx -tencent.secret_key=yyy -tencent.endpoint=cos.ap-guangzhou.myqcloud.com
+	remote.configure -name=cloud6 -type=wasabi -wasabi.access_key=xxx -wasabi.secret_key=yyy -wasabi.endpoint=s3.us-west-1.wasabisys.com -wasabi.region=us-west-1
+	remote.configure -name=cloud7 -type=storj -storj.access_key=xxx -storj.secret_key=yyy -storj.endpoint=https://gateway.us1.storjshare.io
+	remote.configure -name=cloud8 -type=filebase -filebase.access_key=xxx -filebase.secret_key=yyy -filebase.endpoint=https://s3.filebase.com
+
+	# tune transfer concurrency (applies to s3-compatible and azure storage)
+	remote.configure -name=cloud1 -upload_concurrency=4 -download_concurrency=8
+
+	# delete one configuration
+	remote.configure -delete -name=cloud1
+
+`
+}
+
+func (c *commandRemoteConfigure) HasTag(CommandTag) bool {
+	return false
+}
+
+var (
+	isAlpha = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`).MatchString
+)
+
+func (c *commandRemoteConfigure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	conf := &remote_pb.RemoteConf{}
+
+	fs, isDelete, uploadConcurrency, downloadConcurrency := c.configureFlagSet(conf, false)
+	if err = fs.Parse(args); err != nil {
+		return nil
+	}
+
+	if conf.Name == "" {
+		return c.listExistingRemoteStorages(commandEnv, writer)
+	}
+
+	if !isAlpha(conf.Name) {
+		return fmt.Errorf("only letters and numbers allowed in name: %v", conf.Name)
+	}
+
+	if *isDelete {
+		return c.deleteRemoteStorage(commandEnv, writer, conf.Name)
+	}
+
+	// Merge with an existing configuration so a partial update preserves
+	// previously stored credentials and endpoints. Only treat a confirmed
+	// missing entry as a new configuration; propagate all other load errors
+	// so a transient filer failure does not overwrite stored credentials.
+	typeExplicit := false
+	fs.Visit(func(f *flag.Flag) {
+		if f.Name == "type" {
+			typeExplicit = true
+		}
+	})
+	requestedType := conf.Type
+	existing, loadErr := c.loadRemoteStorageConf(commandEnv, conf.Name)
+	if loadErr != nil && !errors.Is(loadErr, filer_pb.ErrNotFound) {
+		return fmt.Errorf("load existing configuration %s: %v", conf.Name, loadErr)
+	}
+	if existing != nil {
+		conf = existing
+		// On an explicit type transition, reset backend-specific fields to
+		// the destination defaults before re-parsing so explicit flags
+		// override. An omitted -type keeps the stored backend.
+		if typeExplicit && requestedType != existing.Type {
+			conf.Type = requestedType
+			c.applyTypeDefaults(conf)
+		}
+		fs, isDelete, uploadConcurrency, downloadConcurrency = c.configureFlagSet(conf, true)
+		if err = fs.Parse(args); err != nil {
+			return nil
+		}
+	}
+
+	if err = applyConcurrency(conf, *uploadConcurrency, *downloadConcurrency); err != nil {
+		return err
+	}
+
+	if conf.Type != "s3" {
+		conf.S3Region = ""
+		conf.S3ForcePathStyle = false
+	}
+
+	return c.saveRemoteStorage(commandEnv, writer, conf)
+
+}
+
+// configureFlagSet builds the remote.configure flag set bound to conf. When
+// existing is true, flag defaults are taken from conf so omitted flags preserve
+// prior values instead of being reset to the hard-coded new-config defaults.
+func (c *commandRemoteConfigure) configureFlagSet(conf *remote_pb.RemoteConf, existing bool) (fs *flag.FlagSet, isDelete *bool, uploadConcurrency *int, downloadConcurrency *int) {
+	fs = flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	isDelete = fs.Bool("delete", false, "delete one remote storage by its name")
+
+	fs.StringVar(&conf.Name, "name", conf.Name, "a short name to identify the remote storage")
+	typeDefault := "s3"
+	if existing {
+		typeDefault = conf.Type
+	}
+	fs.StringVar(&conf.Type, "type", typeDefault, fmt.Sprintf("[%s] storage type", remote_storage.GetAllRemoteStorageNames()))
+
+	uploadConcurrency = fs.Int("upload_concurrency", int(conf.UploadConcurrency), "concurrent part uploads per file (0 = client default: s3 1, azure 16)")
+	downloadConcurrency = fs.Int("download_concurrency", int(conf.DownloadConcurrency), "concurrent part downloads per read (0 = client default: s3 5, azure 16)")
+
+	fs.StringVar(&conf.S3AccessKey, "s3.access_key", conf.S3AccessKey, "s3 access key")
+	fs.StringVar(&conf.S3SecretKey, "s3.secret_key", conf.S3SecretKey, "s3 secret key")
+	s3RegionDefault := "us-east-2"
+	if existing {
+		s3RegionDefault = conf.S3Region
+	}
+	fs.StringVar(&conf.S3Region, "s3.region", s3RegionDefault, "s3 region")
+	fs.StringVar(&conf.S3Endpoint, "s3.endpoint", conf.S3Endpoint, "endpoint for s3-compatible local object store")
+	fs.StringVar(&conf.S3StorageClass, "s3.storage_class", conf.S3StorageClass, "s3 storage class")
+	s3ForcePathStyleDefault := true
+	if existing {
+		s3ForcePathStyleDefault = conf.S3ForcePathStyle
+	}
+	fs.BoolVar(&conf.S3ForcePathStyle, "s3.force_path_style", s3ForcePathStyleDefault, "s3 force path style")
+	fs.BoolVar(&conf.S3V4Signature, "s3.v4_signature", conf.S3V4Signature, "s3 V4 signature")
+	s3SupportTaggingDefault := true
+	if existing {
+		s3SupportTaggingDefault = conf.S3SupportTagging
+	}
+	fs.BoolVar(&conf.S3SupportTagging, "s3.support_tagging", s3SupportTaggingDefault, "s3 supportTagging")
+
+	fs.StringVar(&conf.GcsGoogleApplicationCredentials, "gcs.appCredentialsFile", conf.GcsGoogleApplicationCredentials, "google cloud storage credentials file, default to use env GOOGLE_APPLICATION_CREDENTIALS")
+	fs.StringVar(&conf.GcsProjectId, "gcs.projectId", conf.GcsProjectId, "google cloud storage project id, default to use env GOOGLE_CLOUD_PROJECT")
+
+	fs.StringVar(&conf.AzureAccountName, "azure.account_name", conf.AzureAccountName, "azure account name, default to use env AZURE_STORAGE_ACCOUNT")
+	fs.StringVar(&conf.AzureAccountKey, "azure.account_key", conf.AzureAccountKey, "azure account key, default to use env AZURE_STORAGE_ACCESS_KEY. Leave empty to authenticate with Entra ID")
+	fs.StringVar(&conf.AzureClientId, "azure.client_id", conf.AzureClientId, "azure user-assigned identity to authenticate, when no account key is given. Workload identity also reads env AZURE_TENANT_ID and AZURE_FEDERATED_TOKEN_FILE")
+	fs.StringVar(&conf.AzureEndpoint, "azure.endpoint", conf.AzureEndpoint, "azure blob service url, for accounts outside the public cloud, e.g. https://xxx.blob.core.usgovcloudapi.net/")
+
+	fs.StringVar(&conf.BackblazeKeyId, "b2.key_id", conf.BackblazeKeyId, "backblaze keyID")
+	fs.StringVar(&conf.BackblazeApplicationKey, "b2.application_key", conf.BackblazeApplicationKey, "backblaze applicationKey. Note that your Master Application Key will not work with the S3 Compatible API. You must create a new key that is eligible for use. For more information: https://help.backblaze.com/hc/en-us/articles/360047425453")
+	fs.StringVar(&conf.BackblazeEndpoint, "b2.endpoint", conf.BackblazeEndpoint, "backblaze endpoint")
+	b2RegionDefault := "us-west-002"
+	if existing {
+		b2RegionDefault = conf.BackblazeRegion
+	}
+	fs.StringVar(&conf.BackblazeRegion, "b2.region", b2RegionDefault, "backblaze region")
+
+	fs.StringVar(&conf.AliyunAccessKey, "aliyun.access_key", conf.AliyunAccessKey, "Aliyun access key, default to use env ALICLOUD_ACCESS_KEY_ID")
+	fs.StringVar(&conf.AliyunSecretKey, "aliyun.secret_key", conf.AliyunSecretKey, "Aliyun secret key, default to use env ALICLOUD_ACCESS_KEY_SECRET")
+	fs.StringVar(&conf.AliyunEndpoint, "aliyun.endpoint", conf.AliyunEndpoint, "Aliyun endpoint")
+	fs.StringVar(&conf.AliyunRegion, "aliyun.region", conf.AliyunRegion, "Aliyun region")
+
+	fs.StringVar(&conf.TencentSecretId, "tencent.secret_id", conf.TencentSecretId, "Tencent Secret Id, default to use env COS_SECRETID")
+	fs.StringVar(&conf.TencentSecretKey, "tencent.secret_key", conf.TencentSecretKey, "Tencent secret key, default to use env COS_SECRETKEY")
+	fs.StringVar(&conf.TencentEndpoint, "tencent.endpoint", conf.TencentEndpoint, "Tencent endpoint")
+
+	fs.StringVar(&conf.BaiduAccessKey, "baidu.access_key", conf.BaiduAccessKey, "Baidu access key, default to use env BDCLOUD_ACCESS_KEY")
+	fs.StringVar(&conf.BaiduSecretKey, "baidu.secret_key", conf.BaiduSecretKey, "Baidu secret key, default to use env BDCLOUD_SECRET_KEY")
+	fs.StringVar(&conf.BaiduEndpoint, "baidu.endpoint", conf.BaiduEndpoint, "Baidu endpoint")
+	fs.StringVar(&conf.BaiduRegion, "baidu.region", conf.BaiduRegion, "Baidu region")
+
+	fs.StringVar(&conf.WasabiAccessKey, "wasabi.access_key", conf.WasabiAccessKey, "Wasabi access key")
+	fs.StringVar(&conf.WasabiSecretKey, "wasabi.secret_key", conf.WasabiSecretKey, "Wasabi secret key")
+	fs.StringVar(&conf.WasabiEndpoint, "wasabi.endpoint", conf.WasabiEndpoint, "Wasabi endpoint, see https://wasabi.com/wp-content/themes/wasabi/docs/API_Guide/index.html#t=topics%2Fapidiff-intro.htm")
+	fs.StringVar(&conf.WasabiRegion, "wasabi.region", conf.WasabiRegion, "Wasabi region")
+
+	fs.StringVar(&conf.FilebaseAccessKey, "filebase.access_key", conf.FilebaseAccessKey, "Filebase access key")
+	fs.StringVar(&conf.FilebaseSecretKey, "filebase.secret_key", conf.FilebaseSecretKey, "Filebase secret key")
+	fs.StringVar(&conf.FilebaseEndpoint, "filebase.endpoint", conf.FilebaseEndpoint, "Filebase endpoint, https://s3.filebase.com")
+
+	fs.StringVar(&conf.StorjAccessKey, "storj.access_key", conf.StorjAccessKey, "Storj access key")
+	fs.StringVar(&conf.StorjSecretKey, "storj.secret_key", conf.StorjSecretKey, "Storj secret key")
+	fs.StringVar(&conf.StorjEndpoint, "storj.endpoint", conf.StorjEndpoint, "Storj endpoint")
+
+	return
+}
+
+// maxRemoteConcurrency caps configured remote transfer concurrency so one
+// configuration cannot request an unbounded number of network workers.
+const maxRemoteConcurrency = 1024
+
+// applyTypeDefaults resets backend-specific fields to their new-config defaults
+// for conf.Type, used when the storage type changes on update.
+func (c *commandRemoteConfigure) applyTypeDefaults(conf *remote_pb.RemoteConf) {
+	conf.S3Region = ""
+	conf.S3ForcePathStyle = false
+	conf.S3SupportTagging = false
+	conf.S3V4Signature = false
+	conf.BackblazeRegion = ""
+	if conf.Type == "s3" {
+		conf.S3Region = "us-east-2"
+		conf.S3ForcePathStyle = true
+		conf.S3SupportTagging = true
+	} else if conf.Type == "b2" {
+		conf.BackblazeRegion = "us-west-002"
+	}
+}
+
+func applyConcurrency(conf *remote_pb.RemoteConf, upload, download int) error {
+	if upload < 0 || upload > maxRemoteConcurrency {
+		return fmt.Errorf("upload_concurrency must be between 0 and %d", maxRemoteConcurrency)
+	}
+	if download < 0 || download > maxRemoteConcurrency {
+		return fmt.Errorf("download_concurrency must be between 0 and %d", maxRemoteConcurrency)
+	}
+	conf.UploadConcurrency = uint32(upload)
+	conf.DownloadConcurrency = uint32(download)
+	return nil
+}
+
+func (c *commandRemoteConfigure) loadRemoteStorageConf(commandEnv *CommandEnv, name string) (*remote_pb.RemoteConf, error) {
+	var conf *remote_pb.RemoteConf
+	err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		content, readErr := filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, name+filer.REMOTE_STORAGE_CONF_SUFFIX)
+		if readErr != nil {
+			return readErr
+		}
+		conf = &remote_pb.RemoteConf{}
+		if unmarshalErr := proto.Unmarshal(content, conf); unmarshalErr != nil {
+			return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, name, unmarshalErr)
+		}
+		return nil
+	})
+	return conf, err
+}
+
+func (c *commandRemoteConfigure) listExistingRemoteStorages(commandEnv *CommandEnv, writer io.Writer) error {
+
+	return filer_pb.ReadDirAllEntries(context.Background(), commandEnv, util.FullPath(filer.DirectoryEtcRemote), "", func(entry *filer_pb.Entry, isLast bool) error {
+		if len(entry.Content) == 0 {
+			fmt.Fprintf(writer, "skipping %s\n", entry.Name)
+			return nil
+		}
+		if !strings.HasSuffix(entry.Name, filer.REMOTE_STORAGE_CONF_SUFFIX) {
+			return nil
+		}
+		conf := &remote_pb.RemoteConf{}
+
+		if err := proto.Unmarshal(entry.Content, conf); err != nil {
+			return fmt.Errorf("unmarshal %s/%s: %v", filer.DirectoryEtcRemote, entry.Name, err)
+		}
+
+		// change secret key to stars
+		conf.S3SecretKey = strings.Repeat("*", len(conf.S3SecretKey))
+		conf.AliyunSecretKey = strings.Repeat("*", len(conf.AliyunSecretKey))
+		conf.BaiduAccessKey = strings.Repeat("*", len(conf.BaiduAccessKey))
+		conf.FilebaseSecretKey = strings.Repeat("*", len(conf.FilebaseSecretKey))
+		conf.StorjSecretKey = strings.Repeat("*", len(conf.StorjSecretKey))
+		conf.TencentSecretKey = strings.Repeat("*", len(conf.TencentSecretKey))
+		conf.WasabiSecretKey = strings.Repeat("*", len(conf.WasabiSecretKey))
+
+		return filer.ProtoToText(writer, conf)
+
+	})
+
+}
+
+func (c *commandRemoteConfigure) deleteRemoteStorage(commandEnv *CommandEnv, writer io.Writer, storageName string) error {
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		name := storageName + filer.REMOTE_STORAGE_CONF_SUFFIX
+		err := filer_pb.DoRemove(context.Background(), client, filer.DirectoryEtcRemote, name, true, true, false, false, nil)
+
+		if err == nil {
+			fmt.Fprintf(writer, "removed: %s\n", storageName)
+		}
+
+		return err
+
+	})
+
+}
+
+func (c *commandRemoteConfigure) saveRemoteStorage(commandEnv *CommandEnv, writer io.Writer, conf *remote_pb.RemoteConf) error {
+
+	data, err := proto.Marshal(conf)
+	if err != nil {
+		return err
+	}
+
+	if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer.SaveInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, conf.Name+filer.REMOTE_STORAGE_CONF_SUFFIX, data)
+	}); err != nil && err != filer_pb.ErrNotFound {
+		return err
+	}
+
+	return nil
+
+}

weed/shell/command_remote_copy_local.go

@@ -0,0 +1,376 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteCopyLocal{})
+}
+
+type commandRemoteCopyLocal struct {
+}
+
+func (c *commandRemoteCopyLocal) Name() string {
+	return "remote.copy.local"
+}
+
+func (c *commandRemoteCopyLocal) Help() string {
+	return `copy local files to remote storage
+
+	# assume a remote storage is configured to name "cloud1"
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+	# mount and pull one bucket
+	remote.mount -dir=/xxx -remote=cloud1/bucket
+
+	# copy local files to remote storage
+	remote.copy.local -dir=/xxx                                    # copy all local-only files
+	remote.copy.local -dir=/xxx -concurrent=16                     # with custom concurrency
+	remote.copy.local -dir=/xxx -include=*.pdf                     # only copy PDF files
+	remote.copy.local -dir=/xxx -exclude=*.tmp                     # exclude temporary files
+	remote.copy.local -dir=/xxx -dryRun=true                       # show what would be done without making changes
+	remote.copy.local -dir=/xxx -forceUpdate=true                  # force update even if remote exists
+	remote.copy.local -dir=/xxx -delete                            # also delete remote files that do not exist locally
+
+	This command will:
+	1. Find local files that don't exist on remote storage
+	2. Copy these files to remote storage
+	3. Update local metadata with remote information
+	4. With -delete, remove remote files that do not exist locally (similar to rsync --delete)
+
+	This is useful when:
+	- You deleted filer logs and need to copy existing files
+	- You have local files that were never copied to remote
+	- You want to ensure all local files are backed up to remote
+	- You want scheduled one-shot backups that also propagate local deletions (with -delete)
+
+	Notes on -delete:
+	- only remote files under -dir are considered; paths outside it are never deleted
+	- -include/-exclude patterns also limit which remote files are deleted
+	- size/age filters only apply to copying, not to deletion
+	- use -dryRun=true first to review what would be deleted
+
+ `
+}
+
+func (c *commandRemoteCopyLocal) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteCopyLocal) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteCopyLocalCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteCopyLocalCommand.String("dir", "", "a directory in filer")
+	concurrency := remoteCopyLocalCommand.Int("concurrent", 16, "concurrent file operations")
+	dryRun := remoteCopyLocalCommand.Bool("dryRun", false, "show what would be done without making changes")
+	forceUpdate := remoteCopyLocalCommand.Bool("forceUpdate", false, "force update even if remote exists")
+	deleteExtraneous := remoteCopyLocalCommand.Bool("delete", false, "delete extraneous files from remote storage (files that do not exist locally), similar to rsync --delete")
+	fileFilter := newFileFilter(remoteCopyLocalCommand)
+
+	if err = remoteCopyLocalCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *dir == "" {
+		return fmt.Errorf("need to specify -dir option")
+	}
+
+	mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
+	if detectErr != nil {
+		jsonPrintln(writer, mappings)
+		return detectErr
+	}
+
+	// perform local to remote copy
+	return c.doLocalToRemoteCopy(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *concurrency, *dryRun, *forceUpdate, *deleteExtraneous, fileFilter)
+}
+
+func (c *commandRemoteCopyLocal) doLocalToRemoteCopy(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCopy util.FullPath, remoteConf *remote_pb.RemoteConf, concurrency int, dryRun bool, forceUpdate bool, deleteExtraneous bool, fileFilter *FileFilter) error {
+
+	// Get remote storage client
+	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
+	if err != nil {
+		return fmt.Errorf("failed to get remote storage: %w", err)
+	}
+
+	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToCopy)
+
+	// Step 1: Collect all local files that are part of the remote mount
+	localFiles := make(map[string]*filer_pb.Entry)
+	err = recursivelyTraverseDirectory(commandEnv, dirToCopy, func(dir util.FullPath, entry *filer_pb.Entry) bool {
+		// Only consider files that are part of remote mount
+		if isInMountedDirectory(dir, localMountedDir) {
+			fullPath := string(dir.Child(entry.Name))
+			localFiles[fullPath] = entry
+		}
+		return true
+	})
+	if err != nil {
+		return fmt.Errorf("failed to traverse local directory: %w", err)
+	}
+
+	fmt.Fprintf(writer, "Found %d files/directories in local storage\n", len(localFiles))
+
+	// Step 2: Check which files exist on remote storage
+	remoteFiles := make(map[string]bool) // full path -> isDirectory
+	err = remoteStorage.Traverse(remote, func(remoteDir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
+		localDir := filer.MapRemoteStorageLocationPathToFullPath(localMountedDir, remoteMountedLocation, remoteDir)
+		fullPath := string(localDir.Child(name))
+		remoteFiles[fullPath] = isDirectory
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to traverse remote storage: %w", err)
+	}
+
+	fmt.Fprintf(writer, "Found %d files/directories in remote storage\n", len(remoteFiles))
+
+	// Step 3: Determine files to copy and files to delete
+	plan := planLocalToRemoteSync(localFiles, remoteFiles, dirToCopy, forceUpdate, deleteExtraneous, fileFilter)
+
+	fmt.Fprintf(writer, "Files to copy: %d\n", len(plan.filesToCopy))
+	if deleteExtraneous {
+		fmt.Fprintf(writer, "Files to delete from remote: %d\n", len(plan.filesToDelete))
+	}
+
+	if dryRun {
+		fmt.Fprintf(writer, "DRY RUN - showing what would be done:\n")
+		for _, path := range plan.filesToCopy {
+			fmt.Fprintf(writer, "COPY: %s\n", path)
+		}
+		for _, path := range plan.filesToDelete {
+			fmt.Fprintf(writer, "DELETE: %s\n", path)
+		}
+		return nil
+	}
+
+	if len(plan.filesToCopy) == 0 && len(plan.filesToDelete) == 0 {
+		fmt.Fprintf(writer, "No files to copy\n")
+		return nil
+	}
+
+	var wg sync.WaitGroup
+	limitedConcurrentExecutor := util.NewLimitedConcurrentExecutor(concurrency)
+	var firstErr error
+	var errOnce sync.Once
+	var successCount atomic.Int64
+	var outputMu sync.Mutex
+
+	// Step 4: Copy files to remote storage
+	for _, pathToCopy := range plan.filesToCopy {
+		wg.Add(1)
+		localPath := pathToCopy // Capture for closure
+		limitedConcurrentExecutor.Execute(func() {
+			defer wg.Done()
+
+			localEntry := localFiles[localPath]
+			if localEntry == nil {
+				outputMu.Lock()
+				fmt.Fprintf(writer, "Warning: skipping copy for %s (local entry not found)\n", localPath)
+				outputMu.Unlock()
+				return
+			}
+
+			outputMu.Lock()
+			fmt.Fprintf(writer, "Copying %s... ", localPath)
+			outputMu.Unlock()
+
+			dir, _ := util.FullPath(localPath).DirAndName()
+			remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, util.FullPath(localPath))
+
+			// Copy the file to remote storage
+			err := syncFileToRemote(commandEnv, remoteStorage, remoteConf, remoteLocation, util.FullPath(dir), localEntry)
+			if err != nil {
+				outputMu.Lock()
+				fmt.Fprintf(writer, "failed: %v\n", err)
+				outputMu.Unlock()
+				errOnce.Do(func() {
+					firstErr = err
+				})
+				return
+			}
+
+			successCount.Add(1)
+			outputMu.Lock()
+			fmt.Fprintf(writer, "done\n")
+			outputMu.Unlock()
+		})
+	}
+
+	wg.Wait()
+	if firstErr != nil {
+		// skip deletion when any copy failed, to stay on the safe side
+		return firstErr
+	}
+
+	if len(plan.filesToCopy) > 0 {
+		fmt.Fprintf(writer, "Successfully copied %d files to remote storage\n", successCount.Load())
+	}
+
+	// Step 5: Delete extraneous files from remote storage
+	if len(plan.filesToDelete) == 0 {
+		return nil
+	}
+
+	var deleteErr error
+	var deleteErrOnce sync.Once
+	var deletedCount atomic.Int64
+
+	for _, pathToDelete := range plan.filesToDelete {
+		wg.Add(1)
+		localPath := pathToDelete // Capture for closure
+		limitedConcurrentExecutor.Execute(func() {
+			defer wg.Done()
+
+			remoteLocation := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, util.FullPath(localPath))
+			if err := remoteStorage.DeleteFile(remoteLocation); err != nil {
+				outputMu.Lock()
+				fmt.Fprintf(writer, "Deleting %s... failed: %v\n", localPath, err)
+				outputMu.Unlock()
+				deleteErrOnce.Do(func() {
+					deleteErr = err
+				})
+				return
+			}
+
+			deletedCount.Add(1)
+			outputMu.Lock()
+			fmt.Fprintf(writer, "Deleting %s... done\n", localPath)
+			outputMu.Unlock()
+		})
+	}
+	wg.Wait()
+
+	if deleteErr != nil {
+		return deleteErr
+	}
+
+	fmt.Fprintf(writer, "Successfully deleted %d files from remote storage\n", deletedCount.Load())
+	return nil
+}
+
+type localToRemoteSyncPlan struct {
+	filesToCopy   []string
+	filesToDelete []string
+}
+
+func planLocalToRemoteSync(localFiles map[string]*filer_pb.Entry, remoteFiles map[string]bool, dirToCopy util.FullPath, forceUpdate bool, deleteExtraneous bool, fileFilter *FileFilter) *localToRemoteSyncPlan {
+	plan := &localToRemoteSyncPlan{}
+
+	for localPath, localEntry := range localFiles {
+		// Skip directories
+		if localEntry.IsDirectory {
+			continue
+		}
+
+		// Apply file filter
+		if !fileFilter.matches(localEntry) {
+			continue
+		}
+
+		// Copy if the file doesn't exist on remote, or if force update is requested
+		if _, foundOnRemote := remoteFiles[localPath]; !foundOnRemote || forceUpdate {
+			plan.filesToCopy = append(plan.filesToCopy, localPath)
+		}
+	}
+	sort.Strings(plan.filesToCopy)
+
+	if !deleteExtraneous {
+		return plan
+	}
+
+	for remotePath, isDirectory := range remoteFiles {
+		// Traverse lists by key prefix (no delimiter), so it can return entries
+		// outside dirToCopy that merely share its name prefix (e.g. a sibling
+		// "foobar" when dirToCopy maps to prefix "foo"). Never delete anything
+		// outside the requested subtree.
+		if !util.FullPath(remotePath).IsUnder(dirToCopy) {
+			continue
+		}
+		if _, foundLocally := localFiles[remotePath]; foundLocally {
+			continue
+		}
+		// object stores expose no real directories; only files are deleted
+		if isDirectory {
+			continue
+		}
+		// name filters also protect remote files from deletion;
+		// size/age filters need local attributes, so they only apply to copying
+		if !fileFilter.matchesName(util.FullPath(remotePath).Name()) {
+			continue
+		}
+		plan.filesToDelete = append(plan.filesToDelete, remotePath)
+	}
+	sort.Strings(plan.filesToDelete)
+
+	return plan
+}
+
+func syncFileToRemote(commandEnv *CommandEnv, remoteStorage remote_storage.RemoteStorageClient, remoteConf *remote_pb.RemoteConf, remoteLocation *remote_pb.RemoteStorageLocation, dir util.FullPath, localEntry *filer_pb.Entry) error {
+
+	// Upload to remote storage using the same approach as filer_remote_sync
+	var remoteEntry *filer_pb.RemoteEntry
+	var err error
+
+	err = util.Retry("writeFile", func() error {
+		// Create a reader for the file content
+		reader := filer.NewFileReader(commandEnv, localEntry)
+
+		remoteEntry, err = remoteStorage.WriteFile(remoteLocation, localEntry, reader)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to upload to remote storage: %w", err)
+	}
+
+	// Update local entry with remote information
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		ctx := context.Background()
+
+		// Update local entry with remote information
+		remoteEntry.LastLocalSyncTsNs = time.Now().UnixNano()
+		localEntry.RemoteEntry = remoteEntry
+
+		// Update the entry
+		_, updateErr := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
+			Directory: string(dir),
+			Entry:     localEntry,
+		})
+		if updateErr != nil {
+			return fmt.Errorf("failed to update local entry: %w", updateErr)
+		}
+
+		return nil
+	})
+}
+
+func isInMountedDirectory(dir util.FullPath, mountedDir util.FullPath) bool {
+	if string(dir) == string(mountedDir) {
+		return true
+	}
+	// Ensure mountedDir ends with separator to avoid matching sibling directories
+	// e.g., "/mnt/remote2" should not match "/mnt/remote"
+	mountedDirStr := string(mountedDir)
+	if !strings.HasSuffix(mountedDirStr, "/") {
+		mountedDirStr += "/"
+	}
+	return strings.HasPrefix(string(dir)+"/", mountedDirStr)
+}

weed/shell/command_remote_copy_local_test.go

@@ -0,0 +1,267 @@
+package shell
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func newTestFileFilter(include, exclude string) *FileFilter {
+	defaultInt64 := int64(-1)
+	return &FileFilter{
+		include:     &include,
+		exclude:     &exclude,
+		minSize:     &defaultInt64,
+		maxSize:     &defaultInt64,
+		minAge:      &defaultInt64,
+		maxAge:      &defaultInt64,
+		minCacheAge: &defaultInt64,
+		now:         time.Now().Unix(),
+	}
+}
+
+func testFileEntry(name string) *filer_pb.Entry {
+	return &filer_pb.Entry{
+		Name:       name,
+		Attributes: &filer_pb.FuseAttributes{},
+	}
+}
+
+func testDirEntry(name string) *filer_pb.Entry {
+	return &filer_pb.Entry{
+		Name:        name,
+		IsDirectory: true,
+		Attributes:  &filer_pb.FuseAttributes{},
+	}
+}
+
+func TestPlanLocalToRemoteSync(t *testing.T) {
+	tests := []struct {
+		name              string
+		dirToCopy         string
+		localFiles        map[string]*filer_pb.Entry
+		remoteFiles       map[string]bool // path -> isDirectory
+		forceUpdate       bool
+		deleteExtraneous  bool
+		fileFilter        *FileFilter
+		wantFilesToCopy   []string
+		wantFilesToDelete []string
+	}{
+		{
+			name:      "copy local-only files, no delete flag",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+				"/mnt/b.txt": testFileEntry("b.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/b.txt":      false,
+				"/mnt/orphan.txt": false,
+			},
+			fileFilter:      newTestFileFilter("", ""),
+			wantFilesToCopy: []string{"/mnt/a.txt"},
+		},
+		{
+			name:      "delete flag removes remote-only files",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt":       false,
+				"/mnt/orphan.txt":  false,
+				"/mnt/orphan2.txt": false,
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("", ""),
+			wantFilesToDelete: []string{"/mnt/orphan.txt", "/mnt/orphan2.txt"},
+		},
+		{
+			name:       "delete with empty local storage removes all remote files",
+			dirToCopy:  "/mnt",
+			localFiles: map[string]*filer_pb.Entry{},
+			remoteFiles: map[string]bool{
+				"/mnt/orphan.txt": false,
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("", ""),
+			wantFilesToDelete: []string{"/mnt/orphan.txt"},
+		},
+		{
+			name:      "delete respects include pattern",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.pdf": testFileEntry("a.pdf"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.pdf":      false,
+				"/mnt/orphan.pdf": false,
+				"/mnt/keep.txt":   false,
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("*.pdf", ""),
+			wantFilesToDelete: []string{"/mnt/orphan.pdf"},
+		},
+		{
+			name:      "delete respects exclude pattern",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt":      false,
+				"/mnt/orphan.txt": false,
+				"/mnt/keep.bak":   false,
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("", "*.bak"),
+			wantFilesToDelete: []string{"/mnt/orphan.txt"},
+		},
+		{
+			name:      "delete ignores size/age filters",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt":      false,
+				"/mnt/orphan.txt": false,
+			},
+			deleteExtraneous: true,
+			fileFilter: func() *FileFilter {
+				ff := newTestFileFilter("", "")
+				minSize := int64(1000000) // would exclude orphan.txt from copying if it applied to deletion
+				ff.minSize = &minSize
+				return ff
+			}(),
+			wantFilesToDelete: []string{"/mnt/orphan.txt"},
+		},
+		{
+			name:      "delete removes orphaned files, directory entries left alone",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt":               false,
+				"/mnt/old":                 true,
+				"/mnt/old/sub":             true,
+				"/mnt/old/sub/orphan.txt":  false,
+				"/mnt/old/sub2":            true,
+				"/mnt/old/sub2/orphan.txt": false,
+			},
+			deleteExtraneous: true,
+			fileFilter:       newTestFileFilter("", ""),
+			wantFilesToDelete: []string{
+				"/mnt/old/sub/orphan.txt",
+				"/mnt/old/sub2/orphan.txt",
+			},
+		},
+		{
+			name:      "exclude pattern protects remote files under nested directory",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt":          false,
+				"/mnt/old":            true,
+				"/mnt/old/orphan.txt": false,
+				"/mnt/old/keep.bak":   false,
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("", "*.bak"),
+			wantFilesToDelete: []string{"/mnt/old/orphan.txt"},
+		},
+		{
+			name:      "remote files outside -dir are never deleted",
+			dirToCopy: "/mnt/foo",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/foo/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/foo/a.txt":      false,
+				"/mnt/foo/orphan.txt": false,
+				"/mnt/foobar/b.txt":   false, // sibling sharing the "foo" key prefix, must be left alone
+			},
+			deleteExtraneous:  true,
+			fileFilter:        newTestFileFilter("", ""),
+			wantFilesToDelete: []string{"/mnt/foo/orphan.txt"},
+		},
+		{
+			name:      "directory present locally is not deleted",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/dir": testDirEntry("dir"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/dir": true,
+			},
+			deleteExtraneous: true,
+			fileFilter:       newTestFileFilter("", ""),
+		},
+		{
+			name:      "forceUpdate copies files that exist on remote",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles: map[string]bool{
+				"/mnt/a.txt": false,
+			},
+			forceUpdate:     true,
+			fileFilter:      newTestFileFilter("", ""),
+			wantFilesToCopy: []string{"/mnt/a.txt"},
+		},
+		{
+			name:      "local directories are not copied",
+			dirToCopy: "/mnt",
+			localFiles: map[string]*filer_pb.Entry{
+				"/mnt/dir":   testDirEntry("dir"),
+				"/mnt/a.txt": testFileEntry("a.txt"),
+			},
+			remoteFiles:     map[string]bool{},
+			fileFilter:      newTestFileFilter("", ""),
+			wantFilesToCopy: []string{"/mnt/a.txt"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			plan := planLocalToRemoteSync(tt.localFiles, tt.remoteFiles, util.FullPath(tt.dirToCopy), tt.forceUpdate, tt.deleteExtraneous, tt.fileFilter)
+			if !reflect.DeepEqual(plan.filesToCopy, tt.wantFilesToCopy) {
+				t.Errorf("filesToCopy = %v, want %v", plan.filesToCopy, tt.wantFilesToCopy)
+			}
+			if !reflect.DeepEqual(plan.filesToDelete, tt.wantFilesToDelete) {
+				t.Errorf("filesToDelete = %v, want %v", plan.filesToDelete, tt.wantFilesToDelete)
+			}
+		})
+	}
+}
+
+func TestFileFilter_matchesName(t *testing.T) {
+	tests := []struct {
+		name     string
+		include  string
+		exclude  string
+		fileName string
+		want     bool
+	}{
+		{"no filters", "", "", "a.txt", true},
+		{"include match", "*.pdf", "", "a.pdf", true},
+		{"include mismatch", "*.pdf", "", "a.txt", false},
+		{"exclude match", "", "*.tmp", "a.tmp", false},
+		{"exclude mismatch", "", "*.tmp", "a.txt", true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ff := newTestFileFilter(tt.include, tt.exclude)
+			if got := ff.matchesName(tt.fileName); got != tt.want {
+				t.Errorf("matchesName(%q) = %v, want %v", tt.fileName, got, tt.want)
+			}
+		})
+	}
+}

weed/shell/command_remote_meta_sync.go

@@ -0,0 +1,399 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteMetaSync{})
+}
+
+type commandRemoteMetaSync struct {
+}
+
+func (c *commandRemoteMetaSync) Name() string {
+	return "remote.meta.sync"
+}
+
+func (c *commandRemoteMetaSync) Help() string {
+	return `synchronize the local file meta data with the remote file metadata
+
+	# assume a remote storage is configured to name "cloud1"
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+	# mount and pull one bucket
+	remote.mount -dir=/xxx -remote=cloud1/bucket
+
+	After mount, if the remote file can be changed, 
+	run this command to synchronize the metadata of the mounted folder or any sub folder
+
+		remote.meta.sync -dir=/xxx
+		remote.meta.sync -dir=/xxx/some/subdir
+
+	Local metadata for files and directories removed from the remote is also
+	removed by default; pass -delete=false to keep it.
+
+	S3 listings do not report Content-Encoding; pass -statFiles to stat each
+	new or changed file so that metadata is synchronized too.
+
+	This is designed to run regularly. So you can add it to some cronjob.
+
+	If there are no other operations changing remote files, this operation is not needed.
+
+`
+}
+
+func (c *commandRemoteMetaSync) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteMetaSyncCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteMetaSyncCommand.String("dir", "", "a directory in filer")
+	deleteStale := remoteMetaSyncCommand.Bool("delete", true, "remove local metadata of files and directories deleted from remote")
+	statFiles := remoteMetaSyncCommand.Bool("statFiles", false, "stat each new or changed file when the listing does not report all metadata (S3 listings lack Content-Encoding); costs one extra remote request per file")
+
+	if err = remoteMetaSyncCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
+	if detectErr != nil {
+		jsonPrintln(writer, mappings)
+		return detectErr
+	}
+
+	// pull metadata from remote
+	if err = pullMetadata(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *deleteStale, *statFiles); err != nil {
+		return fmt.Errorf("cache meta data: %w", err)
+	}
+
+	return nil
+}
+
+func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) {
+	return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, dir)
+}
+
+/*
+This function update entry.RemoteEntry if the remote has any changes.
+
+To pull remote updates, or created for the first time, the criteria is:
+
+	entry == nil or (entry.RemoteEntry != nil and (entry.RemoteEntry.RemoteTag != remote.RemoteTag or entry.RemoteEntry.RemoteMTime < remote.RemoteMTime ))
+
+After the meta pull, the entry.RemoteEntry will have:
+
+	remoteEntry.LastLocalSyncTsNs == 0
+	Attributes.FileSize = uint64(remoteEntry.RemoteSize)
+	Attributes.Mtime = remoteEntry.RemoteMtime
+	remoteEntry.RemoteTag   = actual remote tag
+	chunks = nil
+
+When reading the file content or pulling the file content in "remote.cache", the criteria is:
+
+	Attributes.FileSize > 0 and len(chunks) == 0
+
+After caching the file content, the entry.RemoteEntry will be
+
+	remoteEntry.LastLocalSyncTsNs == time.Now.UnixNano()
+	Attributes.FileSize = uint64(remoteEntry.RemoteSize)
+	Attributes.Mtime = remoteEntry.RemoteMtime
+	chunks = non-empty
+
+When "weed filer.remote.sync" to upload local changes to remote, the criteria is:
+
+	Attributes.Mtime > remoteEntry.RemoteMtime
+
+Right after "weed filer.remote.sync", the entry.RemoteEntry will be
+
+	remoteEntry.LastLocalSyncTsNs = time.Now.UnixNano()
+	remoteEntry.RemoteSize  = actual remote size, which should equal to entry.Attributes.FileSize
+	remoteEntry.RemoteMtime = actual remote mtime, which should be a little greater than entry.Attributes.Mtime
+	remoteEntry.RemoteTag   = actual remote tag
+
+If entry does not exist, need to pull meta
+If entry.RemoteEntry == nil, this is a new local change and should not be overwritten
+
+	If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag {
+	  the remote version is updated, need to pull meta
+	}
+*/
+
+// remoteEntryFileMode returns the POSIX mode for an entry synced from a remote
+// whose listing carries no mode. It matches what SeaweedFS S3 assigns native
+// objects (files 0660) so a mounted bucket matches the source, deriving the
+// directory mode with the same 0111 traversal mask the filer uses for
+// auto-created parents (0660 -> 0771).
+func remoteEntryFileMode(isDirectory bool) uint32 {
+	mode := uint32(0660)
+	if isDirectory {
+		mode = uint32(os.ModeDir) | mode | 0111
+	}
+	return mode
+}
+
+// remoteChild is one entry of a single remote directory level: a file carries
+// its RemoteEntry, a directory carries none.
+type remoteChild struct {
+	isDirectory bool
+	remoteEntry *filer_pb.RemoteEntry
+}
+
+func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf, deleteStale bool, statFiles bool) error {
+
+	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
+	if err != nil {
+		return err
+	}
+
+	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToCache)
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return pullMetadataDirectory(context.Background(), client, writer, remoteStorage, dirToCache, remote, deleteStale, statFiles)
+	})
+}
+
+// pullMetadataDirectory reconciles one local directory with its remote
+// counterpart and recurses into every remote subdirectory. Listing with a
+// delimiter surfaces subdirectories, including empty ones, as their own
+// entries, so directories are materialized locally even when they hold no
+// files.
+func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, remoteStorage remote_storage.RemoteStorageClient, localDir util.FullPath, remoteLoc *remote_pb.RemoteStorageLocation, deleteStale bool, statFiles bool) error {
+
+	remoteChildren := make(map[string]*remoteChild)
+	if err := remoteStorage.ListDirectory(ctx, remoteLoc, func(dir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
+		remoteChildren[name] = &remoteChild{isDirectory: isDirectory, remoteEntry: remoteEntry}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	localChildren, err := listLocalDirectory(ctx, client, localDir)
+	if err != nil {
+		return err
+	}
+
+	for name, child := range remoteChildren {
+		localPath := localDir.Child(name)
+		existingEntry := localChildren[name]
+
+		if existingEntry != nil && existingEntry.IsDirectory != child.isDirectory {
+			if existingEntry.RemoteEntry == nil {
+				// a local change conflicts with a changed remote type; keep local
+				fmt.Fprintf(writer, "%s (skip)\n", localPath)
+				continue
+			}
+			// the remote swapped file for directory (or vice versa); drop the
+			// stale entry so it is recreated with the right type, but keep any
+			// local-only descendants of a directory
+			if existingEntry.IsDirectory {
+				if err := deleteRemoteBackedEntry(ctx, client, writer, localDir, name, existingEntry); err != nil {
+					return err
+				}
+				refreshed, err := listLocalDirectory(ctx, client, localDir)
+				if err != nil {
+					return err
+				}
+				if refreshed[name] != nil {
+					// local-only entries remain, so the directory stays and the
+					// remote file cannot take its place
+					fmt.Fprintf(writer, "%s (skip)\n", localPath)
+					continue
+				}
+			} else {
+				fmt.Fprintf(writer, "%s (delete)\n", localPath)
+				if err := deleteLocalEntry(ctx, client, localDir, name); err != nil {
+					return err
+				}
+			}
+			existingEntry = nil
+		}
+
+		// enrich a listing that does not report encodings with a per-file stat,
+		// skipping files whose stored metadata is already present and unchanged
+		if statFiles && !child.isDirectory && child.remoteEntry.RemoteContentEncoding == nil {
+			needsStat := existingEntry == nil
+			if existingEntry != nil && existingEntry.RemoteEntry != nil {
+				needsStat = existingEntry.RemoteEntry.RemoteContentEncoding == nil ||
+					existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
+					existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
+					existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize
+			}
+			if needsStat {
+				if statEntry, statErr := remoteStorage.StatFile(childRemoteLocation(remoteLoc, name)); statErr == nil && statEntry != nil {
+					child.remoteEntry = statEntry
+				}
+			}
+		}
+
+		if existingEntry == nil {
+			if err := createRemoteEntry(ctx, client, writer, localDir, name, child, remoteLoc.Name); err != nil {
+				return err
+			}
+		} else if !child.isDirectory {
+			if existingEntry.RemoteEntry == nil {
+				// a local change that should not be overwritten
+				fmt.Fprintf(writer, "%s (skip)\n", localPath)
+			} else if existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
+				existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
+				existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize ||
+				(child.remoteEntry.RemoteContentEncoding != nil &&
+					existingEntry.RemoteEntry.GetRemoteContentEncoding() != child.remoteEntry.GetRemoteContentEncoding()) ||
+				// persist the stat-derived encoding so the next run skips the stat
+				(statFiles && existingEntry.RemoteEntry.RemoteContentEncoding == nil &&
+					child.remoteEntry.RemoteContentEncoding != nil) {
+				fmt.Fprintf(writer, "%s (update)\n", localPath)
+				if err := doSaveRemoteEntry(client, string(localDir), existingEntry, child.remoteEntry); err != nil {
+					return err
+				}
+			} else {
+				fmt.Fprintf(writer, "%s (skip)\n", localPath)
+			}
+		}
+
+		if child.isDirectory {
+			if err := pullMetadataDirectory(ctx, client, writer, remoteStorage, localPath, childRemoteLocation(remoteLoc, name), deleteStale, statFiles); err != nil {
+				return err
+			}
+		}
+	}
+
+	if deleteStale {
+		for name, existingEntry := range localChildren {
+			if _, ok := remoteChildren[name]; ok {
+				continue
+			}
+			if err := deleteRemoteBackedEntry(ctx, client, writer, localDir, name, existingEntry); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// deleteRemoteBackedEntry removes a local entry whose remote source is gone. A
+// file is deleted; a directory is descended into locally so its remote-backed
+// children are cleaned, and the directory itself is removed only when it was
+// remote-backed and holds no remaining local-only entries. Entries without a
+// RemoteEntry are local changes and are never touched.
+func deleteRemoteBackedEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, localDir util.FullPath, name string, existingEntry *filer_pb.Entry) error {
+	localPath := localDir.Child(name)
+
+	if !existingEntry.IsDirectory {
+		if existingEntry.RemoteEntry == nil {
+			return nil
+		}
+		fmt.Fprintf(writer, "%s (delete)\n", localPath)
+		return deleteLocalEntry(ctx, client, localDir, name)
+	}
+
+	children, err := listLocalDirectory(ctx, client, localPath)
+	if err != nil {
+		return err
+	}
+	for childName, childEntry := range children {
+		if err := deleteRemoteBackedEntry(ctx, client, writer, localPath, childName, childEntry); err != nil {
+			return err
+		}
+	}
+
+	if existingEntry.RemoteEntry == nil {
+		return nil
+	}
+	empty, err := isLocalDirectoryEmpty(ctx, client, localPath)
+	if err != nil {
+		return err
+	}
+	if empty {
+		fmt.Fprintf(writer, "%s (delete)\n", localPath)
+		return deleteLocalEntry(ctx, client, localDir, name)
+	}
+	return nil
+}
+
+func deleteLocalEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, localDir util.FullPath, name string) error {
+	return filer_pb.DoRemove(ctx, client, string(localDir), name, true, false, false, false, nil)
+}
+
+func isLocalDirectoryEmpty(ctx context.Context, client filer_pb.SeaweedFilerClient, dir util.FullPath) (bool, error) {
+	empty := true
+	err := filer_pb.SeaweedList(ctx, client, string(dir), "", func(entry *filer_pb.Entry, isLast bool) error {
+		empty = false
+		return nil
+	}, "", false, 1)
+	return empty, err
+}
+
+func createRemoteEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, localDir util.FullPath, name string, child *remoteChild, storageName string) error {
+	attributes := &filer_pb.FuseAttributes{
+		FileMode: remoteEntryFileMode(child.isDirectory),
+		TtlSec:   0, // Remote entries should not have TTL
+	}
+	remoteEntry := child.remoteEntry
+	if child.isDirectory {
+		// remote listings carry no directory timestamp; stamp with sync time
+		now := time.Now().Unix()
+		attributes.Crtime = now
+		attributes.Mtime = now
+		// mark the directory remote-backed so it can be reconciled and removed
+		// once it disappears from the remote
+		remoteEntry = &filer_pb.RemoteEntry{StorageName: storageName}
+	} else {
+		attributes.FileSize = uint64(child.remoteEntry.RemoteSize)
+		attributes.Mtime = child.remoteEntry.RemoteMtime
+	}
+	_, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+		Directory: string(localDir),
+		Entry: &filer_pb.Entry{
+			Name:        name,
+			IsDirectory: child.isDirectory,
+			Attributes:  attributes,
+			Extended:    filer.MergeRemoteContentEncoding(remoteEntry, nil),
+			RemoteEntry: remoteEntry,
+		},
+	})
+	fmt.Fprintf(writer, "%s (create)\n", localDir.Child(name))
+	return err
+}
+
+func childRemoteLocation(remoteLoc *remote_pb.RemoteStorageLocation, name string) *remote_pb.RemoteStorageLocation {
+	return &remote_pb.RemoteStorageLocation{
+		Name:   remoteLoc.Name,
+		Bucket: remoteLoc.Bucket,
+		Path:   string(util.FullPath(remoteLoc.Path).Child(name)),
+	}
+}
+
+func listLocalDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dir util.FullPath) (map[string]*filer_pb.Entry, error) {
+	const paginationLimit = 10000
+	entries := make(map[string]*filer_pb.Entry)
+	lastFileName := ""
+	for {
+		count := 0
+		if err := filer_pb.SeaweedList(ctx, client, string(dir), "", func(entry *filer_pb.Entry, isLast bool) error {
+			entries[entry.Name] = entry
+			lastFileName = entry.Name
+			count++
+			return nil
+		}, lastFileName, false, paginationLimit); err != nil {
+			return nil, err
+		}
+		if count < paginationLimit {
+			return entries, nil
+		}
+	}
+}

weed/shell/command_remote_mount.go

@@ -0,0 +1,232 @@
+package shell
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/protobuf/proto"
+)
+
+type MetadataCacheStrategy string
+
+const (
+	MetadataCacheEager MetadataCacheStrategy = "eager"
+	MetadataCacheLazy  MetadataCacheStrategy = "lazy"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteMount{})
+}
+
+type commandRemoteMount struct {
+}
+
+func (c *commandRemoteMount) Name() string {
+	return "remote.mount"
+}
+
+func (c *commandRemoteMount) Help() string {
+	return `mount remote storage and optionally pull its metadata
+
+	# assume a remote storage is configured to name "cloud1"
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+
+	# mount and pull one bucket (full upfront metadata sync)
+	remote.mount -dir=/xxx -remote=cloud1/bucket
+	# mount without upfront sync; metadata is fetched lazily on access
+	remote.mount -dir=/xxx -remote=cloud1/bucket -metadataStrategy=lazy
+	# mount and pull one directory in the bucket
+	remote.mount -dir=/xxx -remote=cloud1/bucket/dir1
+	# mount with on-demand directory listing cached for 5 minutes
+	remote.mount -dir=/xxx -remote=cloud1/bucket -listingCacheTTL=300
+	# mount as a streaming source: reads go to the remote instead of waiting for the local cache
+	remote.mount -dir=/xxx -remote=cloud1/bucket -cacheWait=0
+
+	# after mount, start a separate process to write updates to remote storage
+	weed filer.remote.sync -filer=<filerHost>:<filerPort> -dir=/xxx
+
+`
+}
+
+func (c *commandRemoteMount) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteMountCommand.String("dir", "", "a directory in filer")
+	nonEmpty := remoteMountCommand.Bool("nonempty", false, "allows the mounting over a non-empty directory")
+	metadataStrategy := remoteMountCommand.String("metadataStrategy", string(MetadataCacheEager), "lazy: skip upfront metadata pull; eager: full metadata pull (default)")
+	remote := remoteMountCommand.String("remote", "", "a directory in remote storage, ex. <storageName>/<bucket>/path/to/dir")
+	listingCacheTTL := remoteMountCommand.Int("listingCacheTTL", 0, "seconds to cache remote directory listings (0 = disabled)")
+	cacheWait := remoteMountCommand.Duration("cacheWait", -1, "how long a read of an uncached object waits for the local cache, ex. 0 or 500ms (default: by object size)")
+
+	if err = remoteMountCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *dir == "" {
+		_, err = listExistingRemoteStorageMounts(commandEnv, writer)
+		return err
+	}
+
+	// find configuration for remote storage
+	remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote))
+	if err != nil {
+		return fmt.Errorf("find configuration for %s: %v", *remote, err)
+	}
+
+	remoteStorageLocation, err := remote_storage.ParseRemoteLocation(remoteConf.Type, *remote)
+	if err != nil {
+		return err
+	}
+	remoteStorageLocation.ListingCacheTtlSeconds = int32(*listingCacheTTL)
+	if *cacheWait >= 0 {
+		waitMs := cacheWait.Milliseconds()
+		if waitMs > math.MaxInt32 {
+			return fmt.Errorf("cacheWait %v is too long", *cacheWait)
+		}
+		// truncating to 0 would read as "never wait" instead of the asked-for wait
+		if waitMs == 0 && *cacheWait > 0 {
+			return fmt.Errorf("cacheWait %v is shorter than 1ms", *cacheWait)
+		}
+		remoteStorageLocation.CacheWaitMs = proto.Int32(int32(waitMs))
+	}
+
+	strategy := MetadataCacheStrategy(strings.ToLower(*metadataStrategy))
+	if strategy != MetadataCacheLazy && strategy != MetadataCacheEager {
+		return fmt.Errorf("metadataStrategy must be %s or %s, got %q", MetadataCacheLazy, MetadataCacheEager, *metadataStrategy)
+	}
+
+	if err = ensureMountDirectory(commandEnv, *dir, *nonEmpty, remoteConf); err != nil {
+		return fmt.Errorf("mount setup: %w", err)
+	}
+
+	if strategy == MetadataCacheEager {
+		if err = pullMetadata(commandEnv, writer, util.FullPath(*dir), remoteStorageLocation, util.FullPath(*dir), remoteConf, false, false); err != nil {
+			return fmt.Errorf("cache metadata: %w", err)
+		}
+	}
+
+	// store a mount configuration in filer
+	if err = filer.InsertMountMapping(commandEnv, *dir, remoteStorageLocation); err != nil {
+		return fmt.Errorf("save mount mapping: %w", err)
+	}
+
+	return nil
+}
+
+func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (mappings *remote_pb.RemoteStorageMapping, err error) {
+
+	// read current mapping
+	mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
+	if err != nil {
+		return mappings, err
+	}
+
+	jsonPrintln(writer, mappings)
+
+	return
+
+}
+
+func jsonPrintln(writer io.Writer, message proto.Message) error {
+	return filer.ProtoToText(writer, message)
+}
+
+func ensureMountDirectory(commandEnv *CommandEnv, dir string, nonEmpty bool, remoteConf *remote_pb.RemoteConf) error {
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		parent, name := util.FullPath(dir).DirAndName()
+		_, lookupErr := filer_pb.LookupEntry(context.Background(), client, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: parent,
+			Name:      name,
+		})
+		if lookupErr != nil {
+			if errors.Is(lookupErr, filer_pb.ErrNotFound) {
+				_, createErr := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+					Directory: parent,
+					Entry: &filer_pb.Entry{
+						Name:        name,
+						IsDirectory: true,
+						Attributes: &filer_pb.FuseAttributes{
+							Mtime:    time.Now().Unix(),
+							Crtime:   time.Now().Unix(),
+							FileMode: uint32(0755 | os.ModeDir),
+						},
+						RemoteEntry: &filer_pb.RemoteEntry{
+							StorageName: remoteConf.Name,
+						},
+					},
+				})
+				return createErr
+			}
+			return lookupErr
+		}
+
+		mountToDirIsEmpty := true
+		listErr := filer_pb.SeaweedList(context.Background(), client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
+			mountToDirIsEmpty = false
+			return nil
+		}, "", false, 1)
+
+		if listErr != nil {
+			return fmt.Errorf("list %s: %v", dir, listErr)
+		}
+
+		if !mountToDirIsEmpty {
+			if !nonEmpty {
+				return fmt.Errorf("dir %s is not empty", dir)
+			}
+		}
+
+		return nil
+	})
+}
+
+// if an entry has synchronized metadata but has not synchronized content
+//
+//	entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize
+//	entry.Attributes.Mtime    == entry.RemoteEntry.RemoteMtime
+//	entry.RemoteEntry.LastLocalSyncTsNs == 0
+//
+// if an entry has synchronized metadata but has synchronized content before
+//
+//	entry.Attributes.FileSize == entry.RemoteEntry.RemoteSize
+//	entry.Attributes.Mtime    == entry.RemoteEntry.RemoteMtime
+//	entry.RemoteEntry.LastLocalSyncTsNs > 0
+//
+// if an entry has synchronized metadata but has new updates
+//
+//	entry.Attributes.Mtime * 1,000,000,000    > entry.RemoteEntry.LastLocalSyncTsNs
+func doSaveRemoteEntry(client filer_pb.SeaweedFilerClient, localDir string, existingEntry *filer_pb.Entry, remoteEntry *filer_pb.RemoteEntry) error {
+	existingEntry.RemoteEntry = remoteEntry
+	existingEntry.Attributes.FileSize = uint64(remoteEntry.RemoteSize)
+	existingEntry.Attributes.Mtime = remoteEntry.RemoteMtime
+	existingEntry.Attributes.Md5 = nil
+	existingEntry.Attributes.TtlSec = 0 // Remote entries should not have TTL
+	existingEntry.Extended = filer.MergeRemoteContentEncoding(remoteEntry, existingEntry.Extended)
+	existingEntry.Chunks = nil
+	existingEntry.Content = nil
+	_, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+		Directory: localDir,
+		Entry:     existingEntry,
+	})
+	if updateErr != nil {
+		return updateErr
+	}
+	return nil
+}

weed/shell/command_remote_mount_buckets.go

@@ -0,0 +1,128 @@
+package shell
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"path/filepath"
+	"regexp"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteMountBuckets{})
+}
+
+type commandRemoteMountBuckets struct {
+}
+
+func (c *commandRemoteMountBuckets) Name() string {
+	return "remote.mount.buckets"
+}
+
+func (c *commandRemoteMountBuckets) Help() string {
+	return `mount all buckets in remote storage and pull its metadata
+
+	# assume a remote storage is configured to name "cloud1"
+	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+
+	# mount all buckets
+	remote.mount.buckets -remote=cloud1
+
+	# after mount, start a separate process to write updates to remote storage
+	weed filer.remote.gateway -filer=<filerHost>:<filerPort> -createBucketAt=cloud1
+
+`
+}
+
+func (c *commandRemoteMountBuckets) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteMountBucketsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	remote := remoteMountBucketsCommand.String("remote", "", "an already configured storage name")
+	bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'")
+	trimBucketSuffix := remoteMountBucketsCommand.Bool("trimBucketSuffix", true, "remote suffix auto generated by 'weed filer.remote.sync'")
+	apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets")
+	if err = remoteMountBucketsCommand.Parse(args); err != nil {
+		return nil
+	}
+	infoAboutSimulationMode(writer, *apply, "-apply")
+
+	if *remote == "" {
+		_, err = listExistingRemoteStorageMounts(commandEnv, writer)
+		return err
+	}
+
+	// find configuration for remote storage
+	remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *remote)
+	if err != nil {
+		return fmt.Errorf("find configuration for %s: %v", *remote, err)
+	}
+
+	// get storage client
+	remoteStorageClient, err := remote_storage.GetRemoteStorage(remoteConf)
+	if err != nil {
+		return fmt.Errorf("get storage client for %s: %v", *remote, err)
+	}
+
+	buckets, err := remoteStorageClient.ListBuckets()
+	if err != nil {
+		return fmt.Errorf("list buckets on %s: %v", *remote, err)
+	}
+
+	fillerBucketsPath, err := readFilerBucketsPath(commandEnv)
+	if err != nil {
+		return fmt.Errorf("read filer buckets path: %w", err)
+	}
+
+	hasSuffixPattern, _ := regexp.Compile(".+-[0-9][0-9][0-9][0-9]")
+
+	for _, bucket := range buckets {
+		if *bucketPattern != "" {
+			if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched {
+				continue
+			}
+		}
+
+		fmt.Fprintf(writer, "bucket %s\n", bucket.Name)
+		localBucketName := bucket.Name
+		if *trimBucketSuffix {
+			if hasSuffixPattern.MatchString(localBucketName) {
+				localBucketName = localBucketName[:len(localBucketName)-5]
+				fmt.Fprintf(writer, "  mount bucket %s as %s\n", bucket.Name, localBucketName)
+			}
+		}
+		if *apply {
+
+			dir := util.FullPath(fillerBucketsPath).Child(localBucketName)
+			remoteStorageLocation := &remote_pb.RemoteStorageLocation{
+				Name:   *remote,
+				Bucket: bucket.Name,
+				Path:   "/",
+			}
+
+			if err = ensureMountDirectory(commandEnv, string(dir), true, remoteConf); err != nil {
+				return fmt.Errorf("mount setup on %+v: %v", remoteStorageLocation, err)
+			}
+			if err = pullMetadata(commandEnv, writer, dir, remoteStorageLocation, dir, remoteConf, true, false); err != nil {
+				return fmt.Errorf("cache metadata on %+v: %v", remoteStorageLocation, err)
+			}
+
+			// store a mount configuration in filer
+			if err = filer.InsertMountMapping(commandEnv, string(dir), remoteStorageLocation); err != nil {
+				return fmt.Errorf("save mount mapping %s to %+v: %v", dir, remoteStorageLocation, err)
+			}
+
+		}
+	}
+
+	return nil
+}

weed/shell/command_remote_uncache.go

@@ -0,0 +1,208 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteUncache{})
+}
+
+type commandRemoteUncache struct {
+}
+
+func (c *commandRemoteUncache) Name() string {
+	return "remote.uncache"
+}
+
+func (c *commandRemoteUncache) Help() string {
+	return `keep the metadata but remote cache the file content for mounted directories or files
+
+	This is designed to run regularly. So you can add it to some cronjob.
+	If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.
+
+	remote.uncache -dir=/xxx
+	remote.uncache -dir=/xxx/some/sub/dir
+	remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
+	remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
+	remote.uncache -minSize=1024000    # uncache files larger than 100K
+	remote.uncache -minAge=3600        # uncache files older than 1 hour (created time)
+	remote.uncache -minCacheAge=3600   # uncache files older than 1 hour (cached time)
+
+`
+}
+
+func (c *commandRemoteUncache) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteUncacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteUncacheCommand.String("dir", "", "a directory in filer")
+	fileFiler := newFileFilter(remoteUncacheCommand)
+
+	if err = remoteUncacheCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
+	if listErr != nil {
+		return listErr
+	}
+	if *dir != "" {
+		var localMountedDir string
+		for k := range mappings.Mappings {
+			if strings.HasPrefix(*dir, k) {
+				localMountedDir = k
+			}
+		}
+		if localMountedDir == "" {
+			jsonPrintln(writer, mappings)
+			fmt.Fprintf(writer, "%s is not mounted\n", *dir)
+			return nil
+		}
+
+		// pull content from remote
+		if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
+			return fmt.Errorf("uncache content data: %w", err)
+		}
+		return nil
+	}
+
+	for key, _ := range mappings.Mappings {
+		if err := c.uncacheContentData(commandEnv, writer, util.FullPath(key), fileFiler); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {
+
+	return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {
+
+		if !mayHaveCachedToLocal(entry) {
+			return true // true means recursive traversal should continue
+		}
+
+		if !fileFilter.matches(entry) {
+			return true
+		}
+
+		if entry.RemoteEntry.LastLocalSyncTsNs/1e9 < entry.Attributes.Mtime {
+			return true // should not uncache an entry that is not synchronized with remote
+		}
+
+		entry.RemoteEntry.LastLocalSyncTsNs = 0
+		entry.Chunks = nil
+
+		fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))
+
+		err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			_, updateErr := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+				Directory: string(dir),
+				Entry:     entry,
+			})
+			return updateErr
+		})
+		if err != nil {
+			fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
+			return false
+		}
+		fmt.Fprintf(writer, "Done\n")
+
+		return true
+	})
+}
+
+type FileFilter struct {
+	include     *string
+	exclude     *string
+	minSize     *int64
+	maxSize     *int64
+	minAge      *int64
+	maxAge      *int64
+	minCacheAge *int64
+	now         int64
+}
+
+func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
+	ff = &FileFilter{}
+	ff.include = remoteMountCommand.String("include", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
+	ff.exclude = remoteMountCommand.String("exclude", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
+	ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
+	ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
+	ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds (created time)")
+	ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds (created time)")
+	ff.minCacheAge = remoteMountCommand.Int64("minCacheAge", -1, "minimum file cache age in seconds (last cached time)")
+	ff.now = time.Now().Unix()
+	return
+}
+
+// matchesName applies only the name-based include/exclude patterns,
+// usable for remote entries where local attributes are not available.
+func (ff *FileFilter) matchesName(name string) bool {
+	if *ff.include != "" {
+		if ok, _ := filepath.Match(*ff.include, name); !ok {
+			return false
+		}
+	}
+	if *ff.exclude != "" {
+		if ok, _ := filepath.Match(*ff.exclude, name); ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
+	if entry.Attributes == nil {
+		return false
+	}
+	if !ff.matchesName(entry.Name) {
+		return false
+	}
+	if *ff.minSize != -1 {
+		if int64(entry.Attributes.FileSize) < *ff.minSize {
+			return false
+		}
+	}
+	if *ff.maxSize != -1 {
+		if int64(entry.Attributes.FileSize) > *ff.maxSize {
+			return false
+		}
+	}
+	if *ff.minAge != -1 {
+		if entry.Attributes.Crtime+*ff.minAge > ff.now {
+			return false
+		}
+	}
+	if *ff.maxAge != -1 {
+		if entry.Attributes.Crtime+*ff.maxAge < ff.now {
+			return false
+		}
+	}
+	if *ff.minCacheAge != -1 {
+		lastCachedTime := entry.Attributes.Crtime
+		if entry.RemoteEntry != nil && entry.RemoteEntry.LastLocalSyncTsNs > 0 {
+			lastCachedTime = entry.RemoteEntry.LastLocalSyncTsNs / 1e9
+		}
+		if lastCachedTime+*ff.minCacheAge > ff.now {
+			return false
+		}
+	}
+	return true
+}

weed/shell/command_remote_uncache_test.go

@@ -0,0 +1,104 @@
+package shell
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func TestFileFilter_matches_minCacheAge(t *testing.T) {
+	now := time.Now().Unix()
+
+	tests := []struct {
+		name        string
+		minCacheAge int64
+		entry       *filer_pb.Entry
+		want        bool
+	}{
+		{
+			name:        "no minCacheAge",
+			minCacheAge: -1,
+			entry: &filer_pb.Entry{
+				Attributes: &filer_pb.FuseAttributes{Crtime: now - 100},
+			},
+			want: true,
+		},
+		{
+			name:        "recent cache, should not match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes: &filer_pb.FuseAttributes{Crtime: now - 7200},
+				RemoteEntry: &filer_pb.RemoteEntry{
+					LastLocalSyncTsNs: now * 1e9,
+				},
+			},
+			want: false,
+		},
+		{
+			name:        "old cache, should match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes: &filer_pb.FuseAttributes{Crtime: now - 7200},
+				RemoteEntry: &filer_pb.RemoteEntry{
+					LastLocalSyncTsNs: (now - 4000) * 1e9,
+				},
+			},
+			want: true,
+		},
+		{
+			name:        "no remote entry, uses crtime - recent, should not match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes: &filer_pb.FuseAttributes{Crtime: now - 100},
+			},
+			want: false,
+		},
+		{
+			name:        "no remote entry, uses crtime - old, should match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes: &filer_pb.FuseAttributes{Crtime: now - 4000},
+			},
+			want: true,
+		},
+		{
+			name:        "remote entry with 0 sync ts, uses crtime - recent, should not match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes:  &filer_pb.FuseAttributes{Crtime: now - 100},
+				RemoteEntry: &filer_pb.RemoteEntry{LastLocalSyncTsNs: 0},
+			},
+			want: false,
+		},
+		{
+			name:        "nil attributes, should not match",
+			minCacheAge: 3600,
+			entry: &filer_pb.Entry{
+				Attributes: nil,
+			},
+			want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			defaultString := ""
+			defaultInt64 := int64(-1)
+			ff := &FileFilter{
+				include:     &defaultString,
+				exclude:     &defaultString,
+				minSize:     &defaultInt64,
+				maxSize:     &defaultInt64,
+				minAge:      &defaultInt64,
+				maxAge:      &defaultInt64,
+				minCacheAge: &tt.minCacheAge,
+				now:         now,
+			}
+
+			if got := ff.matches(tt.entry); got != tt.want {
+				t.Errorf("FileFilter.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

weed/shell/command_remote_unmount.go

@@ -0,0 +1,127 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandRemoteUnmount{})
+}
+
+type commandRemoteUnmount struct {
+}
+
+func (c *commandRemoteUnmount) Name() string {
+	return "remote.unmount"
+}
+
+func (c *commandRemoteUnmount) Help() string {
+	return `unmount remote storage
+
+	# assume a remote storage is configured to name "s3_1"
+	remote.configure -name=s3_1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
+	# mount and pull one bucket
+	remote.mount -dir=/xxx -remote=s3_1/bucket
+
+	# unmount the mounted directory and remove its cache
+	remote.unmount -dir=/xxx
+
+`
+}
+
+func (c *commandRemoteUnmount) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	remoteMountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+
+	dir := remoteMountCommand.String("dir", "", "a directory in filer")
+
+	if err = remoteMountCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress)
+	if listErr != nil {
+		return listErr
+	}
+	if *dir == "" {
+		return jsonPrintln(writer, mappings)
+	}
+
+	_, found := mappings.Mappings[*dir]
+	if !found {
+		return fmt.Errorf("directory %s is not mounted", *dir)
+	}
+
+	// delete the mount mapping first: the filer reloads it on the spot, so the
+	// purge below stays local instead of deleting the remote objects
+	fmt.Fprintf(writer, "deleting mount for %s ...\n", *dir)
+	if err = filer.DeleteMountMapping(commandEnv, *dir); err != nil {
+		return fmt.Errorf("delete mount mapping: %w", err)
+	}
+
+	// purge mounted data
+	fmt.Fprintf(writer, "purge %s ...\n", *dir)
+	if err = c.purgeMountedData(commandEnv, *dir); err != nil {
+		return fmt.Errorf("purge mounted data: %w", err)
+	}
+
+	// reset remote sync offset in case the folder is mounted again
+	if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *dir, time.Now().UnixNano()); err != nil {
+		return fmt.Errorf("reset remote.sync offset for %s: %v", *dir, err)
+	}
+
+	return nil
+}
+
+func (c *commandRemoteUnmount) purgeMountedData(commandEnv *CommandEnv, dir string) error {
+
+	// find existing directory, and ensure the directory is empty
+	err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		ctx := context.Background()
+		parent, name := util.FullPath(dir).DirAndName()
+		lookupResp, lookupErr := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: parent,
+			Name:      name,
+		})
+		if lookupErr != nil {
+			return fmt.Errorf("lookup %s: %v", dir, lookupErr)
+		}
+
+		oldEntry := lookupResp.Entry
+
+		deleteError := filer_pb.DoRemove(ctx, client, parent, name, true, true, true, false, nil)
+		if deleteError != nil {
+			return fmt.Errorf("delete %s: %v", dir, deleteError)
+		}
+
+		mkdirErr := filer_pb.DoMkdir(ctx, client, parent, name, func(entry *filer_pb.Entry) {
+			entry.Attributes = oldEntry.Attributes
+			entry.Extended = oldEntry.Extended
+			entry.Attributes.Crtime = time.Now().Unix()
+			entry.Attributes.Mtime = time.Now().Unix()
+		})
+		if mkdirErr != nil {
+			return fmt.Errorf("mkdir %s: %v", dir, mkdirErr)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

weed/shell/command_s3_accesskey_create.go

@@ -0,0 +1,90 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyCreate{})
+}
+
+type commandS3AccessKeyCreate struct {
+}
+
+func (c *commandS3AccessKeyCreate) Name() string {
+	return "s3.accesskey.create"
+}
+
+func (c *commandS3AccessKeyCreate) Help() string {
+	return `create an additional access key for an S3 IAM user
+
+	s3.accesskey.create -user <username>
+	s3.accesskey.create -user <username> -access_key <key> -secret_key <secret>
+
+	Generates a new credential pair for an existing user. If -access_key and
+	-secret_key are omitted, they are generated automatically.
+`
+}
+
+func (c *commandS3AccessKeyCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	accessKey := f.String("access_key", "", "access key (generated if omitted)")
+	secretKey := f.String("secret_key", "", "secret key (generated if omitted)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	ak := *accessKey
+	sk := *secretKey
+
+	if ak == "" && sk == "" {
+		var err error
+		ak, err = iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+		if err != nil {
+			return fmt.Errorf("generate access key: %v", err)
+		}
+		sk, err = iam.GenerateSecretAccessKey()
+		if err != nil {
+			return fmt.Errorf("generate secret key: %v", err)
+		}
+	} else if ak == "" || sk == "" {
+		return fmt.Errorf("both -access_key and -secret_key must be provided together, or omit both to auto-generate")
+	}
+
+	err := commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		_, err := client.CreateAccessKey(ctx, &iam_pb.CreateAccessKeyRequest{
+			Username: *user,
+			Credential: &iam_pb.Credential{
+				AccessKey: ak,
+				SecretKey: sk,
+				Status:    iam.AccessKeyStatusActive,
+			},
+		})
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Created access key for user %q\n", *user)
+	fmt.Fprintf(writer, "Access Key: %s\n", ak)
+	fmt.Fprintf(writer, "Secret Key: %s\n", sk)
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}

weed/shell/command_s3_accesskey_delete.go

@@ -0,0 +1,62 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyDelete{})
+}
+
+type commandS3AccessKeyDelete struct {
+}
+
+func (c *commandS3AccessKeyDelete) Name() string {
+	return "s3.accesskey.delete"
+}
+
+func (c *commandS3AccessKeyDelete) Help() string {
+	return `delete an access key from an S3 IAM user
+
+	s3.accesskey.delete -user <username> -access_key <key>
+`
+}
+
+func (c *commandS3AccessKeyDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	accessKey := f.String("access_key", "", "access key to delete")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+	if *accessKey == "" {
+		return fmt.Errorf("-access_key is required")
+	}
+
+	err := commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		_, err := client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
+			Username:  *user,
+			AccessKey: *accessKey,
+		})
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Deleted access key %s from user %q\n", *accessKey, *user)
+	return nil
+}

weed/shell/command_s3_accesskey_list.go

@@ -0,0 +1,68 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"text/tabwriter"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyList{})
+}
+
+type commandS3AccessKeyList struct {
+}
+
+func (c *commandS3AccessKeyList) Name() string {
+	return "s3.accesskey.list"
+}
+
+func (c *commandS3AccessKeyList) Help() string {
+	return `list access keys for an S3 IAM user
+
+	s3.accesskey.list -user <username>
+`
+}
+
+func (c *commandS3AccessKeyList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return err
+		}
+
+		if len(resp.Identity.Credentials) == 0 {
+			fmt.Fprintf(writer, "No access keys for user %q.\n", *user)
+			return nil
+		}
+
+		tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "ACCESS KEY\tSTATUS")
+		for _, cred := range resp.Identity.Credentials {
+			st := cred.Status
+			if st == "" {
+				st = "Active"
+			}
+			fmt.Fprintf(tw, "%s\t%s\n", cred.AccessKey, st)
+		}
+		return tw.Flush()
+	})
+}

weed/shell/command_s3_accesskey_rotate.go

@@ -0,0 +1,98 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AccessKeyRotate{})
+}
+
+type commandS3AccessKeyRotate struct {
+}
+
+func (c *commandS3AccessKeyRotate) Name() string {
+	return "s3.accesskey.rotate"
+}
+
+func (c *commandS3AccessKeyRotate) Help() string {
+	return `rotate an access key for an S3 IAM user
+
+	s3.accesskey.rotate -user <username> -access_key <old_key>
+
+	Creates a new credential pair and deletes the old one. There is a brief
+	window where both keys are valid.
+`
+}
+
+func (c *commandS3AccessKeyRotate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AccessKeyRotate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "user name")
+	oldKey := f.String("access_key", "", "access key to rotate")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+	if *oldKey == "" {
+		return fmt.Errorf("-access_key is required")
+	}
+
+	newAK, err := iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+	if err != nil {
+		return fmt.Errorf("generate access key: %v", err)
+	}
+	newSK, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		return fmt.Errorf("generate secret key: %v", err)
+	}
+
+	err = commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		// Create new key first so there's no gap without credentials
+		_, err := client.CreateAccessKey(ctx, &iam_pb.CreateAccessKeyRequest{
+			Username: *user,
+			Credential: &iam_pb.Credential{
+				AccessKey: newAK,
+				SecretKey: newSK,
+				Status:    iam.AccessKeyStatusActive,
+			},
+		})
+		if err != nil {
+			return fmt.Errorf("create new key: %v", err)
+		}
+
+		// Delete old key
+		_, err = client.DeleteAccessKey(ctx, &iam_pb.DeleteAccessKeyRequest{
+			Username:  *user,
+			AccessKey: *oldKey,
+		})
+		if err != nil {
+			return fmt.Errorf("delete old key (new key %s was already created): %v", newAK, err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Rotated access key for user %q\n", *user)
+	fmt.Fprintf(writer, "Old Key:    %s (deleted)\n", *oldKey)
+	fmt.Fprintf(writer, "Access Key: %s\n", newAK)
+	fmt.Fprintf(writer, "Secret Key: %s\n", newSK)
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}

weed/shell/command_s3_anonymous_get.go

@@ -0,0 +1,82 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousGet{})
+}
+
+type commandS3AnonymousGet struct {
+}
+
+func (c *commandS3AnonymousGet) Name() string {
+	return "s3.anonymous.get"
+}
+
+func (c *commandS3AnonymousGet) Help() string {
+	return `show anonymous access for a bucket
+
+	s3.anonymous.get -bucket <bucket_name>
+`
+}
+
+func (c *commandS3AnonymousGet) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousGet) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucket := f.String("bucket", "", "bucket name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucket == "" {
+		return fmt.Errorf("-bucket is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+		if err != nil {
+			st, ok := status.FromError(err)
+			if ok && st.Code() == codes.NotFound {
+				fmt.Fprintf(writer, "Bucket: %s\nAccess: none\n", *bucket)
+				return nil
+			}
+			return err
+		}
+		if resp.Identity == nil {
+			fmt.Fprintf(writer, "Bucket: %s\nAccess: none\n", *bucket)
+			return nil
+		}
+
+		var actions []string
+		for _, a := range resp.Identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) == 2 && parts[1] == *bucket {
+				actions = append(actions, parts[0])
+			}
+		}
+
+		fmt.Fprintf(writer, "Bucket: %s\n", *bucket)
+		if len(actions) == 0 {
+			fmt.Fprintln(writer, "Access: none")
+		} else {
+			sort.Strings(actions)
+			fmt.Fprintf(writer, "Access: %s\n", strings.Join(actions, ", "))
+		}
+
+		return nil
+	})
+}

weed/shell/command_s3_anonymous_list.go

@@ -0,0 +1,84 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousList{})
+}
+
+type commandS3AnonymousList struct {
+}
+
+func (c *commandS3AnonymousList) Name() string {
+	return "s3.anonymous.list"
+}
+
+func (c *commandS3AnonymousList) Help() string {
+	return `list all buckets with anonymous access
+
+	s3.anonymous.list
+`
+}
+
+func (c *commandS3AnonymousList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+		if err != nil {
+			st, ok := status.FromError(err)
+			if ok && st.Code() == codes.NotFound {
+				fmt.Fprintln(writer, "No anonymous access configured.")
+				return nil
+			}
+			return err
+		}
+		if resp.Identity == nil {
+			fmt.Fprintln(writer, "No anonymous access configured.")
+			return nil
+		}
+
+		// Group actions by bucket
+		bucketActions := map[string][]string{}
+		for _, a := range resp.Identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) == 2 {
+				bucketActions[parts[1]] = append(bucketActions[parts[1]], parts[0])
+			}
+		}
+
+		if len(bucketActions) == 0 {
+			fmt.Fprintln(writer, "No anonymous access configured.")
+			return nil
+		}
+
+		// Sort bucket names
+		buckets := make([]string, 0, len(bucketActions))
+		for b := range bucketActions {
+			buckets = append(buckets, b)
+		}
+		sort.Strings(buckets)
+
+		tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "BUCKET\tACCESS")
+		for _, b := range buckets {
+			actions := bucketActions[b]
+			sort.Strings(actions)
+			fmt.Fprintf(tw, "%s\t%s\n", b, strings.Join(actions, ", "))
+		}
+		return tw.Flush()
+	})
+}

weed/shell/command_s3_anonymous_set.go

@@ -0,0 +1,137 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const anonymousUserName = "anonymous"
+
+func init() {
+	Commands = append(Commands, &commandS3AnonymousSet{})
+}
+
+type commandS3AnonymousSet struct {
+}
+
+func (c *commandS3AnonymousSet) Name() string {
+	return "s3.anonymous.set"
+}
+
+func (c *commandS3AnonymousSet) Help() string {
+	return `set anonymous (public) access on a bucket
+
+	s3.anonymous.set -bucket <bucket_name> -access Read,List
+	s3.anonymous.set -bucket <bucket_name> -access none
+
+	Supported actions: Read, Write, List, Tagging, Admin
+	Use "none" to remove all anonymous access for the bucket.
+
+	This manages the special "anonymous" user's actions. It does not
+	use IAM policies — it sets legacy per-bucket actions directly.
+`
+}
+
+func (c *commandS3AnonymousSet) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3AnonymousSet) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucket := f.String("bucket", "", "bucket name")
+	access := f.String("access", "", "comma-separated actions: Read,Write,List,Tagging,Admin or none")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucket == "" {
+		return fmt.Errorf("-bucket is required")
+	}
+	if *access == "" {
+		return fmt.Errorf("-access is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		// Get or create anonymous user
+		identity, isNew, err := getOrCreateAnonymousUser(ctx, client)
+		if err != nil {
+			return err
+		}
+
+		// Remove existing actions for this bucket
+		var kept []string
+		for _, a := range identity.Actions {
+			parts := strings.SplitN(a, ":", 2)
+			if len(parts) != 2 || parts[1] != *bucket {
+				kept = append(kept, a)
+			}
+		}
+
+		// Add new actions unless "none"
+		canonicalActions := map[string]string{
+			"read": "Read", "write": "Write", "list": "List",
+			"tagging": "Tagging", "admin": "Admin",
+		}
+		if strings.ToLower(strings.TrimSpace(*access)) != "none" {
+			seen := make(map[string]struct{})
+			for _, action := range strings.Split(*access, ",") {
+				action = strings.TrimSpace(action)
+				if action != "" {
+					canonical, ok := canonicalActions[strings.ToLower(action)]
+					if !ok {
+						return fmt.Errorf("invalid action %q: supported actions are Read, Write, List, Tagging, Admin", action)
+					}
+					if _, dup := seen[canonical]; dup {
+						continue
+					}
+					seen[canonical] = struct{}{}
+					kept = append(kept, canonical+":"+*bucket)
+				}
+			}
+		}
+
+		identity.Actions = kept
+
+		if isNew {
+			_, err = client.CreateUser(ctx, &iam_pb.CreateUserRequest{Identity: identity})
+		} else {
+			_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+				Username: anonymousUserName,
+				Identity: identity,
+			})
+		}
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(writer, "Set anonymous access on bucket %q to: %s\n", *bucket, *access)
+		return nil
+	})
+}
+
+func getOrCreateAnonymousUser(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) (*iam_pb.Identity, bool, error) {
+	resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: anonymousUserName})
+	if err == nil {
+		if resp.Identity == nil {
+			return nil, false, fmt.Errorf("anonymous user returned nil identity")
+		}
+		return resp.Identity, false, nil
+	}
+
+	st, ok := status.FromError(err)
+	if ok && st != nil && st.Code() == codes.NotFound {
+		return &iam_pb.Identity{
+			Name:    anonymousUserName,
+			Actions: []string{},
+		}, true, nil
+	}
+
+	return nil, false, fmt.Errorf("failed to get anonymous user: %w", err)
+}

weed/shell/command_s3_bucket_access.go

@@ -0,0 +1,202 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// canonicalActions maps lowercased action names to their canonical form.
+var canonicalActions = map[string]string{
+	"read":    "Read",
+	"write":   "Write",
+	"list":    "List",
+	"tagging": "Tagging",
+	"admin":   "Admin",
+}
+
+func init() {
+	Commands = append(Commands, &commandS3BucketAccess{})
+}
+
+type commandS3BucketAccess struct {
+}
+
+func (c *commandS3BucketAccess) Name() string {
+	return "s3.bucket.access"
+}
+
+func (c *commandS3BucketAccess) Help() string {
+	return `view or set per-bucket access for a user
+
+	Example:
+		# View current access for a user on a bucket
+		s3.bucket.access -name <bucket_name> -user <username>
+
+		# Grant anonymous read and list access
+		s3.bucket.access -name <bucket_name> -user anonymous -access Read,List
+
+		# Grant full anonymous access
+		s3.bucket.access -name <bucket_name> -user anonymous -access Read,Write,List
+
+		# Remove all access for a user on a bucket
+		s3.bucket.access -name <bucket_name> -user <username> -access none
+
+	Supported action names (comma-separated):
+		Read, Write, List, Tagging, Admin
+
+	The user is auto-created if it does not exist. Actions are scoped to
+	the specified bucket (stored as "Action:bucket" in the identity).
+`
+}
+
+func (c *commandS3BucketAccess) HasTag(tag CommandTag) bool {
+	return tag == Hidden
+}
+
+func (c *commandS3BucketAccess) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	userName := bucketCommand.String("user", "", "user name")
+	access := bucketCommand.String("access", "", "comma-separated actions: Read,Write,List,Tagging,Admin or none")
+	if err = bucketCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+	if *userName == "" {
+		return fmt.Errorf("empty user name")
+	}
+
+	accessStr := strings.TrimSpace(*access)
+
+	// Validate and normalize actions to canonical casing
+	if accessStr != "" && strings.ToLower(accessStr) != "none" {
+		var normalized []string
+		for _, a := range strings.Split(accessStr, ",") {
+			a = strings.TrimSpace(a)
+			canonical, ok := canonicalActions[strings.ToLower(a)]
+			if !ok {
+				return fmt.Errorf("invalid action %q: must be Read, Write, List, Tagging, Admin, or none", a)
+			}
+			normalized = append(normalized, canonical)
+		}
+		accessStr = strings.Join(normalized, ",")
+	}
+
+	err = commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+
+		// Get or create user
+		identity, isNewUser, getErr := getOrCreateIdentity(ctx, client, *userName)
+		if getErr != nil {
+			return getErr
+		}
+
+		// View mode: show current bucket-scoped actions
+		if accessStr == "" {
+			return displayBucketAccess(writer, *bucketName, *userName, identity)
+		}
+
+		// Set mode: update actions
+		updateBucketActions(identity, *bucketName, accessStr)
+
+		// Show the resulting identity
+		var buf bytes.Buffer
+		filer.ProtoToText(&buf, identity)
+		fmt.Fprint(writer, buf.String())
+		fmt.Fprintln(writer)
+
+		// Save
+		if isNewUser {
+			if _, err := client.CreateUser(ctx, &iam_pb.CreateUserRequest{Identity: identity}); err != nil {
+				return fmt.Errorf("failed to create user %s: %w", *userName, err)
+			}
+			fmt.Fprintf(writer, "Created user %q and set access on bucket %s.\n", *userName, *bucketName)
+		} else {
+			if _, err := client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{Username: *userName, Identity: identity}); err != nil {
+				return fmt.Errorf("failed to update user %s: %w", *userName, err)
+			}
+			fmt.Fprintf(writer, "Updated access for user %q on bucket %s.\n", *userName, *bucketName)
+		}
+		return nil
+	})
+
+	return err
+}
+
+func getOrCreateIdentity(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient, userName string) (*iam_pb.Identity, bool, error) {
+	resp, getErr := client.GetUser(ctx, &iam_pb.GetUserRequest{
+		Username: userName,
+	})
+	if getErr == nil && resp.Identity != nil {
+		return resp.Identity, false, nil
+	}
+
+	st, ok := status.FromError(getErr)
+	if ok && st.Code() == codes.NotFound {
+		return &iam_pb.Identity{
+			Name:        userName,
+			Credentials: []*iam_pb.Credential{},
+			Actions:     []string{},
+			PolicyNames: []string{},
+		}, true, nil
+	}
+
+	return nil, false, fmt.Errorf("failed to get user %s: %v", userName, getErr)
+}
+
+func displayBucketAccess(writer io.Writer, bucketName, userName string, identity *iam_pb.Identity) error {
+	suffix := ":" + bucketName
+	var actions []string
+	for _, a := range identity.Actions {
+		if strings.HasSuffix(a, suffix) {
+			actions = append(actions, strings.TrimSuffix(a, suffix))
+		}
+	}
+	fmt.Fprintf(writer, "Bucket: %s\n", bucketName)
+	fmt.Fprintf(writer, "User:   %s\n", userName)
+	if len(actions) == 0 {
+		fmt.Fprintln(writer, "Access: none")
+	} else {
+		sort.Strings(actions)
+		fmt.Fprintf(writer, "Access: %s\n", strings.Join(actions, ","))
+	}
+	return nil
+}
+
+// updateBucketActions removes existing actions for the bucket and adds the new ones.
+func updateBucketActions(identity *iam_pb.Identity, bucketName, accessStr string) {
+	suffix := ":" + bucketName
+
+	// Remove existing actions for this bucket
+	var kept []string
+	for _, a := range identity.Actions {
+		if !strings.HasSuffix(a, suffix) {
+			kept = append(kept, a)
+		}
+	}
+
+	// Add new actions (unless "none")
+	if strings.ToLower(accessStr) != "none" {
+		for _, action := range strings.Split(accessStr, ",") {
+			action = strings.TrimSpace(action)
+			if action != "" {
+				kept = append(kept, action+suffix)
+			}
+		}
+	}
+
+	identity.Actions = kept
+}

weed/shell/command_s3_bucket_access_test.go

@@ -0,0 +1,119 @@
+package shell
+
+import (
+	"bytes"
+	"slices"
+	"sort"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func TestUpdateBucketActions_SetActions(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "anonymous",
+		Actions: []string{},
+	}
+
+	updateBucketActions(identity, "my-bucket", "Read,List")
+
+	expected := []string{"Read:my-bucket", "List:my-bucket"}
+	sort.Strings(identity.Actions)
+	sort.Strings(expected)
+	if !slices.Equal(identity.Actions, expected) {
+		t.Errorf("got %v, want %v", identity.Actions, expected)
+	}
+}
+
+func TestUpdateBucketActions_ReplaceActions(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "anonymous",
+		Actions: []string{"Read:my-bucket", "List:my-bucket"},
+	}
+
+	updateBucketActions(identity, "my-bucket", "Write")
+
+	expected := []string{"Write:my-bucket"}
+	if !slices.Equal(identity.Actions, expected) {
+		t.Errorf("got %v, want %v", identity.Actions, expected)
+	}
+}
+
+func TestUpdateBucketActions_None(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "anonymous",
+		Actions: []string{"Read:my-bucket", "List:my-bucket"},
+	}
+
+	updateBucketActions(identity, "my-bucket", "none")
+
+	if len(identity.Actions) != 0 {
+		t.Errorf("expected empty actions, got %v", identity.Actions)
+	}
+}
+
+func TestUpdateBucketActions_PreservesOtherBuckets(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "testuser",
+		Actions: []string{"Read:bucket-a", "Write:bucket-b", "List:bucket-a"},
+	}
+
+	updateBucketActions(identity, "bucket-a", "Write")
+
+	expected := []string{"Write:bucket-b", "Write:bucket-a"}
+	sort.Strings(identity.Actions)
+	sort.Strings(expected)
+	if !slices.Equal(identity.Actions, expected) {
+		t.Errorf("got %v, want %v", identity.Actions, expected)
+	}
+}
+
+func TestUpdateBucketActions_PreservesGlobalActions(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "testuser",
+		Actions: []string{"Admin", "Read:my-bucket"},
+	}
+
+	updateBucketActions(identity, "my-bucket", "none")
+
+	expected := []string{"Admin"}
+	if !slices.Equal(identity.Actions, expected) {
+		t.Errorf("got %v, want %v", identity.Actions, expected)
+	}
+}
+
+func TestDisplayBucketAccess(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "anonymous",
+		Actions: []string{"Read:my-bucket", "List:my-bucket", "Write:other-bucket"},
+	}
+
+	var buf bytes.Buffer
+	err := displayBucketAccess(&buf, "my-bucket", "anonymous", identity)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	output := buf.String()
+	if !bytes.Contains([]byte(output), []byte("List,Read")) {
+		t.Errorf("expected 'List,Read' in output, got: %s", output)
+	}
+}
+
+func TestDisplayBucketAccess_None(t *testing.T) {
+	identity := &iam_pb.Identity{
+		Name:    "anonymous",
+		Actions: []string{"Write:other-bucket"},
+	}
+
+	var buf bytes.Buffer
+	err := displayBucketAccess(&buf, "my-bucket", "anonymous", identity)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	output := buf.String()
+	if !bytes.Contains([]byte(output), []byte("Access: none")) {
+		t.Errorf("expected 'Access: none' in output, got: %s", output)
+	}
+}

weed/shell/command_s3_bucket_create.go

@@ -0,0 +1,137 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketCreate{})
+}
+
+type commandS3BucketCreate struct {
+}
+
+func (c *commandS3BucketCreate) Name() string {
+	return "s3.bucket.create"
+}
+
+func (c *commandS3BucketCreate) Help() string {
+	return `create a bucket with a given name
+
+	Example:
+		s3.bucket.create -name <bucket_name>
+		s3.bucket.create -name <bucket_name> -owner <identity_name>
+		s3.bucket.create -name <bucket_name> -withLock
+
+	The -owner flag sets the bucket owner identity. This is important when using
+	S3 IAM authentication, as non-admin users can only access buckets they own.
+	If not specified, the bucket will have no owner and will only be accessible
+	by admin users.
+
+	The -owner value should match the identity name configured in your S3 IAM
+	system (the "name" field in s3.json identities configuration).
+
+	The -withLock flag enables S3 Object Lock on the bucket. This provides WORM
+	(Write Once Read Many) protection for objects. Once enabled, Object Lock
+	cannot be disabled. Versioning is automatically enabled when using this flag.
+`
+}
+
+func (c *commandS3BucketCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	bucketOwner := bucketCommand.String("owner", "", "bucket owner identity name (for S3 IAM authentication)")
+	withLock := bucketCommand.Bool("withLock", false, "enable Object Lock on the bucket (requires and enables versioning)")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	err = s3bucket.VerifyS3BucketName(*bucketName)
+	if err != nil {
+		return err
+	}
+
+	// Trim whitespace from owner and treat whitespace-only as empty
+	owner := strings.TrimSpace(*bucketOwner)
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		fmt.Fprintln(writer, "create bucket under", filerBucketsPath)
+
+		entry := &filer_pb.Entry{
+			Name:        *bucketName,
+			IsDirectory: true,
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime:    time.Now().Unix(),
+				Crtime:   time.Now().Unix(),
+				FileMode: uint32(0777 | os.ModeDir),
+			},
+		}
+
+		// Set bucket owner if specified
+		if owner != "" {
+			if entry.Extended == nil {
+				entry.Extended = make(map[string][]byte)
+			}
+			entry.Extended[s3_constants.AmzIdentityId] = []byte(owner)
+		}
+
+		// Enable Object Lock if specified
+		if *withLock {
+			if entry.Extended == nil {
+				entry.Extended = make(map[string][]byte)
+			}
+			// Enable versioning (required for Object Lock)
+			entry.Extended[s3_constants.ExtVersioningKey] = []byte(s3_constants.VersioningEnabled)
+			// Enable Object Lock
+			entry.Extended[s3_constants.ExtObjectLockEnabledKey] = []byte(s3_constants.ObjectLockEnabled)
+		}
+
+		if _, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
+			Directory: filerBucketsPath,
+			Entry:     entry,
+		}); err != nil {
+			return err
+		}
+
+		fmt.Fprintln(writer, "created bucket", *bucketName)
+		if owner != "" {
+			fmt.Fprintln(writer, "bucket owner:", owner)
+		}
+		if *withLock {
+			fmt.Fprintln(writer, "Object Lock: enabled")
+			fmt.Fprintln(writer, "Versioning: enabled")
+		}
+
+		return nil
+
+	})
+
+	return err
+
+}

weed/shell/command_s3_bucket_delete.go

@@ -0,0 +1,81 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_objectlock"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketDelete{})
+}
+
+type commandS3BucketDelete struct {
+}
+
+func (c *commandS3BucketDelete) Name() string {
+	return "s3.bucket.delete"
+}
+
+func (c *commandS3BucketDelete) Help() string {
+	return `delete a bucket by a given name
+
+	s3.bucket.delete -name <bucket_name>
+`
+}
+
+func (c *commandS3BucketDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	_, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args()))
+	if parseErr != nil {
+		return parseErr
+	}
+
+	var filerBucketsPath string
+	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
+	if err != nil {
+		return fmt.Errorf("read buckets: %w", err)
+	}
+
+	// Check if bucket has Object Lock enabled and if there are locked objects
+	ctx := context.Background()
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return s3_objectlock.CheckBucketForLockedObjects(ctx, client, filerBucketsPath, *bucketName)
+	})
+	if err != nil {
+		return err
+	}
+
+	// delete the collection directly first
+	err = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
+		_, err = client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
+			Name: getCollectionName(commandEnv, *bucketName),
+		})
+		return err
+	})
+	if err != nil {
+		return
+	}
+
+	return filer_pb.Remove(ctx, commandEnv, filerBucketsPath, *bucketName, false, true, true, false, nil)
+
+}

weed/shell/command_s3_bucket_lifecycle_fastpath.go

@@ -0,0 +1,118 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketLifecycleFastpath{})
+}
+
+type commandS3BucketLifecycleFastpath struct {
+}
+
+func (c *commandS3BucketLifecycleFastpath) Name() string {
+	return "s3.bucket.lifecycle.fastpath"
+}
+
+func (c *commandS3BucketLifecycleFastpath) Help() string {
+	return `view or toggle the per-bucket lifecycle TTL fast path
+
+	When enabled, an Expiration.Days lifecycle rule is stamped as a volume
+	TTL at PutObject time, so the volume server reclaims the data on its own
+	and the lifecycle worker skips per-chunk deletes. Off by default: a
+	volume TTL is baked into the object at write time and can't honor a later
+	policy change (rule removed or lengthened), unlike worker-driven
+	expiration. The fast path never applies to versioned or object-locked
+	buckets regardless of this flag.
+
+	Example:
+		# Show the current setting
+		s3.bucket.lifecycle.fastpath -name <bucket_name>
+
+		# Enable
+		s3.bucket.lifecycle.fastpath -name <bucket_name> -enable
+
+		# Disable
+		s3.bucket.lifecycle.fastpath -name <bucket_name> -disable
+`
+}
+
+func (c *commandS3BucketLifecycleFastpath) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketLifecycleFastpath) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	enable := bucketCommand.Bool("enable", false, "enable the lifecycle TTL fast path")
+	disable := bucketCommand.Bool("disable", false, "disable the lifecycle TTL fast path")
+	if err = bucketCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+	if err := s3bucket.VerifyS3BucketName(*bucketName); err != nil {
+		return fmt.Errorf("invalid bucket name %q: %w", *bucketName, err)
+	}
+	if *enable && *disable {
+		return fmt.Errorf("only one of -enable or -disable can be set")
+	}
+
+	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: filerBucketsPath,
+			Name:      *bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("lookup bucket %s: %w", *bucketName, err)
+		}
+		entry := lookupResp.Entry
+
+		if !*enable && !*disable {
+			state := "disabled"
+			if string(entry.Extended[s3_constants.ExtLifecycleTtlFastPathKey]) == "true" {
+				state = "enabled"
+			}
+			fmt.Fprintf(writer, "Bucket: %s\n", *bucketName)
+			fmt.Fprintf(writer, "Lifecycle TTL fast path: %s\n", state)
+			return nil
+		}
+
+		if entry.Extended == nil {
+			entry.Extended = make(map[string][]byte)
+		}
+		state := "disabled"
+		if *enable {
+			entry.Extended[s3_constants.ExtLifecycleTtlFastPathKey] = []byte("true")
+			state = "enabled"
+		} else {
+			delete(entry.Extended, s3_constants.ExtLifecycleTtlFastPathKey)
+		}
+
+		if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+			Directory: filerBucketsPath,
+			Entry:     entry,
+		}); err != nil {
+			return fmt.Errorf("failed to update bucket: %w", err)
+		}
+
+		fmt.Fprintf(writer, "Bucket %s lifecycle TTL fast path %s\n", *bucketName, state)
+		return nil
+	})
+}

weed/shell/command_s3_bucket_list.go

@@ -0,0 +1,108 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketList{})
+}
+
+type commandS3BucketList struct {
+}
+
+func (c *commandS3BucketList) Name() string {
+	return "s3.bucket.list"
+}
+
+func (c *commandS3BucketList) Help() string {
+	return `list all buckets
+
+`
+}
+
+func (c *commandS3BucketList) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	// collect collection information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	collectionInfos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topologyInfo, collectionInfos)
+
+	_, parseErr := commandEnv.parseUrl(findInputDirectory(bucketCommand.Args()))
+	if parseErr != nil {
+		return parseErr
+	}
+
+	var filerBucketsPath string
+	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
+	if err != nil {
+		return fmt.Errorf("read buckets: %w", err)
+	}
+
+	err = filer_pb.List(context.Background(), commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		if !entry.IsDirectory || strings.HasPrefix(entry.Name, ".") {
+			return nil
+		}
+		collection := getCollectionName(commandEnv, entry.Name)
+		var collectionSize, logicalSize, fileCount float64
+		if collectionInfo, found := collectionInfos[collection]; found {
+			collectionSize = collectionInfo.Size
+			logicalSize = collectionInfo.LogicalSize()
+			fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
+		}
+		fmt.Fprintf(writer, "  %s\tsize:%.0f\tlogical:%.0f\tchunk:%.0f", entry.Name, collectionSize, logicalSize, fileCount)
+		if entry.Quota > 0 {
+			fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, logicalSize*100/float64(entry.Quota))
+		}
+		// Show bucket owner (use %q to escape special characters)
+		if entry.Extended != nil {
+			if owner, ok := entry.Extended[s3_constants.AmzIdentityId]; ok && len(owner) > 0 {
+				fmt.Fprintf(writer, "\towner:%q", string(owner))
+			}
+		}
+		fmt.Fprintln(writer)
+		return nil
+	}, "", false, math.MaxUint32)
+	if err != nil {
+		return fmt.Errorf("list buckets under %v: %w", filerBucketsPath, err)
+	}
+
+	return err
+
+}
+
+func readFilerBucketsPath(filerClient filer_pb.FilerClient) (filerBucketsPath string, err error) {
+	err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath = resp.DirBuckets
+
+		return nil
+
+	})
+
+	return filerBucketsPath, err
+}

weed/shell/command_s3_bucket_lock.go

@@ -0,0 +1,152 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketLock{})
+}
+
+type commandS3BucketLock struct {
+}
+
+func (c *commandS3BucketLock) Name() string {
+	return "s3.bucket.lock"
+}
+
+func (c *commandS3BucketLock) Help() string {
+	return `view or enable Object Lock for an S3 bucket
+
+	Example:
+		# View the current Object Lock status of a bucket
+		s3.bucket.lock -name <bucket_name>
+
+		# Enable Object Lock on an existing bucket (irreversible)
+		s3.bucket.lock -name <bucket_name> -enable
+
+	Object Lock provides WORM (Write Once Read Many) protection for objects.
+	Once enabled, Object Lock cannot be disabled on a bucket.
+
+	Requirements:
+		- Versioning will be automatically enabled if not already enabled
+		- Object Lock can only be enabled, never disabled (following AWS S3 behavior)
+
+	After enabling Object Lock, you can:
+		- Set default retention policy on the bucket
+		- Set retention and legal hold on individual objects
+`
+}
+
+func (c *commandS3BucketLock) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketLock) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	enableLock := bucketCommand.Bool("enable", false, "enable Object Lock on the bucket (irreversible)")
+	if err = bucketCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		// Look up the bucket entry
+		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: filerBucketsPath,
+			Name:      *bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("lookup bucket %s: %w", *bucketName, err)
+		}
+
+		entry := lookupResp.Entry
+
+		// Check current Object Lock status
+		currentLockEnabled := false
+		currentVersioningEnabled := false
+		if entry.Extended != nil {
+			if lockStatus, ok := entry.Extended[s3_constants.ExtObjectLockEnabledKey]; ok {
+				currentLockEnabled = string(lockStatus) == s3_constants.ObjectLockEnabled
+			}
+			if versioningStatus, ok := entry.Extended[s3_constants.ExtVersioningKey]; ok {
+				currentVersioningEnabled = string(versioningStatus) == s3_constants.VersioningEnabled
+			}
+		}
+
+		// If -enable is provided, enable Object Lock
+		if *enableLock {
+			if currentLockEnabled {
+				fmt.Fprintf(writer, "Object Lock is already enabled on bucket %s\n", *bucketName)
+				return nil
+			}
+
+			if entry.Extended == nil {
+				entry.Extended = make(map[string][]byte)
+			}
+
+			// Enable versioning if not already enabled (required for Object Lock)
+			if !currentVersioningEnabled {
+				entry.Extended[s3_constants.ExtVersioningKey] = []byte(s3_constants.VersioningEnabled)
+				fmt.Fprintf(writer, "Enabling versioning on bucket %s (required for Object Lock)\n", *bucketName)
+			}
+
+			// Enable Object Lock
+			entry.Extended[s3_constants.ExtObjectLockEnabledKey] = []byte(s3_constants.ObjectLockEnabled)
+			fmt.Fprintf(writer, "Enabling Object Lock on bucket %s\n", *bucketName)
+
+			// Update the entry
+			if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+				Directory: filerBucketsPath,
+				Entry:     entry,
+			}); err != nil {
+				return fmt.Errorf("failed to update bucket: %w", err)
+			}
+
+			fmt.Fprintf(writer, "Object Lock enabled successfully.\n")
+			fmt.Fprintf(writer, "WARNING: This action is irreversible. Object Lock cannot be disabled.\n")
+			return nil
+		}
+
+		// Display current status (no flags provided)
+		fmt.Fprintf(writer, "Bucket: %s\n", *bucketName)
+		fmt.Fprintf(writer, "Path: %s\n", util.NewFullPath(filerBucketsPath, *bucketName))
+
+		if currentVersioningEnabled {
+			fmt.Fprintf(writer, "Versioning: Enabled\n")
+		} else {
+			fmt.Fprintf(writer, "Versioning: Disabled\n")
+		}
+
+		if currentLockEnabled {
+			fmt.Fprintf(writer, "Object Lock: Enabled\n")
+		} else {
+			fmt.Fprintf(writer, "Object Lock: Disabled\n")
+			fmt.Fprintf(writer, "\nTo enable Object Lock, run:\n")
+			fmt.Fprintf(writer, "  s3.bucket.lock -name %s -enable\n", *bucketName)
+		}
+
+		return nil
+	})
+
+	return err
+}

weed/shell/command_s3_bucket_owner.go

@@ -0,0 +1,154 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketOwner{})
+}
+
+type commandS3BucketOwner struct {
+}
+
+func (c *commandS3BucketOwner) Name() string {
+	return "s3.bucket.owner"
+}
+
+func (c *commandS3BucketOwner) Help() string {
+	return `view or change the owner of an S3 bucket
+
+	Example:
+		# View the current owner of a bucket
+		s3.bucket.owner -name <bucket_name>
+
+		# Set or change the owner of a bucket
+		s3.bucket.owner -name <bucket_name> -owner <identity_name>
+
+		# Remove the owner (make bucket admin-only)
+		s3.bucket.owner -name <bucket_name> -delete
+
+	The owner identity determines which S3 user can access the bucket.
+	Non-admin users can only access buckets they own. Admin users can
+	access all buckets regardless of ownership.
+
+	The -owner value should match the identity name configured in your
+	S3 IAM system (the "name" field in s3.json identities configuration).
+`
+}
+
+func (c *commandS3BucketOwner) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketOwner) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	bucketOwner := bucketCommand.String("owner", "", "new bucket owner identity name")
+	deleteOwner := bucketCommand.Bool("delete", false, "remove the bucket owner (make admin-only)")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	// Trim whitespace from owner
+	owner := strings.TrimSpace(*bucketOwner)
+
+	// Validate flags: can't use both -owner and -delete
+	if owner != "" && *deleteOwner {
+		return fmt.Errorf("cannot use both -owner and -delete flags together")
+	}
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		// Look up the bucket entry
+		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: filerBucketsPath,
+			Name:      *bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("lookup bucket %s: %w", *bucketName, err)
+		}
+
+		entry := lookupResp.Entry
+
+		// If -owner is provided, set the owner
+		if owner != "" {
+			if entry.Extended == nil {
+				entry.Extended = make(map[string][]byte)
+			}
+			entry.Extended[s3_constants.AmzIdentityId] = []byte(owner)
+			// The S3 API derives the owning account from this identity when no
+			// account id is recorded; a leftover one would keep the old owner.
+			delete(entry.Extended, s3_constants.ExtAmzOwnerKey)
+			fmt.Fprintf(writer, "Setting owner of bucket %s to: %s\n", *bucketName, owner)
+
+			// Update the entry
+			if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+				Directory: filerBucketsPath,
+				Entry:     entry,
+			}); err != nil {
+				return fmt.Errorf("failed to update bucket: %w", err)
+			}
+
+			fmt.Fprintf(writer, "Bucket owner updated successfully.\n")
+			return nil
+		}
+
+		// If -delete is provided, remove the owner
+		if *deleteOwner {
+			if entry.Extended != nil {
+				delete(entry.Extended, s3_constants.AmzIdentityId)
+				delete(entry.Extended, s3_constants.ExtAmzOwnerKey)
+			}
+			fmt.Fprintf(writer, "Removing owner from bucket %s\n", *bucketName)
+
+			// Update the entry
+			if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+				Directory: filerBucketsPath,
+				Entry:     entry,
+			}); err != nil {
+				return fmt.Errorf("failed to update bucket: %w", err)
+			}
+
+			fmt.Fprintf(writer, "Bucket owner removed. Bucket is now admin-only.\n")
+			return nil
+		}
+
+		// Display current owner (no flags provided)
+		fmt.Fprintf(writer, "Bucket: %s\n", *bucketName)
+		fmt.Fprintf(writer, "Path: %s\n", util.NewFullPath(filerBucketsPath, *bucketName))
+
+		if entry.Extended != nil {
+			if ownerBytes, ok := entry.Extended[s3_constants.AmzIdentityId]; ok && len(ownerBytes) > 0 {
+				fmt.Fprintf(writer, "Owner: %s\n", string(ownerBytes))
+			} else {
+				fmt.Fprintf(writer, "Owner: (none - admin access only)\n")
+			}
+		} else {
+			fmt.Fprintf(writer, "Owner: (none - admin access only)\n")
+		}
+
+		return nil
+	})
+
+	return err
+}

weed/shell/command_s3_bucket_quota.go

@@ -0,0 +1,120 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketQuota{})
+}
+
+type commandS3BucketQuota struct {
+}
+
+func (c *commandS3BucketQuota) Name() string {
+	return "s3.bucket.quota"
+}
+
+func (c *commandS3BucketQuota) Help() string {
+	return `set/remove/enable/disable quota for a bucket
+
+	Example:
+		s3.bucket.quota -name=<bucket_name> -op=set -sizeMB=1024
+
+	Removing or disabling the quota also clears the read-only flag that
+	s3.bucket.quota.enforce may have set on the bucket.
+`
+}
+
+func (c *commandS3BucketQuota) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketQuota) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	operationName := bucketCommand.String("op", "set", "operation name [set|get|remove|enable|disable]")
+	sizeMB := bucketCommand.Int64("sizeMB", 0, "bucket quota size in MiB")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+
+		ctx := context.Background()
+
+		resp, err := client.GetFilerConfiguration(ctx, &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		lookupResp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
+			Directory: filerBucketsPath,
+			Name:      *bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("did not find bucket %s: %v", *bucketName, err)
+		}
+		bucketEntry := lookupResp.Entry
+
+		clearReadOnly := false
+		switch *operationName {
+		case "set":
+			bucketEntry.Quota = *sizeMB * 1024 * 1024
+		case "get":
+			fmt.Fprintf(writer, "bucket quota: %dMiB \n", bucketEntry.Quota/1024/1024)
+			return nil
+		case "remove":
+			bucketEntry.Quota = 0
+			clearReadOnly = true
+		case "enable":
+			if bucketEntry.Quota < 0 {
+				bucketEntry.Quota = -bucketEntry.Quota
+			}
+		case "disable":
+			if bucketEntry.Quota > 0 {
+				bucketEntry.Quota = -bucketEntry.Quota
+			}
+			clearReadOnly = true
+		}
+
+		if err := filer_pb.UpdateEntry(context.Background(), client, &filer_pb.UpdateEntryRequest{
+			Directory: filerBucketsPath,
+			Entry:     bucketEntry,
+		}); err != nil {
+			return err
+		}
+
+		println("updated quota for bucket", *bucketName)
+
+		if clearReadOnly {
+			// with the quota removed or disabled, s3.bucket.quota.enforce can no
+			// longer clear a read-only flag it turned on, so lift it here
+			cleared, err := filer.ClearBucketReadOnly(ctx, client, filerBucketsPath, *bucketName)
+			if err != nil {
+				return err
+			}
+			if cleared {
+				fmt.Fprintf(writer, "cleared read-only for bucket %s\n", *bucketName)
+			}
+		}
+
+		return nil
+
+	})
+
+	return err
+
+}

weed/shell/command_s3_bucket_quota_check.go

@@ -0,0 +1,122 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketQuotaEnforce{})
+}
+
+type commandS3BucketQuotaEnforce struct {
+}
+
+func (c *commandS3BucketQuotaEnforce) Name() string {
+	return "s3.bucket.quota.enforce"
+}
+
+func (c *commandS3BucketQuotaEnforce) Help() string {
+	return `check quota for all buckets, make the bucket read only if over the limit
+
+	Example:
+		s3.bucket.quota.enforce -apply
+`
+}
+
+func (c *commandS3BucketQuotaEnforce) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	applyQuotaLimit := bucketCommand.Bool("apply", false, "actually change the buckets readonly attribute")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+	infoAboutSimulationMode(writer, *applyQuotaLimit, "-apply")
+
+	// collect collection information
+	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
+	if err != nil {
+		return err
+	}
+	collectionInfos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topologyInfo, collectionInfos)
+
+	// read buckets path
+	var filerBucketsPath string
+	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
+	if err != nil {
+		return fmt.Errorf("read buckets: %w", err)
+	}
+
+	// read existing filer configuration
+	fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient)
+	if err != nil {
+		return err
+	}
+
+	// process each bucket
+	hasConfChanges := false
+	err = filer_pb.List(context.Background(), commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		if !entry.IsDirectory {
+			return nil
+		}
+		collection := getCollectionName(commandEnv, entry.Name)
+		var collectionSize float64
+		if collectionInfo, found := collectionInfos[collection]; found {
+			collectionSize = collectionInfo.LogicalSize()
+		}
+		if c.processEachBucket(fc, filerBucketsPath, entry, writer, collectionSize) {
+			hasConfChanges = true
+		}
+		return nil
+	}, "", false, math.MaxUint32)
+	if err != nil {
+		return fmt.Errorf("list buckets under %v: %w", filerBucketsPath, err)
+	}
+
+	// apply the configuration changes
+	if hasConfChanges && *applyQuotaLimit {
+
+		var buf2 bytes.Buffer
+		fc.ToText(&buf2)
+
+		if err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			return filer.SaveInsideFiler(context.Background(), client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName, buf2.Bytes())
+		}); err != nil && err != filer_pb.ErrNotFound {
+			return err
+		}
+	}
+
+	return err
+
+}
+
+func (c *commandS3BucketQuotaEnforce) processEachBucket(fc *filer.FilerConf, filerBucketsPath string, entry *filer_pb.Entry, writer io.Writer, collectionSize float64) (hasConfChanges bool) {
+
+	locPrefix := filerBucketsPath + "/" + entry.Name + "/"
+
+	readOnly, hasConfChanges := fc.ApplyBucketQuotaReadOnly(locPrefix, collectionSize, float64(entry.Quota))
+	if hasConfChanges {
+		fmt.Fprintf(writer, "  %s\tsize:%.0f", entry.Name, collectionSize)
+		fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, collectionSize*100/float64(entry.Quota))
+		fmt.Fprintln(writer)
+		if readOnly {
+			fmt.Fprintf(writer, "    changing bucket %s to read only!\n", entry.Name)
+		} else {
+			fmt.Fprintf(writer, "    changing bucket %s to writable.\n", entry.Name)
+		}
+	}
+
+	return
+}

weed/shell/command_s3_bucket_versioning.go

@@ -0,0 +1,167 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3BucketVersioning{})
+}
+
+type commandS3BucketVersioning struct {
+}
+
+func (c *commandS3BucketVersioning) Name() string {
+	return "s3.bucket.versioning"
+}
+
+func (c *commandS3BucketVersioning) Help() string {
+	return `view or update S3 bucket versioning configuration
+
+	Example:
+		# View the current versioning status
+		s3.bucket.versioning -name <bucket_name>
+
+		# Enable versioning
+		s3.bucket.versioning -name <bucket_name> -enable
+
+		# Suspend versioning
+		s3.bucket.versioning -name <bucket_name> -suspend
+
+		# Set versioning status explicitly (Enabled or Suspended)
+		s3.bucket.versioning -name <bucket_name> -status Enabled
+
+	Object Lock requires versioning to be enabled and prevents suspending it.
+`
+}
+
+func (c *commandS3BucketVersioning) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3BucketVersioning) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	bucketName := bucketCommand.String("name", "", "bucket name")
+	statusFlag := bucketCommand.String("status", "", "versioning status: Enabled or Suspended")
+	enable := bucketCommand.Bool("enable", false, "enable versioning on the bucket")
+	suspend := bucketCommand.Bool("suspend", false, "suspend versioning on the bucket")
+	if err = bucketCommand.Parse(args); err != nil {
+		return err
+	}
+
+	if *bucketName == "" {
+		return fmt.Errorf("empty bucket name")
+	}
+
+	desiredStatus, err := normalizeVersioningStatus(*statusFlag, *enable, *suspend)
+	if err != nil {
+		return err
+	}
+
+	err = commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return fmt.Errorf("get filer configuration: %w", err)
+		}
+		filerBucketsPath := resp.DirBuckets
+
+		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
+			Directory: filerBucketsPath,
+			Name:      *bucketName,
+		})
+		if err != nil {
+			return fmt.Errorf("lookup bucket %s: %w", *bucketName, err)
+		}
+
+		entry := lookupResp.Entry
+		currentStatus, lockEnabled := getBucketVersioningState(entry)
+
+		if desiredStatus == "" {
+			if lockEnabled {
+				currentStatus = s3_constants.VersioningEnabled
+			}
+			fmt.Fprintf(writer, "Bucket: %s\n", *bucketName)
+			switch currentStatus {
+			case s3_constants.VersioningEnabled:
+				fmt.Fprintf(writer, "Versioning: Enabled\n")
+			case s3_constants.VersioningSuspended:
+				fmt.Fprintf(writer, "Versioning: Suspended\n")
+			case "":
+				fmt.Fprintf(writer, "Versioning: Unconfigured\n")
+			default:
+				fmt.Fprintf(writer, "Versioning: %s\n", currentStatus)
+			}
+			if lockEnabled {
+				fmt.Fprintf(writer, "Object Lock: Enabled\n")
+			}
+			return nil
+		}
+
+		if lockEnabled && desiredStatus == s3_constants.VersioningSuspended {
+			return fmt.Errorf("cannot suspend versioning on bucket %s: Object Lock is enabled", *bucketName)
+		}
+
+		if entry.Extended == nil {
+			entry.Extended = make(map[string][]byte)
+		}
+		entry.Extended[s3_constants.ExtVersioningKey] = []byte(desiredStatus)
+
+		if _, err := client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
+			Directory: filerBucketsPath,
+			Entry:     entry,
+		}); err != nil {
+			return fmt.Errorf("failed to update bucket: %w", err)
+		}
+
+		fmt.Fprintf(writer, "Bucket %s versioning set to %s\n", *bucketName, desiredStatus)
+		return nil
+	})
+
+	return err
+}
+
+func normalizeVersioningStatus(statusFlag string, enable bool, suspend bool) (string, error) {
+	if enable && suspend {
+		return "", fmt.Errorf("only one of -enable or -suspend can be set")
+	}
+	if (enable || suspend) && statusFlag != "" {
+		return "", fmt.Errorf("use either -status or -enable/-suspend, not both")
+	}
+	if enable {
+		return s3_constants.VersioningEnabled, nil
+	}
+	if suspend {
+		return s3_constants.VersioningSuspended, nil
+	}
+	if statusFlag == "" {
+		return "", nil
+	}
+	switch strings.ToLower(strings.TrimSpace(statusFlag)) {
+	case "enabled":
+		return s3_constants.VersioningEnabled, nil
+	case "suspended":
+		return s3_constants.VersioningSuspended, nil
+	default:
+		return "", fmt.Errorf("invalid versioning status %q: must be Enabled or Suspended", statusFlag)
+	}
+}
+
+func getBucketVersioningState(entry *filer_pb.Entry) (status string, lockEnabled bool) {
+	if entry.Extended == nil {
+		return "", false
+	}
+	if versioning, ok := entry.Extended[s3_constants.ExtVersioningKey]; ok {
+		status = string(versioning)
+	}
+	if lockStatus, ok := entry.Extended[s3_constants.ExtObjectLockEnabledKey]; ok {
+		lockEnabled = string(lockStatus) == s3_constants.ObjectLockEnabled
+	}
+	return status, lockEnabled
+}

weed/shell/command_s3_circuitbreaker.go

@@ -0,0 +1,364 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+var LoadConfig = loadConfig
+
+func init() {
+	Commands = append(Commands, &commandS3CircuitBreaker{})
+}
+
+type commandS3CircuitBreaker struct {
+}
+
+func (c *commandS3CircuitBreaker) Name() string {
+	return "s3.circuitBreaker"
+}
+
+func (c *commandS3CircuitBreaker) Help() string {
+	return `configure and apply s3 circuit breaker options for each bucket
+
+	# examples
+	# add circuit breaker config for global
+	s3.circuitBreaker -global -type count -actions Read,Write -values 500,200 -apply
+
+	# disable global config
+	s3.circuitBreaker -global -disable -apply
+
+	# add circuit breaker config for buckets x,y,z
+	s3.circuitBreaker -buckets x,y,z -type count -actions Read,Write -values 200,100 -apply
+
+	# disable circuit breaker config of x
+	s3.circuitBreaker -buckets x -disable -apply
+
+	# delete circuit breaker config of x
+	s3.circuitBreaker -buckets x -delete -apply
+
+	# clear all circuit breaker config
+	s3.circuitBreaker -delete -apply
+	`
+}
+
+func (c *commandS3CircuitBreaker) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	dir := s3_constants.CircuitBreakerConfigDir
+	file := s3_constants.CircuitBreakerConfigFile
+
+	s3CircuitBreakerCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	buckets := s3CircuitBreakerCommand.String("buckets", "", "the bucket name(s) to configure, eg: -buckets x,y,z")
+	global := s3CircuitBreakerCommand.Bool("global", false, "configure global circuit breaker")
+
+	actions := s3CircuitBreakerCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin")
+	limitType := s3CircuitBreakerCommand.String("type", "", "'Count' or 'MB'; Count represents the number of simultaneous requests, and MB represents the content size of all simultaneous requests")
+	values := s3CircuitBreakerCommand.String("values", "", "comma separated values")
+
+	disabled := s3CircuitBreakerCommand.Bool("disable", false, "disable global or buckets circuit breaker")
+	deleted := s3CircuitBreakerCommand.Bool("delete", false, "delete circuit breaker config")
+
+	apply := s3CircuitBreakerCommand.Bool("apply", false, "update and apply current configuration")
+
+	if err = s3CircuitBreakerCommand.Parse(args); err != nil {
+		return nil
+
+	}
+
+	var buf bytes.Buffer
+	err = LoadConfig(commandEnv, dir, file, &buf)
+	if err != nil {
+		return err
+	}
+
+	cbCfg := &s3_pb.S3CircuitBreakerConfig{
+		Buckets: make(map[string]*s3_pb.S3CircuitBreakerOptions),
+	}
+	if buf.Len() > 0 {
+		if err = filer.ParseS3ConfigurationFromBytes(buf.Bytes(), cbCfg); err != nil {
+			return err
+		}
+	}
+
+	if *deleted {
+		cmdBuckets, cmdActions, _, err := c.initActionsAndValues(buckets, actions, limitType, values, true)
+		if err != nil {
+			return err
+		}
+
+		if len(cmdBuckets) <= 0 && !*global {
+			if len(cmdActions) > 0 {
+				deleteGlobalActions(cbCfg, cmdActions, limitType)
+				if cbCfg.Buckets != nil {
+					var allBuckets []string
+					for bucket := range cbCfg.Buckets {
+						allBuckets = append(allBuckets, bucket)
+					}
+					deleteBucketsActions(allBuckets, cbCfg, cmdActions, limitType)
+				}
+			} else {
+				cbCfg.Global = nil
+				cbCfg.Buckets = nil
+			}
+		} else {
+			if len(cmdBuckets) > 0 {
+				deleteBucketsActions(cmdBuckets, cbCfg, cmdActions, limitType)
+			}
+			if *global {
+				deleteGlobalActions(cbCfg, cmdActions, nil)
+			}
+		}
+	} else {
+		cmdBuckets, cmdActions, cmdValues, err := c.initActionsAndValues(buckets, actions, limitType, values, *disabled)
+		if err != nil {
+			return err
+		}
+
+		if len(cmdActions) > 0 && len(*buckets) <= 0 && !*global {
+			return fmt.Errorf("one of -global and -buckets must be specified")
+		}
+
+		if len(*buckets) > 0 {
+			for _, bucket := range cmdBuckets {
+				var cbOptions *s3_pb.S3CircuitBreakerOptions
+				var exists bool
+				if cbOptions, exists = cbCfg.Buckets[bucket]; !exists {
+					cbOptions = &s3_pb.S3CircuitBreakerOptions{}
+					cbCfg.Buckets[bucket] = cbOptions
+				}
+				cbOptions.Enabled = !*disabled
+
+				if len(cmdActions) > 0 {
+					err = insertOrUpdateValues(cbOptions, cmdActions, cmdValues, limitType)
+					if err != nil {
+						return err
+					}
+				}
+
+				if len(cbOptions.Actions) <= 0 && !cbOptions.Enabled {
+					delete(cbCfg.Buckets, bucket)
+				}
+			}
+		}
+
+		if *global {
+			globalOptions := cbCfg.Global
+			if globalOptions == nil {
+				globalOptions = &s3_pb.S3CircuitBreakerOptions{Actions: make(map[string]int64, len(cmdActions))}
+				cbCfg.Global = globalOptions
+			}
+			globalOptions.Enabled = !*disabled
+
+			if len(cmdActions) > 0 {
+				err = insertOrUpdateValues(globalOptions, cmdActions, cmdValues, limitType)
+				if err != nil {
+					return err
+				}
+			}
+
+			if len(globalOptions.Actions) <= 0 && !globalOptions.Enabled {
+				cbCfg.Global = nil
+			}
+		}
+	}
+
+	buf.Reset()
+	err = filer.ProtoToText(&buf, cbCfg)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprint(writer, buf.String())
+	fmt.Fprintln(writer)
+
+	if *apply {
+		if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+			return filer.SaveInsideFiler(context.Background(), client, dir, file, buf.Bytes())
+		}); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func loadConfig(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error {
+	if err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		return filer.ReadEntry(commandEnv.MasterClient, client, dir, file, buf)
+	}); err != nil && err != filer_pb.ErrNotFound {
+		return err
+	}
+	return nil
+}
+
+func insertOrUpdateValues(cbOptions *s3_pb.S3CircuitBreakerOptions, cmdActions []string, cmdValues []int64, limitType *string) error {
+	if len(*limitType) == 0 {
+		return fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed")
+	}
+
+	if cbOptions.Actions == nil {
+		cbOptions.Actions = make(map[string]int64, len(cmdActions))
+	}
+
+	if len(cmdValues) > 0 {
+		for i, action := range cmdActions {
+			cbOptions.Actions[s3_constants.Concat(action, *limitType)] = cmdValues[i]
+		}
+	}
+	return nil
+}
+
+func deleteBucketsActions(cmdBuckets []string, cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) {
+	if cbCfg.Buckets == nil {
+		return
+	}
+
+	if len(cmdActions) == 0 {
+		for _, bucket := range cmdBuckets {
+			delete(cbCfg.Buckets, bucket)
+		}
+	} else {
+		for _, bucket := range cmdBuckets {
+			if cbOption, ok := cbCfg.Buckets[bucket]; ok {
+				if len(cmdActions) > 0 && cbOption.Actions != nil {
+					for _, action := range cmdActions {
+						delete(cbOption.Actions, s3_constants.Concat(action, *limitType))
+					}
+				}
+
+				if len(cbOption.Actions) == 0 && !cbOption.Enabled {
+					delete(cbCfg.Buckets, bucket)
+				}
+			}
+		}
+	}
+
+	if len(cbCfg.Buckets) == 0 {
+		cbCfg.Buckets = nil
+	}
+}
+
+func deleteGlobalActions(cbCfg *s3_pb.S3CircuitBreakerConfig, cmdActions []string, limitType *string) {
+	globalOptions := cbCfg.Global
+	if globalOptions == nil {
+		return
+	}
+
+	if len(cmdActions) == 0 && globalOptions.Actions != nil {
+		globalOptions.Actions = nil
+		return
+	} else {
+		for _, action := range cmdActions {
+			delete(globalOptions.Actions, s3_constants.Concat(action, *limitType))
+		}
+	}
+
+	if len(globalOptions.Actions) == 0 && !globalOptions.Enabled {
+		cbCfg.Global = nil
+	}
+}
+
+func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitType, values *string, parseValues bool) (cmdBuckets, cmdActions []string, cmdValues []int64, err error) {
+	if len(*buckets) > 0 {
+		cmdBuckets = strings.Split(*buckets, ",")
+	}
+
+	if len(*actions) > 0 {
+		cmdActions = strings.Split(*actions, ",")
+
+		//check action valid
+		for _, action := range cmdActions {
+			var found bool
+			for _, allowedAction := range s3_constants.AllowedActions {
+				if allowedAction == action {
+					found = true
+				}
+			}
+			if !found {
+				return nil, nil, nil, fmt.Errorf("value(%s) of flag[-action] not valid, allowed actions: %v", *actions, s3_constants.AllowedActions)
+			}
+		}
+	}
+
+	if !parseValues {
+		if len(cmdActions) < 0 {
+			for _, action := range s3_constants.AllowedActions {
+				cmdActions = append(cmdActions, action)
+			}
+		}
+
+		if len(*limitType) > 0 {
+			switch *limitType {
+			case s3_constants.LimitTypeCount:
+				elements := strings.Split(*values, ",")
+				if len(cmdActions) != len(elements) {
+					if len(elements) != 1 || len(elements) == 0 {
+						return nil, nil, nil, fmt.Errorf("count of flag[-actions] and flag[-counts] not equal")
+					}
+					v, err := strconv.Atoi(elements[0])
+					if err != nil {
+						return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)")
+					}
+					for range cmdActions {
+						cmdValues = append(cmdValues, int64(v))
+					}
+				} else {
+					for _, value := range elements {
+						v, err := strconv.Atoi(value)
+						if err != nil {
+							return nil, nil, nil, fmt.Errorf("value of -values must be a legal number(s)")
+						}
+						cmdValues = append(cmdValues, int64(v))
+					}
+				}
+			case s3_constants.LimitTypeBytes:
+				elements := strings.Split(*values, ",")
+				if len(cmdActions) != len(elements) {
+					if len(elements) != 1 || len(elements) == 0 {
+						return nil, nil, nil, fmt.Errorf("values count of -actions and -values not equal")
+					}
+					v, err := parseMBToBytes(elements[0])
+					if err != nil {
+						return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)")
+					}
+					for range cmdActions {
+						cmdValues = append(cmdValues, v)
+					}
+				} else {
+					for _, value := range elements {
+						v, err := parseMBToBytes(value)
+						if err != nil {
+							return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)")
+						}
+						cmdValues = append(cmdValues, v)
+					}
+				}
+			default:
+				return nil, nil, nil, fmt.Errorf("type not valid, only 'count' and 'bytes' are allowed")
+			}
+		} else {
+			*limitType = ""
+		}
+	}
+	return cmdBuckets, cmdActions, cmdValues, nil
+}
+
+func parseMBToBytes(valStr string) (int64, error) {
+	v, err := strconv.Atoi(valStr)
+	v *= 1024 * 1024
+	return int64(v), err
+}

weed/shell/command_s3_circuitbreaker_test.go

@@ -0,0 +1,292 @@
+package shell
+
+import (
+	"bytes"
+	"encoding/json"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+type Case struct {
+	args   []string
+	result string
+}
+
+var (
+	TestCases = []*Case{
+		//add circuit breaker config for global
+		{
+			args: strings.Split("-global -type Count -actions Read,Write -values 500,200", " "),
+			result: `{
+				"global": {
+					"enabled": true,
+					"actions": {
+						"Read:Count": "500",
+						"Write:Count": "200"
+					}
+				}
+			}`,
+		},
+
+		//disable global config
+		{
+			args: strings.Split("-global -disable", " "),
+			result: `{
+				"global": {
+					"actions": {
+						"Read:Count": "500",
+						"Write:Count": "200"
+					}
+				}
+			}`,
+		},
+
+		//add circuit breaker config for buckets x,y,z
+		{
+			args: strings.Split("-buckets x,y,z -type Count -actions Read,Write -values 200,100", " "),
+			result: `{
+				"global": {
+					"actions": {
+						"Read:Count": "500",
+						"Write:Count": "200"
+					}
+				},
+				"buckets": {
+					"x": {
+						"enabled": true,
+						"actions": {
+							"Read:Count": "200",
+							"Write:Count": "100"
+						}
+					},
+					"y": {
+						"enabled": true,
+						"actions": {
+							"Read:Count": "200",
+							"Write:Count": "100"
+						}
+					},
+					"z": {
+						"enabled": true,
+						"actions": {
+							"Read:Count": "200",
+							"Write:Count": "100"
+						}
+					}
+				}
+			}`,
+		},
+
+		//disable circuit breaker config of x
+		{
+			args: strings.Split("-buckets x -disable", " "),
+			result: `{
+			  "global": {
+				"actions": {
+				  "Read:Count": "500",
+				  "Write:Count": "200"
+				}
+			  },
+			  "buckets": {
+				"x": {
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				},
+				"y": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				},
+				"z": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				}
+			  }
+			}`,
+		},
+
+		//delete circuit breaker config of x
+		{
+			args: strings.Split("-buckets x -delete", " "),
+			result: `{
+			  "global": {
+				"actions": {
+				  "Read:Count": "500",
+				  "Write:Count": "200"
+				}
+			  },
+			  "buckets": {
+				"y": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				},
+				"z": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				}
+			  }
+			}`,
+		},
+
+		//configure the circuit breaker for the size of the uploaded file for bucket x,y
+		{
+			args: strings.Split("-buckets x,y -type MB -actions Write -values 1024", " "),
+			result: `{
+			  "global": {
+				"actions": {
+				  "Read:Count": "500",
+				  "Write:Count": "200"
+				}
+			  },
+			  "buckets": {
+				"x": {
+				  "enabled": true,
+				  "actions": {
+					"Write:MB": "1073741824"
+				  }
+				},
+				"y": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100",
+					"Write:MB": "1073741824"
+				  }
+				},
+				"z": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				}
+			  }
+			}`,
+		},
+
+		//delete the circuit breaker configuration for the size of the uploaded file of bucket x,y
+		{
+			args: strings.Split("-buckets x,y -type MB -actions Write -delete", " "),
+			result: `{
+			  "global": {
+				"actions": {
+				  "Read:Count": "500",
+				  "Write:Count": "200"
+				}
+			  },
+			  "buckets": {
+				"x": {
+				  "enabled": true
+				},
+				"y": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				},
+				"z": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				}
+			  }
+			}`,
+		},
+
+		//enable global circuit breaker config (without -disable flag)
+		{
+			args: strings.Split("-global", " "),
+			result: `{
+			  "global": {
+				"enabled": true,
+				"actions": {
+				  "Read:Count": "500",
+				  "Write:Count": "200"
+				}
+			  },
+			  "buckets": {
+				"x": {
+				  "enabled": true
+				},
+				"y": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				},
+				"z": {
+				  "enabled": true,
+				  "actions": {
+					"Read:Count": "200",
+					"Write:Count": "100"
+				  }
+				}
+			  }
+			}`,
+		},
+
+		//clear all circuit breaker config
+		{
+			args: strings.Split("-delete", " "),
+			result: `{
+			
+			}`,
+		},
+	}
+)
+
+func TestCircuitBreakerShell(t *testing.T) {
+	var writeBuf bytes.Buffer
+	cmd := &commandS3CircuitBreaker{}
+	LoadConfig = func(commandEnv *CommandEnv, dir string, file string, buf *bytes.Buffer) error {
+		_, err := buf.Write(writeBuf.Bytes())
+		if err != nil {
+			return err
+		}
+		writeBuf.Reset()
+		return nil
+	}
+
+	for i, tc := range TestCases {
+		err := cmd.Do(tc.args, nil, &writeBuf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if i != 0 {
+			result := writeBuf.String()
+
+			actual := make(map[string]interface{})
+			err := json.Unmarshal([]byte(result), &actual)
+			if err != nil {
+				t.Error(err)
+			}
+
+			expect := make(map[string]interface{})
+			err = json.Unmarshal([]byte(result), &expect)
+			if err != nil {
+				t.Error(err)
+			}
+			if !reflect.DeepEqual(actual, expect) {
+				t.Fatal("result of s3 circuit breaker shell command is unexpected!")
+			}
+		}
+	}
+}

weed/shell/command_s3_clean_uploads.go

@@ -0,0 +1,106 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3CleanUploads{})
+}
+
+type commandS3CleanUploads struct{}
+
+func (c *commandS3CleanUploads) Name() string {
+	return "s3.clean.uploads"
+}
+
+func (c *commandS3CleanUploads) Help() string {
+	return `clean up stale multipart uploads
+
+	Example:
+		s3.clean.uploads -timeAgo 1.5h
+
+`
+}
+
+func (c *commandS3CleanUploads) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	bucketCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	uploadedTimeAgo := bucketCommand.Duration("timeAgo", 24*time.Hour, "created time before now. \"1.5h\" or \"2h45m\". Valid time units are \"m\", \"h\"")
+	if err = bucketCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	signingKey := util.GetViper().GetString("jwt.filer_signing.key")
+
+	var filerBucketsPath string
+	filerBucketsPath, err = readFilerBucketsPath(commandEnv)
+	if err != nil {
+		return fmt.Errorf("read buckets: %w", err)
+	}
+
+	var buckets []string
+	err = filer_pb.List(context.Background(), commandEnv, filerBucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
+		buckets = append(buckets, entry.Name)
+		return nil
+	}, "", false, math.MaxUint32)
+	if err != nil {
+		return fmt.Errorf("list buckets under %v: %w", filerBucketsPath, err)
+	}
+
+	for _, bucket := range buckets {
+		if err := c.cleanupUploads(commandEnv, writer, filerBucketsPath, bucket, *uploadedTimeAgo, signingKey); err != nil {
+			fmt.Fprintf(writer, "failed cleanup uploads for bucket %s: %v", bucket, err)
+		}
+	}
+
+	return err
+}
+
+func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration, signingKey string) error {
+	uploadsDir := filerBucketsPath + "/" + bucket + "/" + s3_constants.MultipartUploadsFolder
+	var staleUploads []string
+	now := time.Now()
+	err := filer_pb.List(context.Background(), commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
+		ctime := time.Unix(entry.Attributes.Crtime, 0)
+		if ctime.Add(timeAgo).Before(now) {
+			staleUploads = append(staleUploads, entry.Name)
+		}
+		return nil
+	}, "", false, math.MaxUint32)
+	if err != nil {
+		return fmt.Errorf("list uploads under %v: %w", uploadsDir, err)
+	}
+
+	var encodedJwt security.EncodedJwt
+	if signingKey != "" {
+		encodedJwt = security.GenJwtForFilerServer(security.SigningKey(signingKey), 15*60)
+	}
+
+	for _, staleUpload := range staleUploads {
+		deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload)
+		fmt.Fprintf(writer, "purge %s\n", deleteUrl)
+
+		err = util_http.Delete(deleteUrl, string(encodedJwt))
+		if err != nil && err.Error() != "" {
+			return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err)
+		}
+	}
+
+	return nil
+}

weed/shell/command_s3_config_show.go

@@ -0,0 +1,109 @@
+package shell
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"text/tabwriter"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ConfigShow{})
+}
+
+type commandS3ConfigShow struct {
+}
+
+func (c *commandS3ConfigShow) Name() string {
+	return "s3.config.show"
+}
+
+func (c *commandS3ConfigShow) Help() string {
+	return `show a summary of the current S3 IAM configuration
+
+	s3.config.show
+
+	Displays counts and a brief listing of users, policies, service accounts,
+	and groups. Use s3.iam.export for the full JSON dump.
+`
+}
+
+func (c *commandS3ConfigShow) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ConfigShow) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.Configuration
+		if cfg == nil {
+			fmt.Fprintln(writer, "No S3 IAM configuration found.")
+			return nil
+		}
+
+		fmt.Fprintf(writer, "S3 IAM Configuration Summary\n")
+		fmt.Fprintf(writer, "============================\n\n")
+
+		// Users
+		fmt.Fprintf(writer, "Users: %d\n", len(cfg.Identities))
+		if len(cfg.Identities) > 0 {
+			tw := tabwriter.NewWriter(writer, 0, 4, 2, ' ', 0)
+			fmt.Fprintln(tw, "  NAME\tSTATUS\tSOURCE\tKEYS\tPOLICIES")
+			for _, id := range cfg.Identities {
+				status := "enabled"
+				if id.Disabled {
+					status = "disabled"
+				}
+				source := "dynamic"
+				if id.IsStatic {
+					source = "static"
+				}
+				policies := "-"
+				if len(id.PolicyNames) > 0 {
+					policies = joinMax(id.PolicyNames, 3)
+				}
+				fmt.Fprintf(tw, "  %s\t%s\t%s\t%d\t%s\n",
+					id.Name, status, source, len(id.Credentials), policies)
+			}
+			tw.Flush()
+		}
+		fmt.Fprintln(writer)
+
+		// Policies
+		fmt.Fprintf(writer, "Policies: %d\n", len(cfg.Policies))
+		if len(cfg.Policies) > 0 {
+			for _, p := range cfg.Policies {
+				fmt.Fprintf(writer, "  %s\n", p.Name)
+			}
+		}
+		fmt.Fprintln(writer)
+
+		// Service Accounts
+		fmt.Fprintf(writer, "Service Accounts: %d\n", len(cfg.ServiceAccounts))
+		if len(cfg.ServiceAccounts) > 0 {
+			for _, sa := range cfg.ServiceAccounts {
+				status := "enabled"
+				if sa.Disabled {
+					status = "disabled"
+				}
+				fmt.Fprintf(writer, "  %s (parent: %s, %s)\n", sa.Id, sa.ParentUser, status)
+			}
+		}
+		fmt.Fprintln(writer)
+
+		// Groups
+		fmt.Fprintf(writer, "Groups: %d\n", len(cfg.Groups))
+		if len(cfg.Groups) > 0 {
+			for _, g := range cfg.Groups {
+				fmt.Fprintf(writer, "  %s (%d members)\n", g.Name, len(g.Members))
+			}
+		}
+
+		return nil
+	})
+}

weed/shell/command_s3_configure.go

@@ -0,0 +1,279 @@
+package shell
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3Configure{})
+}
+
+type commandS3Configure struct {
+}
+
+func (c *commandS3Configure) Name() string {
+	return "s3.configure"
+}
+
+func (c *commandS3Configure) Help() string {
+	return `configure and apply s3 options for each bucket
+
+	# see the current configuration file content
+	s3.configure
+
+	# create a new identity with account information
+	s3.configure -user=username -actions=Read,Write,List,Tagging -buckets=bucket-name -policies=policy1,policy2 -access_key=key -secret_key=secret -account_id=id -account_display_name=name -account_email=email@example.com -apply
+	`
+}
+
+func (c *commandS3Configure) HasTag(tag CommandTag) bool {
+	return tag == Hidden
+}
+
+func (c *commandS3Configure) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	s3ConfigureCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	actions := s3ConfigureCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin")
+	user := s3ConfigureCommand.String("user", "", "user name")
+	buckets := s3ConfigureCommand.String("buckets", "", "bucket name")
+	accessKey := s3ConfigureCommand.String("access_key", "", "specify the access key")
+	secretKey := s3ConfigureCommand.String("secret_key", "", "specify the secret key")
+	accountId := s3ConfigureCommand.String("account_id", "", "specify the account id")
+	accountDisplayName := s3ConfigureCommand.String("account_display_name", "", "specify the account display name")
+	accountEmail := s3ConfigureCommand.String("account_email", "", "specify the account email address")
+	policies := s3ConfigureCommand.String("policies", "", "comma separated policy names")
+	isDelete := s3ConfigureCommand.Bool("delete", false, "delete users, actions, access keys or policies")
+	apply := s3ConfigureCommand.Bool("apply", false, "update and apply s3 configuration")
+	if err = s3ConfigureCommand.Parse(args); err != nil {
+		return nil
+	}
+
+	// Case 1: List configuration (no user specified)
+	if *user == "" {
+		return c.listConfiguration(commandEnv, writer)
+	}
+
+	// Case 2: Modify specific user
+	var identity *iam_pb.Identity
+	var isNewUser bool
+
+	err = commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+
+		// Try to get existing user
+		resp, getErr := client.GetUser(ctx, &iam_pb.GetUserRequest{
+			Username: *user,
+		})
+
+		if getErr == nil {
+			identity = resp.Identity
+			if identity == nil {
+				// Should not happen if err is nil, but handle defensively
+				isNewUser = true
+				identity = &iam_pb.Identity{Name: *user}
+			}
+		} else {
+			st, ok := status.FromError(getErr)
+			if ok && st.Code() == codes.NotFound {
+				isNewUser = true
+				identity = &iam_pb.Identity{
+					Name:        *user,
+					Credentials: []*iam_pb.Credential{},
+					Actions:     []string{},
+					PolicyNames: []string{},
+				}
+			} else {
+				return fmt.Errorf("failed to get user %s: %v", *user, getErr)
+			}
+		}
+
+		// Apply changes to identity object
+		if err := c.applyChanges(identity, isNewUser, actions, buckets, accessKey, secretKey, policies, isDelete, accountId, accountDisplayName, accountEmail); err != nil {
+			return err
+		}
+
+		// Print changes (Simulation)
+		var buf bytes.Buffer
+		filer.ProtoToText(&buf, identity)
+		fmt.Fprint(writer, buf.String())
+		fmt.Fprintln(writer)
+
+		if !*apply {
+			infoAboutSimulationMode(writer, *apply, "-apply")
+			return nil
+		}
+
+		// Apply changes
+		if *isDelete && *actions == "" && *accessKey == "" && *buckets == "" && *policies == "" {
+			// Delete User
+			_, err := client.DeleteUser(ctx, &iam_pb.DeleteUserRequest{Username: *user})
+			return err
+		} else {
+			// Create or Update User
+			if isNewUser {
+				_, err := client.CreateUser(ctx, &iam_pb.CreateUserRequest{Identity: identity})
+				return err
+			} else {
+				_, err := client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{Username: *user, Identity: identity})
+				return err
+			}
+		}
+	})
+
+	return err
+}
+
+func (c *commandS3Configure) listConfiguration(commandEnv *CommandEnv, writer io.Writer) error {
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		var buf bytes.Buffer
+		filer.ProtoToText(&buf, resp.Configuration)
+		fmt.Fprint(writer, buf.String())
+		fmt.Fprintln(writer)
+		return nil
+	})
+}
+
+func (c *commandS3Configure) applyChanges(identity *iam_pb.Identity, isNewUser bool, actions, buckets, accessKey, secretKey, policies *string, isDelete *bool, accountId, accountDisplayName, accountEmail *string) error {
+
+	// Helper to update account info
+	if *accountId != "" || *accountDisplayName != "" || *accountEmail != "" {
+		if identity.Account == nil {
+			identity.Account = &iam_pb.Account{}
+		}
+		if *accountId != "" {
+			identity.Account.Id = *accountId
+		}
+		if *accountDisplayName != "" {
+			identity.Account.DisplayName = *accountDisplayName
+		}
+		if *accountEmail != "" {
+			identity.Account.EmailAddress = *accountEmail
+		}
+	}
+
+	// Prepare lists
+	var cmdActions []string
+	if *actions != "" {
+		for _, action := range strings.Split(*actions, ",") {
+			if *buckets == "" {
+				cmdActions = append(cmdActions, action)
+			} else {
+				for _, bucket := range strings.Split(*buckets, ",") {
+					cmdActions = append(cmdActions, fmt.Sprintf("%s:%s", action, bucket))
+				}
+			}
+		}
+	}
+
+	var cmdPolicies []string
+	if *policies != "" {
+		for _, policy := range strings.Split(*policies, ",") {
+			if policy != "" {
+				cmdPolicies = append(cmdPolicies, policy)
+			}
+		}
+	}
+
+	if *isDelete {
+		// DELETE LOGIC
+
+		// Remove Actions
+		if len(cmdActions) > 0 {
+			identity.Actions = removeFromSlice(identity.Actions, cmdActions)
+		}
+
+		// Remove Credentials
+		if *accessKey != "" {
+			var keepCredentials []*iam_pb.Credential
+			for _, cred := range identity.Credentials {
+				if cred.AccessKey != *accessKey {
+					keepCredentials = append(keepCredentials, cred)
+				}
+			}
+			identity.Credentials = keepCredentials
+		}
+
+		// Remove Policies
+		if len(cmdPolicies) > 0 {
+			identity.PolicyNames = removeFromSlice(identity.PolicyNames, cmdPolicies)
+		}
+
+	} else {
+		// ADD/UPDATE LOGIC
+
+		// Add Actions
+		identity.Actions = addUniqueToSlice(identity.Actions, cmdActions)
+
+		// Add/Update Credentials
+		if *accessKey != "" && identity.Name != "anonymous" {
+			found := false
+			for _, cred := range identity.Credentials {
+				if cred.AccessKey == *accessKey {
+					found = true
+					if *secretKey != "" {
+						cred.SecretKey = *secretKey
+					}
+					break
+				}
+			}
+			if !found {
+				if *secretKey == "" {
+					return fmt.Errorf("secret_key is required when adding a new access_key")
+				}
+				identity.Credentials = append(identity.Credentials, &iam_pb.Credential{
+					AccessKey: *accessKey,
+					SecretKey: *secretKey,
+					Status:    iam.AccessKeyStatusActive,
+				})
+			}
+		}
+
+		// Add Policies
+		identity.PolicyNames = addUniqueToSlice(identity.PolicyNames, cmdPolicies)
+	}
+
+	return nil
+}
+
+// Helper to remove items from a slice
+func removeFromSlice(current []string, toRemove []string) []string {
+	removeSet := make(map[string]struct{}, len(toRemove))
+	for _, item := range toRemove {
+		removeSet[item] = struct{}{}
+	}
+	var result []string
+	for _, item := range current {
+		if _, found := removeSet[item]; !found {
+			result = append(result, item)
+		}
+	}
+	return result
+}
+
+// Helper to add unique items to a slice
+func addUniqueToSlice(current []string, toAdd []string) []string {
+	existingSet := make(map[string]struct{}, len(current))
+	for _, item := range current {
+		existingSet[item] = struct{}{}
+	}
+	for _, item := range toAdd {
+		if _, found := existingSet[item]; !found {
+			current = append(current, item)
+		}
+	}
+	return current
+}

weed/shell/command_s3_group_add_user.go

@@ -0,0 +1,88 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupAddUser{})
+}
+
+type commandS3GroupAddUser struct {
+}
+
+func (c *commandS3GroupAddUser) Name() string {
+	return "s3.group.add-user"
+}
+
+func (c *commandS3GroupAddUser) Help() string {
+	return `add a user to an S3 IAM group
+
+	s3.group.add-user -group <groupname> -user <username>
+`
+}
+
+func (c *commandS3GroupAddUser) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3GroupAddUser) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	group := f.String("group", "", "group name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+	if *group == "" {
+		return fmt.Errorf("-group is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.GetConfiguration()
+		if cfg == nil {
+			return fmt.Errorf("no IAM configuration found")
+		}
+
+		// Verify user exists
+		userFound := false
+		for _, id := range cfg.Identities {
+			if id.Name == *user {
+				userFound = true
+				break
+			}
+		}
+		if !userFound {
+			return fmt.Errorf("user %s not found", *user)
+		}
+
+		for _, g := range cfg.Groups {
+			if g.Name == *group {
+				// Check if already a member
+				for _, m := range g.Members {
+					if m == *user {
+						return fmt.Errorf("user %s is already a member of group %s", *user, *group)
+					}
+				}
+				g.Members = append(g.Members, *user)
+				if _, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{Configuration: cfg}); err != nil {
+					return err
+				}
+				return json.NewEncoder(writer).Encode(map[string]string{"group": *group, "user": *user})
+			}
+		}
+		return fmt.Errorf("group %s not found", *group)
+	})
+}

weed/shell/command_s3_group_create.go

@@ -0,0 +1,72 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupCreate{})
+}
+
+type commandS3GroupCreate struct {
+}
+
+func (c *commandS3GroupCreate) Name() string {
+	return "s3.group.create"
+}
+
+func (c *commandS3GroupCreate) Help() string {
+	return `create an S3 IAM group
+
+	s3.group.create -name <groupname>
+
+	Creates a new empty group. Add users with s3.group.add-user and
+	attach policies with s3.policy.attach or the IAM API.
+`
+}
+
+func (c *commandS3GroupCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3GroupCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	name := f.String("name", "", "group name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+	if *name == "" {
+		return fmt.Errorf("-name is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.GetConfiguration()
+		if cfg == nil {
+			return fmt.Errorf("no IAM configuration found")
+		}
+
+		for _, g := range cfg.Groups {
+			if g.Name == *name {
+				return fmt.Errorf("group %s already exists", *name)
+			}
+		}
+
+		cfg.Groups = append(cfg.Groups, &iam_pb.Group{Name: *name})
+
+		if _, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{Configuration: cfg}); err != nil {
+			return err
+		}
+
+		return json.NewEncoder(writer).Encode(map[string]string{"group": *name})
+	})
+}

weed/shell/command_s3_group_delete.go

@@ -0,0 +1,74 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupDelete{})
+}
+
+type commandS3GroupDelete struct {
+}
+
+func (c *commandS3GroupDelete) Name() string {
+	return "s3.group.delete"
+}
+
+func (c *commandS3GroupDelete) Help() string {
+	return `delete an S3 IAM group
+
+	s3.group.delete -name <groupname>
+
+	The group must have no members and no attached policies.
+`
+}
+
+func (c *commandS3GroupDelete) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3GroupDelete) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	name := f.String("name", "", "group name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+	if *name == "" {
+		return fmt.Errorf("-name is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.GetConfiguration()
+		if cfg == nil {
+			return fmt.Errorf("no IAM configuration found")
+		}
+
+		for i, g := range cfg.Groups {
+			if g.Name == *name {
+				if len(g.Members) > 0 {
+					return fmt.Errorf("cannot delete group %s: has %d member(s)", *name, len(g.Members))
+				}
+				if len(g.PolicyNames) > 0 {
+					return fmt.Errorf("cannot delete group %s: has %d attached policy(ies)", *name, len(g.PolicyNames))
+				}
+				cfg.Groups = append(cfg.Groups[:i], cfg.Groups[i+1:]...)
+				if _, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{Configuration: cfg}); err != nil {
+					return err
+				}
+				return json.NewEncoder(writer).Encode(map[string]string{"deleted": *name})
+			}
+		}
+		return fmt.Errorf("group %s not found", *name)
+	})
+}

weed/shell/command_s3_group_list.go

@@ -0,0 +1,71 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupList{})
+}
+
+type commandS3GroupList struct {
+}
+
+func (c *commandS3GroupList) Name() string {
+	return "s3.group.list"
+}
+
+func (c *commandS3GroupList) Help() string {
+	return `list S3 IAM groups
+
+	s3.group.list
+
+	Output: JSON array of groups with members and policies.
+`
+}
+
+func (c *commandS3GroupList) HasTag(CommandTag) bool {
+	return false
+}
+
+type s3GroupListEntry struct {
+	Name     string   `json:"name"`
+	Status   string   `json:"status"`
+	Members  int      `json:"members"`
+	Policies []string `json:"policies"`
+}
+
+func (c *commandS3GroupList) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+
+		var result []s3GroupListEntry
+		for _, g := range resp.Configuration.GetGroups() {
+			status := "enabled"
+			if g.Disabled {
+				status = "disabled"
+			}
+			policies := g.PolicyNames
+			if policies == nil {
+				policies = []string{}
+			}
+			result = append(result, s3GroupListEntry{
+				Name:     g.Name,
+				Status:   status,
+				Members:  len(g.Members),
+				Policies: policies,
+			})
+		}
+		if result == nil {
+			result = []s3GroupListEntry{}
+		}
+		return json.NewEncoder(writer).Encode(result)
+	})
+}

weed/shell/command_s3_group_remove_user.go

@@ -0,0 +1,75 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupRemoveUser{})
+}
+
+type commandS3GroupRemoveUser struct {
+}
+
+func (c *commandS3GroupRemoveUser) Name() string {
+	return "s3.group.remove-user"
+}
+
+func (c *commandS3GroupRemoveUser) Help() string {
+	return `remove a user from an S3 IAM group
+
+	s3.group.remove-user -group <groupname> -user <username>
+`
+}
+
+func (c *commandS3GroupRemoveUser) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3GroupRemoveUser) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	group := f.String("group", "", "group name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+	if *group == "" {
+		return fmt.Errorf("-group is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		cfg := resp.GetConfiguration()
+		if cfg == nil {
+			return fmt.Errorf("no IAM configuration found")
+		}
+
+		for _, g := range cfg.Groups {
+			if g.Name == *group {
+				for i, m := range g.Members {
+					if m == *user {
+						g.Members = append(g.Members[:i], g.Members[i+1:]...)
+						if _, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{Configuration: cfg}); err != nil {
+							return err
+						}
+						return json.NewEncoder(writer).Encode(map[string]string{"group": *group, "removed": *user})
+					}
+				}
+				return fmt.Errorf("user %s is not a member of group %s", *user, *group)
+			}
+		}
+		return fmt.Errorf("group %s not found", *group)
+	})
+}

weed/shell/command_s3_group_show.go

@@ -0,0 +1,84 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3GroupShow{})
+}
+
+type commandS3GroupShow struct {
+}
+
+func (c *commandS3GroupShow) Name() string {
+	return "s3.group.show"
+}
+
+func (c *commandS3GroupShow) Help() string {
+	return `show details of an S3 IAM group
+
+	s3.group.show -name <groupname>
+
+	Output: JSON with group name, status, members, and attached policies.
+`
+}
+
+func (c *commandS3GroupShow) HasTag(CommandTag) bool {
+	return false
+}
+
+type s3GroupShowResult struct {
+	Name     string   `json:"name"`
+	Status   string   `json:"status"`
+	Members  []string `json:"members"`
+	Policies []string `json:"policies"`
+}
+
+func (c *commandS3GroupShow) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	name := f.String("name", "", "group name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+	if *name == "" {
+		return fmt.Errorf("-name is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+
+		for _, g := range resp.Configuration.GetGroups() {
+			if g.Name == *name {
+				status := "enabled"
+				if g.Disabled {
+					status = "disabled"
+				}
+				members := g.Members
+				if members == nil {
+					members = []string{}
+				}
+				policies := g.PolicyNames
+				if policies == nil {
+					policies = []string{}
+				}
+				return json.NewEncoder(writer).Encode(s3GroupShowResult{
+					Name:     g.Name,
+					Status:   status,
+					Members:  members,
+					Policies: policies,
+				})
+			}
+		}
+		return fmt.Errorf("group %s not found", *name)
+	})
+}

weed/shell/command_s3_iam_client.go

@@ -0,0 +1,45 @@
+package shell
+
+import (
+	"context"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
+)
+
+// iamRequestTimeout caps every shell-originated IAM gRPC call so the shell
+// can't hang on an unresponsive filer.
+const iamRequestTimeout = 30 * time.Second
+
+// withIamClient invokes fn against the filer's IAM gRPC service. When
+// jwt.filer_signing.key is configured in security.toml, a freshly minted admin
+// Bearer token is attached to the outgoing context so the filer's
+// IamGrpcServer.checkAdminAuth passes; with no key configured the filer
+// accepts unauthenticated calls. The context already has the iamRequestTimeout
+// applied — callers can derive child contexts but should not need their own
+// timeout boilerplate.
+func (ce *CommandEnv) withIamClient(fn func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error) error {
+	return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error {
+		ctx, cancel := context.WithTimeout(iamAdminAuthContext(context.Background()), iamRequestTimeout)
+		defer cancel()
+		return fn(ctx, iam_pb.NewSeaweedIdentityAccessManagementClient(conn))
+	}, ce.option.FilerAddress.ToGrpcAddress(), false, ce.option.GrpcDialOption)
+}
+
+func iamAdminAuthContext(ctx context.Context) context.Context {
+	signingKey := util.GetViper().GetString("jwt.filer_signing.key")
+	if signingKey == "" {
+		return ctx
+	}
+	expiresAfterSec := util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds")
+	token := security.GenJwtForFilerAdmin(security.SigningKey(signingKey), expiresAfterSec)
+	if token == "" {
+		return ctx
+	}
+	return metadata.AppendToOutgoingContext(ctx, "authorization", security.BearerPrefix+string(token))
+}

weed/shell/command_s3_iam_export.go

@@ -0,0 +1,75 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3IAMExport{})
+}
+
+type commandS3IAMExport struct {
+}
+
+func (c *commandS3IAMExport) Name() string {
+	return "s3.iam.export"
+}
+
+func (c *commandS3IAMExport) Help() string {
+	return `export the full S3 IAM configuration as JSON
+
+	s3.iam.export
+	s3.iam.export -file backup.json
+
+	Exports all users, credentials, policies, service accounts, and groups.
+	Without -file, prints to stdout.
+`
+}
+
+func (c *commandS3IAMExport) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3IAMExport) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	file := f.String("file", "", "output file path (stdout if omitted)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetConfiguration(ctx, &iam_pb.GetConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+
+		var out io.Writer = writer
+		outputFile := util.ResolvePath(*file)
+		if outputFile != "" {
+			fp, err := os.OpenFile(outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+			if err != nil {
+				return fmt.Errorf("create file: %v", err)
+			}
+			defer fp.Close()
+			out = fp
+		}
+
+		if err := filer.ProtoToText(out, resp.Configuration); err != nil {
+			return err
+		}
+		fmt.Fprintln(out)
+
+		if outputFile != "" {
+			fmt.Fprintf(writer, "Exported IAM configuration to %s\n", outputFile)
+		}
+		return nil
+	})
+}

weed/shell/command_s3_iam_import.go

@@ -0,0 +1,83 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3IAMImport{})
+}
+
+type commandS3IAMImport struct {
+}
+
+func (c *commandS3IAMImport) Name() string {
+	return "s3.iam.import"
+}
+
+func (c *commandS3IAMImport) Help() string {
+	return `import S3 IAM configuration from a JSON file
+
+	s3.iam.import -file backup.json -apply
+
+	Replaces the entire IAM configuration (users, credentials, policies,
+	service accounts, groups) with the contents of the file.
+
+	Requires -apply to confirm, since this overwrites the current configuration.
+`
+}
+
+func (c *commandS3IAMImport) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3IAMImport) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	file := f.String("file", "", "input JSON file")
+	apply := f.Bool("apply", false, "confirm overwrite of the entire IAM configuration")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *file == "" {
+		return fmt.Errorf("-file is required")
+	}
+	if !*apply {
+		return fmt.Errorf("this overwrites the entire IAM configuration; use -apply to confirm")
+	}
+
+	data, err := os.ReadFile(util.ResolvePath(*file))
+	if err != nil {
+		return fmt.Errorf("read file: %w", err)
+	}
+
+	config := &iam_pb.S3ApiConfiguration{}
+	if err := filer.ParseS3ConfigurationFromBytes(data, config); err != nil {
+		return fmt.Errorf("parse configuration: %w", err)
+	}
+
+	err = commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		_, err := client.PutConfiguration(ctx, &iam_pb.PutConfigurationRequest{
+			Configuration: config,
+		})
+		return err
+	})
+	if err != nil {
+		return fmt.Errorf("put IAM configuration: %w", err)
+	}
+
+	fmt.Fprintf(writer, "Imported IAM configuration from %s\n", *file)
+	fmt.Fprintf(writer, "  Users:            %d\n", len(config.Identities))
+	fmt.Fprintf(writer, "  Policies:         %d\n", len(config.Policies))
+	fmt.Fprintf(writer, "  Service Accounts: %d\n", len(config.ServiceAccounts))
+	fmt.Fprintf(writer, "  Groups:           %d\n", len(config.Groups))
+	return nil
+}

weed/shell/command_s3_lifecycle_run_shard.go

@@ -0,0 +1,341 @@
+package shell
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/credential"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dailyrun"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/dispatcher"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/scheduler"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3LifecycleRunShard{})
+}
+
+type commandS3LifecycleRunShard struct{}
+
+func (c *commandS3LifecycleRunShard) Name() string {
+	return "s3.lifecycle.run-shard"
+}
+
+func (c *commandS3LifecycleRunShard) Help() string {
+	return `manually run one daily-replay pass for the given shards
+
+Drives dailyrun.Run once against the live filer + S3 server: builds
+the engine snapshot from filer-backed bucket configs, opens the
+meta-log subscription per shard, dispatches due actions via
+LifecycleDelete, and walks the live tree for any walker-bound rules.
+Persists each shard's cursor to /etc/s3/lifecycle/daily-cursors/
+so subsequent runs resume.
+
+Used by the s3-tests CI workflow and the test/s3/lifecycle/
+integration tests to drive expirations on demand without standing up
+the full admin+worker plugin stack.
+
+	# single shard
+	s3.lifecycle.run-shard -shard 0 -s3 localhost:8333 -events 100
+
+	# contiguous range
+	s3.lifecycle.run-shard -shards 0-15 -s3 localhost:8333 -events 5000
+
+	# explicit set
+	s3.lifecycle.run-shard -shards 0,3,7 -s3 localhost:8333
+
+	# bounded wall-clock
+	s3.lifecycle.run-shard -shards 0-15 -s3 localhost:8333 -runtime 10s
+`
+}
+
+func (c *commandS3LifecycleRunShard) HasTag(CommandTag) bool { return false }
+
+func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer io.Writer) error {
+	fs := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	shard := fs.Int("shard", -1, "single shard id in [0, 16); use -shards for a range or set")
+	shardsSpec := fs.String("shards", "", "shard range \"lo-hi\" or comma list \"a,b,c\"; mutually exclusive with -shard")
+	s3Endpoint := fs.String("s3", "", "s3 server gRPC endpoint, host:port")
+	eventBudget := fs.Int("events", 1000, "max in-shard events per pass (0 = drain to now)")
+	runtime := fs.Duration("runtime", 0, "wall-clock cap on the whole run; 0 = no timeout")
+	// -refresh drives the inter-pass cadence when the command runs as a
+	// long-lived worker (the s3tests CI workflow case): every refresh
+	// the engine snapshot is re-loaded and dailyrun.Run fires another
+	// pass. 0 means "run once and exit" (the integration-test case).
+	cadence := fs.Duration("refresh", 0, "inter-pass interval; 0 = single pass, then exit")
+	// Obsolete flags kept for back-compat with existing CI scripts and
+	// integration tests. Accept and ignore.
+	_ = fs.Duration("dispatch", 0, "ignored (legacy streaming flag)")
+	_ = fs.Duration("checkpoint", 0, "ignored (legacy streaming flag)")
+	_ = fs.Duration("bootstrap-interval", 0, "ignored (legacy streaming flag)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	shards, err := resolveShardSelection(*shard, *shardsSpec)
+	if err != nil {
+		return err
+	}
+	if *s3Endpoint == "" {
+		return fmt.Errorf("-s3 required (host:port of s3 server gRPC)")
+	}
+	if *eventBudget < 0 {
+		return fmt.Errorf("-events must be >= 0 (0 = unbounded)")
+	}
+
+	bucketsPath, err := resolveBucketsPath(env)
+	if err != nil {
+		return fmt.Errorf("resolve buckets path: %w", err)
+	}
+	fmt.Fprintf(writer, "buckets path: %s\n", bucketsPath)
+
+	dialCtx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second)
+	conn, err := pb.GrpcDial(dialCtx, *s3Endpoint, false, env.option.GrpcDialOption)
+	dialCancel()
+	if err != nil {
+		return fmt.Errorf("dial s3 %s: %w", *s3Endpoint, err)
+	}
+	defer conn.Close()
+	rpcClient := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(conn)
+
+	return env.WithFilerClient(true, func(filerClient filer_pb.SeaweedFilerClient) error {
+		ctx := context.Background()
+		var cancel context.CancelFunc
+		if *runtime > 0 {
+			ctx, cancel = context.WithTimeout(ctx, *runtime)
+			defer cancel()
+		}
+		client := &lifecycleClientCallable{c: rpcClient}
+		listFn := dailyrun.FilerListFunc(filerClient, bucketsPath)
+		walkerDispatch := &dailyrun.WalkerDispatcher{Client: client}
+
+		fmt.Fprintf(writer, "running shards %s (event budget=%d, runtime=%s, refresh=%s)…\n",
+			formatShardLabel(shards), *eventBudget, *runtime, *cadence)
+
+		announcedLoad := false
+		runPass := func() error {
+			// When -refresh is set we MUST cap each pass; otherwise
+			// drainShardEvents blocks until the outer runtime ctx
+			// expires and the loop's ticker never fires. With cadence=0
+			// (one-shot) the pass uses the full ctx and returns when
+			// the runtime cap hits.
+			passCtx := ctx
+			if *cadence > 0 {
+				var passCancel context.CancelFunc
+				// Pass budget = cadence + grace. Grace gives the
+				// drain time to actually process events that arrived
+				// during the cadence window.
+				passCtx, passCancel = context.WithTimeout(ctx, *cadence+5*time.Second)
+				defer passCancel()
+			}
+			eng := engine.New()
+			inputs, parseErrors, err := scheduler.LoadCompileInputs(passCtx, filerClient, bucketsPath)
+			if err != nil {
+				return fmt.Errorf("load lifecycle configs: %w", err)
+			}
+			for i, pe := range parseErrors {
+				if i < 3 {
+					fmt.Fprintf(writer, "warning: %s: %v\n", pe.Bucket, pe.Err)
+				}
+			}
+			if extra := len(parseErrors) - 3; extra > 0 {
+				fmt.Fprintf(writer, "warning: %d additional bucket(s) had malformed lifecycle config\n", extra)
+			}
+			eng.Compile(inputs, engine.CompileOptions{PriorStates: scheduler.AllActivePriorStates(inputs)})
+			if len(inputs) == 0 {
+				return nil
+			}
+			if !announcedLoad {
+				fmt.Fprintf(writer, "loaded lifecycle for %d bucket(s)\n", len(inputs))
+				announcedLoad = true
+			}
+			buckets := make([]string, 0, len(inputs))
+			for _, in := range inputs {
+				if in.Bucket != "" {
+					buckets = append(buckets, in.Bucket)
+				}
+			}
+			walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shardID int) error {
+				return dailyrun.WalkBuckets(walkCtx, view, shardID, buckets, listFn, walkerDispatch)
+			})
+			return dailyrun.Run(passCtx, dailyrun.Config{
+				Shards:      shards,
+				BucketsPath: bucketsPath,
+				Engine:      eng,
+				FilerClient: filerClient,
+				Client:      client,
+				Persister:   &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)},
+				Lister:      dispatcher.NewFilerSiblingLister(filerClient, bucketsPath),
+				// The shell command is used for bounded one-shot sweeps in
+				// integration tests and CI. Fan out across the selected shards
+				// so recovery walks do not serialize 16 shard scans into a 10s
+				// timeout budget.
+				Workers:     len(shards),
+				Walker:      walker,
+				EventBudget: *eventBudget,
+				ClientName:  fmt.Sprintf("shell-lifecycle-%s", formatShardLabel(shards)),
+			})
+		}
+
+		if err := runPass(); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+			return fmt.Errorf("daily_run: %w", err)
+		}
+		// cadence=0 → one-shot (test/s3/lifecycle/ uses this).
+		// cadence>0 → loop until ctx done (s3tests CI workflow uses this).
+		if *cadence > 0 {
+			ticker := time.NewTicker(*cadence)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-ctx.Done():
+					fmt.Fprintf(writer, "shards %s complete; ctx done\n", formatShardLabel(shards))
+					return nil
+				case <-ticker.C:
+					if err := runPass(); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+						fmt.Fprintf(writer, "shards %s pass error: %v\n", formatShardLabel(shards), err)
+					}
+				}
+			}
+		}
+		fmt.Fprintf(writer, "shards %s complete; cursors checkpointed\n", formatShardLabel(shards))
+		return nil
+	})
+}
+
+// resolveShardSelection turns the -shard / -shards flags into a sorted,
+// deduplicated []int. Exactly one form must be specified.
+func resolveShardSelection(singleShard int, shardsSpec string) ([]int, error) {
+	if singleShard >= 0 && shardsSpec != "" {
+		return nil, fmt.Errorf("-shard and -shards are mutually exclusive")
+	}
+	if singleShard < 0 && shardsSpec == "" {
+		return nil, fmt.Errorf("specify -shard <id> or -shards <range|set>")
+	}
+	if singleShard >= 0 {
+		if singleShard >= s3lifecycle.ShardCount {
+			return nil, fmt.Errorf("-shard %d out of [0,%d)", singleShard, s3lifecycle.ShardCount)
+		}
+		return []int{singleShard}, nil
+	}
+	return parseShardsSpec(shardsSpec)
+}
+
+// parseShardsSpec accepts "lo-hi" (inclusive) or "a,b,c" and returns a
+// sorted, deduplicated, in-range []int.
+func parseShardsSpec(spec string) ([]int, error) {
+	spec = strings.TrimSpace(spec)
+	seen := map[int]struct{}{}
+	add := func(v int) error {
+		if v < 0 || v >= s3lifecycle.ShardCount {
+			return fmt.Errorf("shard %d out of [0,%d)", v, s3lifecycle.ShardCount)
+		}
+		seen[v] = struct{}{}
+		return nil
+	}
+	if strings.Contains(spec, "-") && !strings.Contains(spec, ",") {
+		parts := strings.SplitN(spec, "-", 2)
+		lo, err := strconv.Atoi(strings.TrimSpace(parts[0]))
+		if err != nil {
+			return nil, fmt.Errorf("range lo: %w", err)
+		}
+		hi, err := strconv.Atoi(strings.TrimSpace(parts[1]))
+		if err != nil {
+			return nil, fmt.Errorf("range hi: %w", err)
+		}
+		if lo > hi {
+			return nil, fmt.Errorf("range lo %d > hi %d", lo, hi)
+		}
+		for v := lo; v <= hi; v++ {
+			if err := add(v); err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		for _, part := range strings.Split(spec, ",") {
+			part = strings.TrimSpace(part)
+			if part == "" {
+				continue
+			}
+			v, err := strconv.Atoi(part)
+			if err != nil {
+				return nil, fmt.Errorf("shard list: %w", err)
+			}
+			if err := add(v); err != nil {
+				return nil, err
+			}
+		}
+	}
+	if len(seen) == 0 {
+		return nil, fmt.Errorf("empty shard set")
+	}
+	out := make([]int, 0, len(seen))
+	for v := range seen {
+		out = append(out, v)
+	}
+	sort.Ints(out)
+	return out, nil
+}
+
+func formatShardLabel(shards []int) string {
+	if len(shards) == 1 {
+		return fmt.Sprintf("%d", shards[0])
+	}
+	contiguous := true
+	for i := 1; i < len(shards); i++ {
+		if shards[i] != shards[i-1]+1 {
+			contiguous = false
+			break
+		}
+	}
+	if contiguous {
+		return fmt.Sprintf("%d-%d", shards[0], shards[len(shards)-1])
+	}
+	parts := make([]string, len(shards))
+	for i, v := range shards {
+		parts[i] = strconv.Itoa(v)
+	}
+	return strings.Join(parts, ",")
+}
+
+// lifecycleClientCallable adapts the generated grpc client (variadic
+// CallOption tail) to dailyrun.LifecycleClient.
+type lifecycleClientCallable struct {
+	c s3_lifecycle_pb.SeaweedS3LifecycleInternalClient
+}
+
+func (l *lifecycleClientCallable) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
+	ctx, _ = credential.WithS3InternalAdminAuth(ctx)
+	return l.c.LifecycleDelete(ctx, req)
+}
+
+// resolveBucketsPath fetches the filer's configured buckets directory.
+// Falls back to /buckets when the filer doesn't return one.
+func resolveBucketsPath(env *CommandEnv) (string, error) {
+	var path string
+	err := env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
+		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		path = resp.GetDirBuckets()
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+	if path == "" {
+		path = "/buckets"
+	}
+	return path, nil
+}

weed/shell/command_s3_lifecycle_run_shard_test.go

@@ -0,0 +1,92 @@
+package shell
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseShardsSpec_Range(t *testing.T) {
+	got, err := parseShardsSpec("3-7")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	want := []int{3, 4, 5, 6, 7}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestParseShardsSpec_Set(t *testing.T) {
+	got, err := parseShardsSpec("0,3,7")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	want := []int{0, 3, 7}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestParseShardsSpec_DedupSort(t *testing.T) {
+	got, err := parseShardsSpec("7,3,3,0")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	want := []int{0, 3, 7}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestParseShardsSpec_OutOfRange(t *testing.T) {
+	if _, err := parseShardsSpec("16"); err == nil {
+		t.Fatal("expected out-of-range error for 16")
+	}
+	if _, err := parseShardsSpec("-1"); err == nil {
+		t.Fatal("expected out-of-range error for -1")
+	}
+	if _, err := parseShardsSpec("0-16"); err == nil {
+		t.Fatal("expected out-of-range error for 0-16")
+	}
+}
+
+func TestParseShardsSpec_BadRange(t *testing.T) {
+	if _, err := parseShardsSpec("7-3"); err == nil {
+		t.Fatal("expected lo>hi error")
+	}
+}
+
+func TestResolveShardSelection_Mutex(t *testing.T) {
+	if _, err := resolveShardSelection(0, "1,2"); err == nil {
+		t.Fatal("expected mutex error when both -shard and -shards set")
+	}
+	if _, err := resolveShardSelection(-1, ""); err == nil {
+		t.Fatal("expected error when neither set")
+	}
+}
+
+func TestResolveShardSelection_SingleShard(t *testing.T) {
+	got, err := resolveShardSelection(5, "")
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !reflect.DeepEqual(got, []int{5}) {
+		t.Fatalf("got %v, want [5]", got)
+	}
+}
+
+func TestFormatShardLabel(t *testing.T) {
+	cases := []struct {
+		in   []int
+		want string
+	}{
+		{[]int{5}, "5"},
+		{[]int{0, 1, 2, 3}, "0-3"},
+		{[]int{0, 2, 5}, "0,2,5"},
+	}
+	for _, tc := range cases {
+		if got := formatShardLabel(tc.in); got != tc.want {
+			t.Errorf("formatShardLabel(%v)=%q, want %q", tc.in, got, tc.want)
+		}
+	}
+}

weed/shell/command_s3_policy.go

@@ -0,0 +1,144 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3Policy{})
+}
+
+type commandS3Policy struct {
+}
+
+func (c *commandS3Policy) Name() string {
+	return "s3.policy"
+}
+
+func (c *commandS3Policy) Help() string {
+	return `manage s3 policies
+
+	# create or update a policy
+	s3.policy -put -name=mypolicy -file=policy.json
+
+	# list all policies
+	s3.policy -list
+
+	# get a policy
+	s3.policy -get -name=mypolicy
+
+	# delete a policy
+	s3.policy -delete -name=mypolicy
+	`
+}
+
+func (c *commandS3Policy) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3Policy) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+
+	s3PolicyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	put := s3PolicyCommand.Bool("put", false, "create or update a policy")
+	get := s3PolicyCommand.Bool("get", false, "get a policy")
+	list := s3PolicyCommand.Bool("list", false, "list all policies")
+	del := s3PolicyCommand.Bool("delete", false, "delete a policy")
+	name := s3PolicyCommand.String("name", "", "policy name")
+	file := s3PolicyCommand.String("file", "", "policy file (json)")
+
+	if err = s3PolicyCommand.Parse(args); err != nil {
+		return err
+	}
+
+	actionCount := 0
+	for _, v := range []bool{*put, *get, *list, *del} {
+		if v {
+			actionCount++
+		}
+	}
+	if actionCount == 0 {
+		return fmt.Errorf("one of -put, -get, -list, -delete must be specified")
+	}
+	if actionCount > 1 {
+		return fmt.Errorf("only one of -put, -get, -list, -delete can be specified")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		if *put {
+			if *name == "" {
+				return fmt.Errorf("-name is required")
+			}
+			if *file == "" {
+				return fmt.Errorf("-file is required")
+			}
+			data, err := os.ReadFile(util.ResolvePath(*file))
+			if err != nil {
+				return fmt.Errorf("failed to read policy file: %v", err)
+			}
+
+			// Validate JSON
+			var policy policy_engine.PolicyDocument
+			if err := json.Unmarshal(data, &policy); err != nil {
+				return fmt.Errorf("invalid policy json: %v", err)
+			}
+
+			_, err = client.PutPolicy(ctx, &iam_pb.PutPolicyRequest{
+				Name:    *name,
+				Content: string(data),
+			})
+			return err
+		}
+
+		if *get {
+			if *name == "" {
+				return fmt.Errorf("-name is required")
+			}
+			resp, err := client.GetPolicy(ctx, &iam_pb.GetPolicyRequest{
+				Name: *name,
+			})
+			if err != nil {
+				return err
+			}
+			if resp.Content == "" {
+				return fmt.Errorf("policy not found")
+			}
+			fmt.Fprintf(writer, "%s\n", resp.Content)
+			return nil
+		}
+
+		if *list {
+			resp, err := client.ListPolicies(ctx, &iam_pb.ListPoliciesRequest{})
+			if err != nil {
+				return err
+			}
+			for _, policy := range resp.Policies {
+				fmt.Fprintf(writer, "Name: %s\n", policy.Name)
+				fmt.Fprintf(writer, "Content: %s\n", policy.Content)
+				fmt.Fprintf(writer, "---\n")
+			}
+			return nil
+		}
+
+		if *del {
+			if *name == "" {
+				return fmt.Errorf("-name is required")
+			}
+			_, err := client.DeletePolicy(ctx, &iam_pb.DeletePolicyRequest{
+				Name: *name,
+			})
+			return err
+		}
+
+		return nil
+	})
+
+}

weed/shell/command_s3_policy_attach.go

@@ -0,0 +1,86 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3PolicyAttach{})
+}
+
+type commandS3PolicyAttach struct {
+}
+
+func (c *commandS3PolicyAttach) Name() string {
+	return "s3.policy.attach"
+}
+
+func (c *commandS3PolicyAttach) Help() string {
+	return `attach a policy to an S3 IAM user
+
+	s3.policy.attach -policy <policy_name> -user <username>
+
+	The policy must already exist (create it with s3.policy -put).
+`
+}
+
+func (c *commandS3PolicyAttach) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3PolicyAttach) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	policy := f.String("policy", "", "policy name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *policy == "" {
+		return fmt.Errorf("-policy is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		// Verify the policy exists
+		_, err := client.GetPolicy(ctx, &iam_pb.GetPolicyRequest{Name: *policy})
+		if err != nil {
+			return fmt.Errorf("get policy %q: %w", *policy, err)
+		}
+
+		// Get the user
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return fmt.Errorf("get user %q: %w", *user, err)
+		}
+		if resp.Identity == nil {
+			return fmt.Errorf("user %q returned empty identity", *user)
+		}
+
+		// Check if already attached
+		for _, p := range resp.Identity.PolicyNames {
+			if p == *policy {
+				return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+			}
+		}
+
+		resp.Identity.PolicyNames = append(resp.Identity.PolicyNames, *policy)
+		_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+			Username: *user,
+			Identity: resp.Identity,
+		})
+		if err != nil {
+			return err
+		}
+
+		return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+	})
+}

weed/shell/command_s3_policy_detach.go

@@ -0,0 +1,83 @@
+package shell
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3PolicyDetach{})
+}
+
+type commandS3PolicyDetach struct {
+}
+
+func (c *commandS3PolicyDetach) Name() string {
+	return "s3.policy.detach"
+}
+
+func (c *commandS3PolicyDetach) Help() string {
+	return `detach a policy from an S3 IAM user
+
+	s3.policy.detach -policy <policy_name> -user <username>
+`
+}
+
+func (c *commandS3PolicyDetach) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3PolicyDetach) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	policy := f.String("policy", "", "policy name")
+	user := f.String("user", "", "user name")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *policy == "" {
+		return fmt.Errorf("-policy is required")
+	}
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	return commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		resp, err := client.GetUser(ctx, &iam_pb.GetUserRequest{Username: *user})
+		if err != nil {
+			return fmt.Errorf("get user %q: %w", *user, err)
+		}
+		if resp.Identity == nil {
+			return fmt.Errorf("user %q returned empty identity", *user)
+		}
+
+		found := false
+		var kept []string
+		for _, p := range resp.Identity.PolicyNames {
+			if p == *policy {
+				found = true
+			} else {
+				kept = append(kept, p)
+			}
+		}
+		if !found {
+			return fmt.Errorf("policy %q is not attached to user %q", *policy, *user)
+		}
+
+		resp.Identity.PolicyNames = kept
+		_, err = client.UpdateUser(ctx, &iam_pb.UpdateUserRequest{
+			Username: *user,
+			Identity: resp.Identity,
+		})
+		if err != nil {
+			return err
+		}
+
+		return json.NewEncoder(writer).Encode(map[string]string{"policy": *policy, "user": *user})
+	})
+}

weed/shell/command_s3_serviceaccount_create.go

@@ -0,0 +1,139 @@
+package shell
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/iam"
+	"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
+)
+
+func init() {
+	Commands = append(Commands, &commandS3ServiceAccountCreate{})
+}
+
+type commandS3ServiceAccountCreate struct {
+}
+
+func (c *commandS3ServiceAccountCreate) Name() string {
+	return "s3.serviceaccount.create"
+}
+
+func (c *commandS3ServiceAccountCreate) Help() string {
+	return `create a service account for an S3 IAM user
+
+	s3.serviceaccount.create -user <parent_user> -description "my app"
+	s3.serviceaccount.create -user <parent_user> -actions Read,List -expiry 24h
+
+	Service accounts are linked to a parent user and can have restricted
+	permissions (a subset of the parent's actions).
+`
+}
+
+func (c *commandS3ServiceAccountCreate) HasTag(CommandTag) bool {
+	return false
+}
+
+func (c *commandS3ServiceAccountCreate) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
+	f := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	user := f.String("user", "", "parent user name")
+	description := f.String("description", "", "optional description")
+	actions := f.String("actions", "", "comma-separated actions (subset of parent)")
+	expiry := f.Duration("expiry", 0, "expiration duration (e.g. 24h, 0 = no expiration)")
+	if err := f.Parse(args); err != nil {
+		return err
+	}
+
+	if *user == "" {
+		return fmt.Errorf("-user is required")
+	}
+
+	ak, err := iam.GenerateRandomString(iam.AccessKeyIdLength, iam.CharsetUpper)
+	if err != nil {
+		return fmt.Errorf("generate access key: %v", err)
+	}
+	sk, err := iam.GenerateSecretAccessKey()
+	if err != nil {
+		return fmt.Errorf("generate secret key: %v", err)
+	}
+
+	// Generate a unique service account ID matching the format
+	// required by credential.ValidateServiceAccountId: sa:<parent>:<uuid>.
+	// 16 bytes (128 bits) of randomness makes collisions negligible.
+	var idBytes [16]byte
+	if _, err := rand.Read(idBytes[:]); err != nil {
+		return fmt.Errorf("generate service account id: %v", err)
+	}
+	saId := fmt.Sprintf("sa:%s:%s", *user, hex.EncodeToString(idBytes[:]))
+
+	sa := &iam_pb.ServiceAccount{
+		Id:          saId,
+		ParentUser:  *user,
+		Description: *description,
+		Credential: &iam_pb.Credential{
+			AccessKey: ak,
+			SecretKey: sk,
+			Status:    iam.AccessKeyStatusActive,
+		},
+		CreatedAt: time.Now().Unix(),
+	}
+
+	validActions := map[string]string{
+		"read": "Read", "write": "Write", "list": "List",
+		"tagging": "Tagging", "admin": "Admin",
+	}
+	if *actions != "" {
+		seen := make(map[string]struct{})
+		for _, a := range strings.Split(*actions, ",") {
+			a = strings.TrimSpace(a)
+			if a != "" {
+				canonical, ok := validActions[strings.ToLower(a)]
+				if !ok {
+					return fmt.Errorf("invalid action %q: supported actions are Read, Write, List, Tagging, Admin", a)
+				}
+				if _, dup := seen[canonical]; dup {
+					continue
+				}
+				seen[canonical] = struct{}{}
+				sa.Actions = append(sa.Actions, canonical)
+			}
+		}
+	}
+
+	if *expiry < 0 {
+		return fmt.Errorf("-expiry must be >= 0")
+	}
+	if *expiry > 0 {
+		sa.Expiration = time.Now().Add(*expiry).Unix()
+	}
+
+	err = commandEnv.withIamClient(func(ctx context.Context, client iam_pb.SeaweedIdentityAccessManagementClient) error {
+		_, err := client.CreateServiceAccount(ctx, &iam_pb.CreateServiceAccountRequest{
+			ServiceAccount: sa,
+		})
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Created service account for user %q\n", *user)
+	fmt.Fprintf(writer, "ID:         %s\n", saId)
+	fmt.Fprintf(writer, "Access Key: %s\n", ak)
+	fmt.Fprintf(writer, "Secret Key: %s\n", sk)
+	if *description != "" {
+		fmt.Fprintf(writer, "Desc:       %s\n", *description)
+	}
+	if *expiry > 0 {
+		fmt.Fprintf(writer, "Expires:    %s\n", time.Unix(sa.Expiration, 0).Format(time.RFC3339))
+	}
+	fmt.Fprintln(writer)
+	fmt.Fprintln(writer, "Save these credentials - the secret key cannot be retrieved later.")
+	return nil
+}