snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

unmaintained/README.txt

@@ -0,0 +1,9 @@
+Here is the folder of unmaintained go code.
+
+Although its content is useful sometimes, 
+Its content would not be released.
+And the code can be ugly, e.g., not checking exceptions.
+
+If any code is used often, it will be productionized.
+So it is helpful if anyone wants to cleanup a bit.
+

unmaintained/change_superblock/change_superblock.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	fixVolumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumeCollection = flag.String("collection", "", "the volume collection name")
+	fixVolumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+	targetReplica       = flag.String("replication", "", "If just empty, only print out current replication setting.")
+	targetTTL           = flag.String("ttl", "", "If just empty, only print out current ttl setting.")
+)
+
+/*
+This is to change replication factor in .dat file header. Need to shut down the volume servers
+that has those volumes.
+
+ 1. fix the .dat file in place
+    // just see the replication setting
+    go run change_replication.go -volumeId=9 -dir=/Users/chrislu/Downloads
+    Current Volume Replication: 000
+    // fix the replication setting
+    go run change_replication.go -volumeId=9 -dir=/Users/chrislu/Downloads -replication 001
+    Current Volume Replication: 000
+    Changing to: 001
+    Done.
+
+2. copy the fixed .dat and related .idx files to some remote server
+3. restart volume servers or start new volume servers.
+*/
+func main() {
+	flag.Parse()
+	util_http.NewGlobalHttpClient()
+
+	fileName := strconv.Itoa(*fixVolumeId)
+	if *fixVolumeCollection != "" {
+		fileName = *fixVolumeCollection + "_" + fileName
+	}
+	datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDWR, 0644)
+	if err != nil {
+		glog.Fatalf("Open Volume Data File [ERROR]: %v", err)
+	}
+	datBackend := backend.NewDiskFile(datFile)
+	defer datBackend.Close()
+
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
+
+	if err != nil {
+		glog.Fatalf("cannot parse existing super block: %v", err)
+	}
+
+	fmt.Printf("Current Volume Replication: %s\n", superBlock.ReplicaPlacement)
+	fmt.Printf("Current Volume TTL: %s\n", superBlock.Ttl.String())
+
+	hasChange := false
+
+	if *targetReplica != "" {
+		replica, err := super_block.NewReplicaPlacementFromString(*targetReplica)
+
+		if err != nil {
+			glog.Fatalf("cannot parse target replica %s: %v", *targetReplica, err)
+		}
+
+		fmt.Printf("Changing replication to: %s\n", replica)
+
+		superBlock.ReplicaPlacement = replica
+		hasChange = true
+	}
+
+	if *targetTTL != "" {
+		ttl, err := needle.ReadTTL(*targetTTL)
+
+		if err != nil {
+			glog.Fatalf("cannot parse target ttl %s: %v", *targetTTL, err)
+		}
+
+		fmt.Printf("Changing ttl to: %s\n", ttl)
+
+		superBlock.Ttl = ttl
+		hasChange = true
+	}
+
+	if hasChange {
+
+		header := superBlock.Bytes()
+
+		if n, e := datBackend.WriteAt(header, 0); n == 0 || e != nil {
+			glog.Fatalf("cannot write super block: %v", e)
+		}
+
+		fmt.Println("Change Applied.")
+	}
+
+}

unmaintained/check_disk_size/check_disk_size.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"runtime"
+	"syscall"
+)
+
+var (
+	dir = flag.String("dir", ".", "the directory which uses a disk")
+)
+
+func main() {
+	flag.Parse()
+
+	fillInDiskStatus(*dir)
+
+	fmt.Printf("OS: %v\n", runtime.GOOS)
+	fmt.Printf("Arch: %v\n", runtime.GOARCH)
+
+}
+
+func fillInDiskStatus(dir string) {
+	fs := syscall.Statfs_t{}
+	err := syscall.Statfs(dir, &fs)
+	if err != nil {
+		fmt.Printf("failed to statfs on %s: %v\n", dir, err)
+		return
+	}
+	fmt.Printf("statfs: %+v\n", fs)
+	fmt.Println()
+
+	total := fs.Blocks * uint64(fs.Bsize)
+	free := fs.Bfree * uint64(fs.Bsize)
+	fmt.Printf("Total: %d blocks x %d block size = %d bytes\n", fs.Blocks, uint64(fs.Bsize), total)
+	fmt.Printf("Free : %d blocks x %d block size = %d bytes\n", fs.Bfree, uint64(fs.Bsize), free)
+	fmt.Printf("Used : %d blocks x %d block size = %d bytes\n", fs.Blocks-fs.Bfree, uint64(fs.Bsize), total-free)
+	fmt.Printf("Free Percentage : %.2f%%\n", float32((float64(free)/float64(total))*100))
+	fmt.Printf("Used Percentage : %.2f%%\n", float32((float64(total-free)/float64(total))*100))
+	return
+}

unmaintained/compact_leveldb/compact_leveldb.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/errors"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	dir = flag.String("dir", ".", "data directory to store leveldb files")
+)
+
+func main() {
+
+	flag.Parse()
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+		OpenFilesCacheCapacity:        -1,
+	}
+
+	db, err := leveldb.OpenFile(*dir, opts)
+	if errors.IsCorrupted(err) {
+		db, err = leveldb.RecoverFile(*dir, opts)
+	}
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	if err := db.CompactRange(util.Range{}); err != nil {
+		log.Fatal(err)
+	}
+}

unmaintained/diff_volume_servers/diff_volume_servers.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"math"
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc"
+)
+
+var (
+	serversStr       = flag.String("volumeServers", "", "comma-delimited list of volume servers to diff the volume against")
+	volumeId         = flag.Int("volumeId", -1, "a volume id to diff from servers")
+	volumeCollection = flag.String("collection", "", "the volume collection name")
+	grpcDialOption   grpc.DialOption
+)
+
+/*
+Diff the volume's files across multiple volume servers.
+diff_volume_servers -volumeServers 127.0.0.1:8080,127.0.0.1:8081 -volumeId 5
+
+Example Output:
+reference 127.0.0.1:8081
+fileId volumeServer message
+5,01617c3f61 127.0.0.1:8080 wrongSize
+*/
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	util.LoadSecurityConfiguration()
+	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	vid := uint32(*volumeId)
+	servers := pb.ServerAddresses(*serversStr).ToAddresses()
+	if len(servers) < 2 {
+		glog.Fatalf("You must specify more than 1 server\n")
+	}
+	var referenceServer pb.ServerAddress
+	var maxOffset int64
+	allFiles := map[pb.ServerAddress]map[types.NeedleId]needleState{}
+	for _, addr := range servers {
+		files, offset, err := getVolumeFiles(vid, addr)
+		if err != nil {
+			glog.Fatalf("Failed to copy idx from volume server %s\n", err)
+		}
+		allFiles[addr] = files
+		if offset > maxOffset {
+			referenceServer = addr
+		}
+	}
+
+	same := true
+	fmt.Println("reference", referenceServer)
+	fmt.Println("fileId volumeServer message")
+	for nid, n := range allFiles[referenceServer] {
+		for addr, files := range allFiles {
+			if addr == referenceServer {
+				continue
+			}
+			var diffMsg string
+			n2, ok := files[nid]
+			if !ok {
+				if n.state == stateDeleted {
+					continue
+				}
+				diffMsg = "missing"
+			} else if n2.state != n.state {
+				switch n.state {
+				case stateDeleted:
+					diffMsg = "notDeleted"
+				case statePresent:
+					diffMsg = "deleted"
+				}
+			} else if n2.size != n.size {
+				diffMsg = "wrongSize"
+			} else {
+				continue
+			}
+			same = false
+
+			// fetch the needle details
+			var id string
+			var err error
+			if n.state == statePresent {
+				id, err = getNeedleFileId(vid, nid, referenceServer)
+			} else {
+				id, err = getNeedleFileId(vid, nid, addr)
+			}
+			if err != nil {
+				glog.Fatalf("Failed to get needle info %d from volume server %s\n", nid, err)
+			}
+			fmt.Println(id, addr, diffMsg)
+		}
+	}
+	if !same {
+		os.Exit(1)
+	}
+}
+
+const (
+	stateDeleted uint8 = 1
+	statePresent uint8 = 2
+)
+
+type needleState struct {
+	state uint8
+	size  types.Size
+}
+
+func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleState, int64, error) {
+	var idxFile *bytes.Reader
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		copyFileClient, err := vs.CopyFile(ctx, &volume_server_pb.CopyFileRequest{
+			VolumeId:           v,
+			Ext:                ".idx",
+			CompactionRevision: math.MaxUint32,
+			StopOffset:         math.MaxInt64,
+			Collection:         *volumeCollection,
+		})
+		if err != nil {
+			return err
+		}
+		var buf bytes.Buffer
+		for {
+			resp, err := copyFileClient.Recv()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			buf.Write(resp.FileContent)
+		}
+		idxFile = bytes.NewReader(buf.Bytes())
+		return nil
+	})
+	if err != nil {
+		return nil, 0, err
+	}
+
+	var maxOffset int64
+	files := map[types.NeedleId]needleState{}
+	err = idx.WalkIndexFile(idxFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		if offset.IsZero() || size.IsDeleted() {
+			files[key] = needleState{
+				state: stateDeleted,
+				size:  size,
+			}
+		} else {
+			files[key] = needleState{
+				state: statePresent,
+				size:  size,
+			}
+		}
+		if actual := offset.ToActualOffset(); actual > maxOffset {
+			maxOffset = actual
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, 0, err
+	}
+	return files, maxOffset, nil
+}
+
+func getNeedleFileId(v uint32, nid types.NeedleId, addr pb.ServerAddress) (string, error) {
+	var id string
+	err := operation.WithVolumeServerClient(false, addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		resp, err := vs.VolumeNeedleStatus(context.Background(), &volume_server_pb.VolumeNeedleStatusRequest{
+			VolumeId: v,
+			NeedleId: uint64(nid),
+		})
+		if err != nil {
+			return err
+		}
+		id = needle.NewFileId(needle.VolumeId(v), resp.NeedleId, resp.Cookie).String()
+		return nil
+	})
+	return id, err
+}

unmaintained/disk/disk_status.go

@@ -0,0 +1,56 @@
+//go:build !windows && !openbsd && !netbsd && !plan9 && !solaris
+
+package main
+
+import (
+	"log"
+	"syscall"
+)
+
+// go run unmaintained/disk/disk_status.go
+
+type DiskStatus struct {
+	Dir         string  `protobuf:"bytes,1,opt,name=dir,proto3" json:"dir,omitempty"`
+	All         uint64  `protobuf:"varint,2,opt,name=all,proto3" json:"all,omitempty"`
+	Used        uint64  `protobuf:"varint,3,opt,name=used,proto3" json:"used,omitempty"`
+	Free        uint64  `protobuf:"varint,4,opt,name=free,proto3" json:"free,omitempty"`
+	PercentFree float32 `protobuf:"fixed32,5,opt,name=percent_free,json=percentFree,proto3" json:"percent_free,omitempty"`
+	PercentUsed float32 `protobuf:"fixed32,6,opt,name=percent_used,json=percentUsed,proto3" json:"percent_used,omitempty"`
+	DiskType    string  `protobuf:"bytes,7,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"`
+
+	// new fields about avail blocks
+	Avail        uint64  `protobuf:"varint,4,opt,name=avail,proto3" json:"avail,omitempty"`
+	PercentAvail float32 `protobuf:"fixed32,5,opt,name=percent_avail,json=percentAvail,proto3" json:"percent_avail,omitempty"`
+}
+
+func main() {
+	dirs := []string{"/mnt/sdb", "/mnt/sdc", "/mnt/sdd", "/mnt/sde", "/mnt/sdf", "/mnt/sdg", "/mnt/sdh", "/mnt/sdi", "/mnt/sdj"}
+	// dirs := []string{"/mnt/sdb"}
+	for _, dir := range dirs {
+		disk := &DiskStatus{Dir: dir}
+		fillInDiskStatus(disk)
+
+		// bytes, _ := json.Marshal(disk)
+		// log.Printf("disk status %s", bytes)
+		log.Printf("disk: %s avail: %f free: %f", disk.Dir, disk.PercentAvail, disk.PercentFree)
+	}
+}
+
+func fillInDiskStatus(disk *DiskStatus) {
+	fs := syscall.Statfs_t{}
+	err := syscall.Statfs(disk.Dir, &fs)
+	if err != nil {
+		return
+	}
+
+	disk.All = fs.Blocks * uint64(fs.Bsize)
+	disk.Free = fs.Bfree * uint64(fs.Bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+
+	// avail blocks
+	disk.Avail = fs.Bavail * uint64(fs.Bsize)
+	disk.PercentAvail = float32((float64(disk.Avail) / float64(disk.All)) * 100)
+	return
+}

unmaintained/fix_dat/fix_dat.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	fixVolumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumeCollection = flag.String("collection", "", "the volume collection name")
+	fixVolumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+/*
+This is to resolve an one-time issue that caused inconsistency with .dat and .idx files.
+In this case, the .dat file contains all data, but some deletion caused incorrect offset.
+The .idx has all correct offsets.
+
+ 1. fix the .dat file, a new .dat_fixed file will be generated.
+    go run fix_dat.go -volumeId=9 -dir=/Users/chrislu/Downloads
+ 2. move the original .dat and .idx files to some backup folder, and rename .dat_fixed to .dat file
+    mv 9.dat_fixed 9.dat
+ 3. fix the .idx file with the "weed fix"
+    weed fix -volumeId=9 -dir=/Users/chrislu/Downloads
+*/
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	fileName := strconv.Itoa(*fixVolumeId)
+	if *fixVolumeCollection != "" {
+		fileName = *fixVolumeCollection + "_" + fileName
+	}
+	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
+	if err != nil {
+		glog.Fatalf("Read Volume Index %v", err)
+	}
+	defer indexFile.Close()
+	datFileName := path.Join(*fixVolumePath, fileName+".dat")
+	datFile, err := os.OpenFile(datFileName, os.O_RDONLY, 0644)
+	if err != nil {
+		glog.Fatalf("Read Volume Data %v", err)
+	}
+	datBackend := backend.NewDiskFile(datFile)
+	defer datBackend.Close()
+
+	newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed"))
+	if err != nil {
+		glog.Fatalf("Write New Volume Data %v", err)
+	}
+	defer newDatFile.Close()
+
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
+	if err != nil {
+		glog.Fatalf("Read Volume Data superblock %v", err)
+	}
+	newDatFile.Write(superBlock.Bytes())
+
+	iterateEntries(datBackend, indexFile, func(n *needle.Needle, offset int64) {
+		fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize)
+		_, s, _, e := n.Append(datBackend, superBlock.Version)
+		fmt.Printf("size %d error %v\n", s, e)
+	})
+
+}
+
+func iterateEntries(datBackend backend.BackendStorageFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) {
+	// start to read index file
+	var readerOffset int64
+	bytes := make([]byte, 16)
+	count, _ := idxFile.ReadAt(bytes, readerOffset)
+	readerOffset += int64(count)
+
+	// start to read dat file
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
+	if err != nil {
+		fmt.Printf("cannot read dat file super block: %v", err)
+		return
+	}
+	offset := int64(superBlock.BlockSize())
+	version := superBlock.Version
+	n, _, rest, err := needle.ReadNeedleHeader(datBackend, version, offset)
+	if err != nil {
+		fmt.Printf("cannot read needle header: %v", err)
+		return
+	}
+	fmt.Printf("Needle %+v, rest %d\n", n, rest)
+	for n != nil && count > 0 {
+		// parse index file entry
+		key := util.BytesToUint64(bytes[0:8])
+		offsetFromIndex := util.BytesToUint32(bytes[8:12])
+		sizeFromIndex := types.BytesToSize(bytes[12:16])
+		count, _ = idxFile.ReadAt(bytes, readerOffset)
+		readerOffset += int64(count)
+
+		if offsetFromIndex != 0 && offset != int64(offsetFromIndex)*8 {
+			//t := offset
+			offset = int64(offsetFromIndex) * 8
+			//fmt.Printf("Offset change %d => %d\n", t, offset)
+		}
+
+		fmt.Printf("key: %d offsetFromIndex %d n.Size %d sizeFromIndex:%d\n", key, offsetFromIndex, n.Size, sizeFromIndex)
+
+		rest = needle.NeedleBodyLength(sizeFromIndex, version)
+
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					fmt.Println("Recovered in f", r)
+				}
+			}()
+			if _, err = n.ReadNeedleBody(datBackend, version, offset+int64(types.NeedleHeaderSize), rest); err != nil {
+				fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err)
+			}
+		}()
+
+		if n.Size <= types.Size(n.DataSize) {
+			continue
+		}
+		visitNeedle(n, offset)
+
+		offset += types.NeedleHeaderSize + rest
+		//fmt.Printf("==> new entry offset %d\n", offset)
+		if n, _, rest, err = needle.ReadNeedleHeader(datBackend, version, offset); err != nil {
+			if err == io.EOF {
+				return
+			}
+
+			fmt.Printf("cannot read needle header: %v\n", err)
+			return
+		}
+		//fmt.Printf("new entry needle size:%d rest:%d\n", n.Size, rest)
+	}
+
+}

unmaintained/load_test/load_test_leveldb/load_test_leveldb.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+var (
+	dir     = flag.String("dir", "./t", "directory to store level db files")
+	useHash = flag.Bool("isHash", false, "hash the path as the key")
+	dbCount = flag.Int("dbCount", 1, "the number of leveldb")
+)
+
+func main() {
+
+	flag.Parse()
+
+	totalTenants := 300
+	totalYears := 3
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 4,
+	}
+
+	var dbs []*leveldb.DB
+	var chans []chan string
+	for d := 0; d < *dbCount; d++ {
+		dbFolder := fmt.Sprintf("%s/%02d", *dir, d)
+		os.MkdirAll(dbFolder, 0755)
+		db, err := leveldb.OpenFile(dbFolder, opts)
+		if err != nil {
+			log.Printf("filer store open dir %s: %v", *dir, err)
+			return
+		}
+		dbs = append(dbs, db)
+		chans = append(chans, make(chan string, 1024))
+	}
+
+	var wg sync.WaitGroup
+	for d := 0; d < *dbCount; d++ {
+		wg.Add(1)
+		go func(d int) {
+			defer wg.Done()
+
+			ch := chans[d]
+			db := dbs[d]
+
+			for p := range ch {
+				if *useHash {
+					insertAsHash(db, p)
+				} else {
+					insertAsFullPath(db, p)
+				}
+			}
+		}(d)
+	}
+
+	counter := int64(0)
+	lastResetTime := time.Now()
+
+	r := rand.New(rand.NewSource(35))
+
+	for y := 0; y < totalYears; y++ {
+		for m := 0; m < 12; m++ {
+			for d := 0; d < 31; d++ {
+				for h := 0; h < 24; h++ {
+					for min := 0; min < 60; min++ {
+						for i := 0; i < totalTenants; i++ {
+							p := fmt.Sprintf("tenent%03d/%4d/%02d/%02d/%02d/%02d", i, 2015+y, 1+m, 1+d, h, min)
+
+							x := r.Intn(*dbCount)
+
+							chans[x] <- p
+
+							counter++
+						}
+
+						t := time.Now()
+						if lastResetTime.Add(time.Second).Before(t) {
+							p := fmt.Sprintf("%4d/%02d/%02d/%02d/%02d", 2015+y, 1+m, 1+d, h, min)
+							fmt.Printf("%s = %4d put/sec\n", p, counter)
+							counter = 0
+							lastResetTime = t
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for d := 0; d < *dbCount; d++ {
+		close(chans[d])
+	}
+
+	wg.Wait()
+
+}
+
+func insertAsFullPath(db *leveldb.DB, p string) {
+	_, getErr := db.Get([]byte(p), nil)
+	if getErr == leveldb.ErrNotFound {
+		putErr := db.Put([]byte(p), []byte(p), nil)
+		if putErr != nil {
+			log.Printf("failed to put %s", p)
+		}
+	}
+}
+
+func insertAsHash(db *leveldb.DB, p string) {
+	key := fmt.Sprintf("%d:%s", hashToLong(p), p)
+	_, getErr := db.Get([]byte(key), nil)
+	if getErr == leveldb.ErrNotFound {
+		putErr := db.Put([]byte(key), []byte(p), nil)
+		if putErr != nil {
+			log.Printf("failed to put %s", p)
+		}
+	}
+}
+
+func hashToLong(dir string) (v int64) {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	v += int64(b[0])
+	v <<= 8
+	v += int64(b[1])
+	v <<= 8
+	v += int64(b[2])
+	v <<= 8
+	v += int64(b[3])
+	v <<= 8
+	v += int64(b[4])
+	v <<= 8
+	v += int64(b[5])
+	v <<= 8
+	v += int64(b[6])
+	v <<= 8
+	v += int64(b[7])
+
+	return
+}

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+var (
+	dir           = flag.String("dir", "/tmp", "directory to create files")
+	n             = flag.Int("n", 100, "the number of metadata")
+	tailFiler     = flag.String("filer", "localhost:8888", "the filer address")
+	isWrite       = flag.Bool("write", false, "only write")
+	writeInterval = flag.Duration("writeInterval", 0, "write interval, e.g., 1s")
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	if *isWrite {
+		startGenerateMetadata()
+		return
+	}
+
+	expected := 0
+	startSubscribeMetadata(func(event *filer_pb.SubscribeMetadataResponse) error {
+		if event.Directory != *dir {
+			return nil
+		}
+		name := event.EventNotification.NewEntry.Name
+		glog.V(0).Infof("=> %s ts:%+v", name, time.Unix(0, event.TsNs))
+		id := name[4:]
+		if x, err := strconv.Atoi(id); err == nil {
+			if x != expected {
+				return fmt.Errorf("Expected file%d Actual %s\n", expected, name)
+			}
+			expected++
+		} else {
+			return err
+		}
+		time.Sleep(10 * time.Millisecond)
+		return nil
+	})
+
+}
+
+func startGenerateMetadata() {
+	pb.WithFilerClient(false, util.RandomInt32(), pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), func(client filer_pb.SeaweedFilerClient) error {
+
+		for i := 0; i < *n; i++ {
+			name := fmt.Sprintf("file%d", i)
+			glog.V(0).Infof("write %s/%s", *dir, name)
+			if err := filer_pb.CreateEntry(context.Background(), client, &filer_pb.CreateEntryRequest{
+				Directory: *dir,
+				Entry: &filer_pb.Entry{
+					Name: name,
+				},
+			}); err != nil {
+				fmt.Printf("create entry %s: %v\n", name, err)
+				return err
+			}
+			if *writeInterval > 0 {
+				time.Sleep(*writeInterval)
+			}
+		}
+
+		return nil
+
+	})
+}
+
+func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
+
+	prefix := *dir
+	if !strings.HasSuffix(prefix, "/") {
+		prefix = prefix + "/"
+	}
+
+	metadataFollowOption := &pb.MetadataFollowOption{
+		ClientName:             "tail",
+		ClientId:               0,
+		ClientEpoch:            0,
+		SelfSignature:          0,
+		PathPrefix:             prefix,
+		AdditionalPathPrefixes: nil,
+		DirectoriesToWatch:     nil,
+		StartTsNs:              0,
+		StopTsNs:               0,
+		EventErrorType:         pb.TrivialOnError,
+	}
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), metadataFollowOption, eachEntryFunc)
+
+	if tailErr != nil {
+		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
+	}
+}

unmaintained/local-nginx/Makefile

@@ -0,0 +1,25 @@
+
+# Makefile for managing local Nginx instance
+
+# Current directory
+PWD := $(shell pwd)
+
+.PHONY: start stop reload clean
+
+logs:
+	mkdir -p logs
+
+start: logs
+	nginx -p $(PWD) -c nginx.conf
+
+stop:
+	nginx -p $(PWD) -s stop
+
+reload:
+	nginx -p $(PWD) -s reload
+
+check:
+	nginx -p $(PWD) -c nginx.conf -t
+
+clean:
+	rm -rf logs

unmaintained/local-nginx/cert.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDrTCCApWgAwIBAgIUHqubAnjJikbPDqGiqDT87mbcB+owDQYJKoZIhvcNAQEL
+BQAwZjELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
+MRUwEwYDVQQKDAxPcmdhbml6YXRpb24xDTALBgNVBAsMBFVuaXQxEjAQBgNVBAMM
+CWxvY2FsaG9zdDAeFw0yNjAxMDgwODAyMzJaFw0yNzAxMDgwODAyMzJaMGYxCzAJ
+BgNVBAYTAlVTMQ4wDAYDVQQIDAVTdGF0ZTENMAsGA1UEBwwEQ2l0eTEVMBMGA1UE
+CgwMT3JnYW5pemF0aW9uMQ0wCwYDVQQLDARVbml0MRIwEAYDVQQDDAlsb2NhbGhv
+c3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC3gozRVkW8EFsOv75Q
+AgzN0RlOkt+cyesmI71YF0NXOGlYG68Ibb4H7yemX1z663pK3e2MK0k94innW02q
+0TcBvlu1Emv3Uo+iSTC6hkWR51ZkbfJDjERpO37fZhmdGexHAR2JOLGGTkTkbsRu
+/lKg8HBHKcEL2NBI1D6LsOU3n+m9p0eSytzE92qY4hRtxaPgK4UUpMaEq64v8NDk
+HfMFjLXmuxCSUpRTowh4MCcSS1hJCD7SJmsdXHOTQc3dgc/WJOP1rkvCZRUHVaR6
+ujAIsUo9R2esevrWvxZGXFmJJDuq7u1SNnkArQludD93kovkITjb8+KsdhFGkMJR
+xzdxAgMBAAGjUzBRMB0GA1UdDgQWBBSaoIBAFJGjfUpC04hJ0AlMVBuXOzAfBgNV
+HSMEGDAWgBSaoIBAFJGjfUpC04hJ0AlMVBuXOzAPBgNVHRMBAf8EBTADAQH/MA0G
+CSqGSIb3DQEBCwUAA4IBAQCSuLPsFF4ETql1OLr6p5FqzP6+Cg8VVkzaMJskUBA/
+MEwZij+wR6teoAv8AzptByHoycOLRjYXDgweL5Z37Vr1Fo8bSqzGPqrGh6YkBNTZ
+heXFf8g5lOWiv3OypxVtkEFD1tYhdU//5tnipHh08dEjZBuTumMMYhr6xcYBecI7
+Lw6uo+TRrPrPLgcJU53Sivm8c2fGVexGfcklvtW4qvkQik+r9BfnoJ/f12PBq2J5
+nKEQFR9EwULnkqREf0Iw/mDPS275brhdDA8JNlB49GGcRGcYu9tOVGDX5rfdIwXB
+PTaWwrOoY5F4NVPlY7d5E8eLHSt8/zd+56w4g5YaQUaH
+-----END CERTIFICATE-----

unmaintained/local-nginx/gen_certs.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+# Generate a private key
+openssl genrsa -out key.pem 2048
+
+# Generate a self-signed certificate
+openssl req -new -x509 -key key.pem -out cert.pem -days 365 \
+    -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=localhost"
+
+echo "Certificate (cert.pem) and Key (key.pem) generated successfully in $(pwd)"

unmaintained/local-nginx/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3gozRVkW8EFsO
+v75QAgzN0RlOkt+cyesmI71YF0NXOGlYG68Ibb4H7yemX1z663pK3e2MK0k94inn
+W02q0TcBvlu1Emv3Uo+iSTC6hkWR51ZkbfJDjERpO37fZhmdGexHAR2JOLGGTkTk
+bsRu/lKg8HBHKcEL2NBI1D6LsOU3n+m9p0eSytzE92qY4hRtxaPgK4UUpMaEq64v
+8NDkHfMFjLXmuxCSUpRTowh4MCcSS1hJCD7SJmsdXHOTQc3dgc/WJOP1rkvCZRUH
+VaR6ujAIsUo9R2esevrWvxZGXFmJJDuq7u1SNnkArQludD93kovkITjb8+KsdhFG
+kMJRxzdxAgMBAAECggEAUUf932r7EEWeT3YWcZG24iLPtsHPUb8N7KK/5xjGtwdB
+gckhSfZxyUngogxzPqC2VhrecMtrcYGPVe5XBGYOmzMZo6Y5NB7Zh+QrNTLr9agl
+1+3q9IhYdNPpakHq7m7g/MI9tt3po7dfRF7bmt4C804MLTkNTJYIC3oXCgCcRnpJ
+NYLtombx6FNzivUMB9/FGoR3MeVsenCNmbfGyqssKafsoNfBnszg2XT4W9TpynA5
+ThER0ZjAu8Wi4QJ4rGuwgCqoobe9qw5xUdnerj84O/FDxY38zK92yZzxPG39O61U
+kSna0WKAhtvOqYvYe5KvIFs8AbFNBn+/mCEhaPmYEQKBgQDvXjy7maM0L8yDYNs5
+2Mups0DC6E3Xkx5k6O6/TPfBBY949ZBsAT+hDpJA119/UOxGRE/wZaQ+09blyjOD
+B8Nw0m0OOgpERIHhm+uDXpkVhFJXkBQMB7Jsc2tZKxHNY78UD4K4YcsbRGBNZryf
+6Fd6uVad0qMKJF825gN3f7d5twKBgQDEQrxBEensG6D5XWoYTqwvNfgc9BqOAWrJ
+IkX4l99n9yJUHJ+h7wcrzA1FfKVJU5LkvR+YqCxcSyil5BnQFU7s3JqnhIkwfk9R
+wS2pPXLaJS6oQQcr2yRQgP8xDDMlBNABIkw/JARXsPkdxM+y93G4s+O4GIsrOJHX
+zZWjoK74FwKBgDsKjoZjZfsIUzSQnmJNVCbefPnWXOTwd83CGql67Z1hprraLy5E
+vJIYSwDNXtPDfRPz45fTKkkI7XeugTgvoQhiDt40eLXQBA1G7TdxDOdexQSuGBPD
+jkHr0i2jTvfPQ6FKFh8ul+941RwiXCJBJ9qwfmndO7PTttyOrYo7KWFPAoGBAJHb
+jxFFAPaKag4pkKnT/j9v+p96Ayo6f2RP5nSkWlJn5slgJHsyq7+u/MYBps1iIS+C
+JrNLePx2lxUC+vQcgfWi1gVdiJ/Tn5uA2DCJDIx3Sb+d5HiBevGhjQakrHX8p6HQ
+HZ5nZBrP2bR4CmxVXd0JRFoTU4qhljIuBeH/j0mtAoGAN9Hf3txLTm7PbuNy0xOM
+Us+nHsVjIzZvZ6H413kzs85QIf22lw0JEoRBnhtZmxATERrvYJ9YtyZH0Tf+b+eU
+TioM9wDi62tDjgMK+jgoIPid4hgMdHzdjjxJ6kZ9sFjR/3dOKxDwe8jU1YybFqxH
+yYRlpVwN2c++XrbOPfE+m7E=
+-----END PRIVATE KEY-----

unmaintained/local-nginx/mime.types

@@ -0,0 +1,85 @@
+
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/javascript                js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg svgz;
+    image/webp                            webp;
+
+    application/font-woff                 woff;
+    application/java-archive              jar war ear;
+    application/json                      json;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.apple.mpegurl         m3u8;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-fontobject         eot;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/xhtml+xml                 xhtml;
+    application/xspf+xml                  xspf;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/ogg                             ogg;
+    audio/x-m4a                           m4a;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mp2t                            ts;
+    video/mp4                             mp4;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/webm                            webm;
+    video/x-flv                           flv;
+    video/x-m4v                           m4v;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asf x-ms-asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

unmaintained/local-nginx/nginx.conf

@@ -0,0 +1,71 @@
+
+worker_processes  1;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    sendfile        on;
+    keepalive_timeout  65;
+
+    log_format debug '$remote_addr - $remote_user [$time_local] '
+                      '"$request" $status $body_bytes_sent '
+                      'request_body: "$request_body" '
+                      'upstream_status: "$upstream_status" '
+                      'upstream_response_time: "$upstream_response_time" '
+                      'resp_content_type: "$sent_http_content_type" '
+                      'resp_content_length: "$sent_http_content_length"';
+
+    access_log logs/access.log debug;
+    error_log logs/error.log debug;
+
+    # Standard Configuration from Wiki
+    upstream seaweedfs {
+        server 127.0.0.1:8333;
+        keepalive 20;
+    }
+
+    server {
+        listen       443 ssl;
+        server_name  localhost;
+
+        ssl_certificate      cert.pem;
+        ssl_certificate_key  key.pem;
+
+        ssl_session_cache    shared:SSL:1m;
+        ssl_session_timeout  5m;
+
+        ssl_ciphers  HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers  on;
+
+        ignore_invalid_headers off;
+        client_max_body_size 0;
+        client_body_buffer_size 128k;
+        proxy_buffering off;
+
+        location / {
+            proxy_pass http://seaweedfs;
+            
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+            proxy_set_header X-Forwarded-Port $server_port;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            proxy_request_buffering on;
+            chunked_transfer_encoding off;
+
+            # Log response headers
+            add_header X-Debug-Status $upstream_status;
+            add_header X-Debug-Time $upstream_response_time;
+        }
+    }
+}

unmaintained/remove_duplicate_fids/remove_duplicate_fids.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	volumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	volumeCollection = flag.String("collection", "", "the volume collection name")
+	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+func Checksum(n *needle.Needle) string {
+	return fmt.Sprintf("%s%x", n.Id, n.Cookie)
+}
+
+type VolumeFileScanner4SeeDat struct {
+	version needle.Version
+	block   super_block.SuperBlock
+
+	dir        string
+	hashes     map[string]bool
+	dat        *os.File
+	datBackend backend.BackendStorageFile
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
+	scanner.block = superBlock
+	return nil
+
+}
+func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
+
+	if scanner.datBackend == nil {
+		newFileName := filepath.Join(*volumePath, "dat_fixed")
+		newDatFile, err := os.Create(newFileName)
+		if err != nil {
+			glog.Fatalf("Write New Volume Data %v", err)
+		}
+		scanner.datBackend = backend.NewDiskFile(newDatFile)
+		scanner.datBackend.WriteAt(scanner.block.Bytes(), 0)
+	}
+
+	checksum := Checksum(n)
+
+	if scanner.hashes[checksum] {
+		glog.V(0).Infof("duplicate checksum:%s fid:%d,%s%x @ offset:%d", checksum, *volumeId, n.Id, n.Cookie, offset)
+		return nil
+	}
+	scanner.hashes[checksum] = true
+
+	_, s, _, e := n.Append(scanner.datBackend, scanner.version)
+	fmt.Printf("size %d error %v\n", s, e)
+
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	vid := needle.VolumeId(*volumeId)
+
+	outpath, _ := filepath.Abs(filepath.Dir(os.Args[0]))
+
+	scanner := &VolumeFileScanner4SeeDat{
+		dir:    filepath.Join(outpath, "out"),
+		hashes: map[string]bool{},
+	}
+
+	if _, err := os.Stat(scanner.dir); err != nil {
+		if err := os.MkdirAll(scanner.dir, os.ModePerm); err != nil {
+			glog.Fatalf("could not create output dir : %s", err)
+		}
+	}
+
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
+	if err != nil {
+		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+	}
+
+}

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"math/rand"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc"
+)
+
+var (
+	master           = flag.String("master", "127.0.0.1:9333", "the master server")
+	repeat           = flag.Int("n", 5, "repeat how many times")
+	garbageThreshold = flag.Float64("garbageThreshold", 0.3, "garbageThreshold")
+	replication      = flag.String("replication", "", "replication 000, 001, 002, etc")
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	util.LoadSecurityConfiguration()
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	genFile(grpcDialOption, 0)
+
+	go func() {
+		for {
+			println("vacuum threshold", *garbageThreshold)
+			_, _, err := util_http.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold))
+			if err != nil {
+				log.Fatalf("vacuum: %v", err)
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
+	for i := 0; i < *repeat; i++ {
+		// create 2 files, and delete one of them
+
+		assignResult, targetUrl := genFile(grpcDialOption, i)
+
+		util_http.Delete(targetUrl, string(assignResult.Auth))
+
+	}
+
+}
+
+func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
+	assignResult, err := operation.Assign(context.Background(), func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{
+		Count:       1,
+		Replication: *replication,
+	})
+	if err != nil {
+		log.Fatalf("assign: %v", err)
+	}
+
+	data := make([]byte, 1024)
+	rand.Read(data)
+
+	targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
+
+	uploadOption := &operation.UploadOption{
+		UploadUrl:         targetUrl,
+		Filename:          fmt.Sprintf("test%d", i),
+		Cipher:            false,
+		IsInputCompressed: true,
+		MimeType:          "bench/test",
+		PairMap:           nil,
+		Jwt:               assignResult.Auth,
+	}
+
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		log.Fatalf("upload: %v", err)
+	}
+
+	_, err = uploader.UploadData(context.Background(), data, uploadOption)
+	if err != nil {
+		log.Fatalf("upload: %v", err)
+	}
+	return assignResult, targetUrl
+}

unmaintained/s3/benchmark/hsbench.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+hsbench -a accesstoken -s secret -z 4K -d 10 -t 10 -b 10 -u http://localhost:8333 -m "cxipgdx" -bp "hsbench-"

unmaintained/s3/presigned_put/presigned_put.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+// Downloads an item from an S3 Bucket in the region configured in the shared config
+// or AWS_REGION environment variable.
+//
+// Usage:
+//
+//	go run presigned_put.go
+//
+// For this exampl to work, the domainName is needd
+//
+//	weed s3 -domainName=localhost
+func main() {
+	util_http.InitGlobalHttpClient()
+
+	h := md5.New()
+	content := strings.NewReader(stringContent)
+	content.WriteTo(h)
+
+	// Initialize a session in us-west-2 that the SDK will use to load
+	// credentials from the shared credentials file ~/.aws/credentials.
+	sess, err := session.NewSession(&aws.Config{
+		Region:   aws.String("us-east-1"),
+		Endpoint: aws.String("http://localhost:8333"),
+	})
+
+	// Create S3 service client
+	svc := s3.New(sess)
+
+	putRequest, output := svc.PutObjectRequest(&s3.PutObjectInput{
+		Bucket: aws.String("dev"),
+		Key:    aws.String("testKey"),
+	})
+	fmt.Printf("output: %+v\n", output)
+
+	md5s := base64.StdEncoding.EncodeToString(h.Sum(nil))
+	putRequest.HTTPRequest.Header.Set("Content-MD5", md5s)
+
+	url, err := putRequest.Presign(15 * time.Minute)
+	if err != nil {
+		fmt.Println("error presigning request", err)
+		return
+	}
+
+	fmt.Println(url)
+
+	req, err := http.NewRequest("PUT", url, strings.NewReader(stringContent))
+	req.Header.Set("Content-MD5", md5s)
+	if err != nil {
+		fmt.Println("error creating request", url)
+		return
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Printf("error put request: %v\n", err)
+		return
+	}
+	defer util_http.CloseResponse(resp)
+	fmt.Printf("response: %+v\n", resp)
+}
+
+var stringContent = `Generate a Pre-Signed URL for an Amazon S3 PUT Operation with a Specific Payload
+You can generate a pre-signed URL for a PUT operation that checks whether users upload the correct content. When the SDK pre-signs a request, it computes the checksum of the request body and generates an MD5 checksum that is included in the pre-signed URL. Users must upload the same content that produces the same MD5 checksum generated by the SDK; otherwise, the operation fails. This is not the Content-MD5, but the signature. To enforce Content-MD5, simply add the header to the request.
+
+The following example adds a Body field to generate a pre-signed PUT operation that requires a specific payload to be uploaded by users.
+`

unmaintained/see_dat/see_dat.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"flag"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	volumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	volumeCollection = flag.String("collection", "", "the volume collection name")
+	volumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+type VolumeFileScanner4SeeDat struct {
+	version needle.Version
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
+	scanner.version = superBlock.Version
+	return nil
+
+}
+func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
+	return true
+}
+
+func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
+	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
+	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
+		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
+	return nil
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	vid := needle.VolumeId(*volumeId)
+
+	scanner := &VolumeFileScanner4SeeDat{}
+	err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
+	if err != nil {
+		glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
+	}
+}

unmaintained/see_idx/see_idx.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	fixVolumePath       = flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumeCollection = flag.String("collection", "", "the volume collection name")
+	fixVolumeId         = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+)
+
+/*
+This is to see content in .idx files.
+
+	see_idx -v=4 -volumeId=9 -dir=/Users/chrislu/Downloads
+*/
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	fileName := strconv.Itoa(*fixVolumeId)
+	if *fixVolumeCollection != "" {
+		fileName = *fixVolumeCollection + "_" + fileName
+	}
+	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_RDONLY, 0644)
+	if err != nil {
+		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
+	}
+	defer indexFile.Close()
+
+	idx.WalkIndexFile(indexFile, 0, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+		fmt.Printf("key:%v offset:%v size:%v(%v)\n", key, offset, size, util.BytesToHumanReadable(uint64(size)))
+		return nil
+	})
+
+}

unmaintained/see_log_entry/see_log_entry.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	logdataFile = flag.String("logdata", "", "log data file saved under "+filer.SystemLogDir)
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	dst, err := os.OpenFile(*logdataFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *logdataFile, err)
+	}
+	defer dst.Close()
+
+	err = walkLogEntryFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *logdataFile, err)
+	}
+
+}
+
+func walkLogEntryFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		logEntry := &filer_pb.LogEntry{}
+		err := proto.Unmarshal(data, logEntry)
+		if err != nil {
+			log.Printf("unexpected unmarshal filer_pb.LogEntry: %v", err)
+			return nil
+		}
+
+		event := &filer_pb.SubscribeMetadataResponse{}
+		err = proto.Unmarshal(logEntry.Data, event)
+		if err != nil {
+			log.Printf("unexpected unmarshal filer_pb.SubscribeMetadataResponse: %v", err)
+			return nil
+		}
+
+		fmt.Printf("event: %+v\n", event)
+
+	}
+
+}

unmaintained/see_meta/see_meta.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	metaFile = flag.String("meta", "", "meta file generated via fs.meta.save")
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	dst, err := os.OpenFile(*metaFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *metaFile, err)
+	}
+	defer dst.Close()
+
+	err = walkMetaFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *metaFile, err)
+	}
+
+}
+
+func walkMetaFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		fullEntry := &filer_pb.FullEntry{}
+		if err := proto.Unmarshal(data, fullEntry); err != nil {
+			return err
+		}
+
+		fmt.Fprintf(os.Stdout, "file %s %v\n", util.FullPath(fullEntry.Dir).Child(fullEntry.Entry.Name), fullEntry.Entry.Attributes.String())
+		for i, chunk := range fullEntry.Entry.GetChunks() {
+			fmt.Fprintf(os.Stdout, "  chunk: %d %v %d,%x%08x\n", i+1, chunk, chunk.Fid.VolumeId, chunk.Fid.FileKey, chunk.Fid.Cookie)
+		}
+
+	}
+
+}

unmaintained/stream_read_volume/stream_read_volume.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc"
+)
+
+var (
+	volumeServer   = flag.String("volumeServer", "localhost:8080", "a volume server")
+	volumeId       = flag.Int("volumeId", -1, "a volume id to stream read")
+	grpcDialOption grpc.DialOption
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	util.LoadSecurityConfiguration()
+	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	vid := uint32(*volumeId)
+
+	eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
+		fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
+		return nil
+	}
+
+	err := operation.WithVolumeServerClient(true, pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+			VolumeIds: []uint32{vid},
+		})
+		if err != nil {
+			return err
+		}
+		for {
+			resp, err := copyFileClient.Recv()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if err = eachNeedleFunc(resp); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("read %s: %v\n", *volumeServer, err)
+	}
+
+}

unmaintained/stress/ec_stress_runner.go

@@ -0,0 +1,515 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	mrand "math/rand"
+	"net/http"
+	"net/url"
+	"os/signal"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+)
+
+type config struct {
+	MasterAddresses    []string
+	FilerURL           string
+	PathPrefix         string
+	Collection         string
+	FileSizeBytes      int64
+	BatchSize          int
+	WriteInterval      time.Duration
+	CleanupInterval    time.Duration
+	EcMinAge           time.Duration
+	MaxCleanupPerCycle int
+	RequestTimeout     time.Duration
+	MaxRuntime         time.Duration
+	DryRun             bool
+}
+
+type runner struct {
+	cfg config
+
+	httpClient     *http.Client
+	grpcDialOption grpc.DialOption
+
+	mu            sync.Mutex
+	sequence      int64
+	ecFirstSeenAt map[uint32]time.Time
+	rng           *mrand.Rand
+}
+
+type ecVolumeInfo struct {
+	Collection string
+	NodeShards map[pb.ServerAddress][]uint32
+}
+
+type ecCleanupCandidate struct {
+	VolumeID    uint32
+	FirstSeenAt time.Time
+	Info        *ecVolumeInfo
+}
+
+func main() {
+	cfg, err := loadConfig()
+	if err != nil {
+		log.Fatalf("invalid flags: %v", err)
+	}
+
+	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+	defer stop()
+
+	if cfg.MaxRuntime > 0 {
+		runCtx, cancel := context.WithTimeout(ctx, cfg.MaxRuntime)
+		defer cancel()
+		ctx = runCtx
+	}
+
+	r := &runner{
+		cfg:            cfg,
+		httpClient:     &http.Client{Timeout: cfg.RequestTimeout},
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+		ecFirstSeenAt:  make(map[uint32]time.Time),
+		rng:            mrand.New(mrand.NewSource(time.Now().UnixNano())),
+	}
+
+	log.Printf(
+		"starting EC stress runner: masters=%s filer=%s prefix=%s collection=%s file_size=%d batch=%d write_interval=%s cleanup_interval=%s ec_min_age=%s max_cleanup=%d dry_run=%v",
+		strings.Join(cfg.MasterAddresses, ","),
+		cfg.FilerURL,
+		cfg.PathPrefix,
+		cfg.Collection,
+		cfg.FileSizeBytes,
+		cfg.BatchSize,
+		cfg.WriteInterval,
+		cfg.CleanupInterval,
+		cfg.EcMinAge,
+		cfg.MaxCleanupPerCycle,
+		cfg.DryRun,
+	)
+
+	if err := r.run(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+		log.Fatalf("runner stopped with error: %v", err)
+	}
+
+	log.Printf("runner stopped")
+}
+
+func loadConfig() (config, error) {
+	var masters string
+	cfg := config{}
+
+	flag.StringVar(&masters, "masters", "127.0.0.1:9333", "comma-separated master server addresses")
+	flag.StringVar(&cfg.FilerURL, "filer", "http://127.0.0.1:8888", "filer base URL")
+	flag.StringVar(&cfg.PathPrefix, "path_prefix", "/tmp/ec-stress", "filer path prefix for generated files")
+	flag.StringVar(&cfg.Collection, "collection", "ec_stress", "target collection for stress data")
+
+	fileSizeMB := flag.Int("file_size_mb", 8, "size per generated file in MB")
+	flag.IntVar(&cfg.BatchSize, "batch_size", 4, "files generated per write cycle")
+	flag.DurationVar(&cfg.WriteInterval, "write_interval", 5*time.Second, "interval between write cycles")
+	flag.DurationVar(&cfg.CleanupInterval, "cleanup_interval", 2*time.Minute, "interval between EC cleanup cycles")
+	flag.DurationVar(&cfg.EcMinAge, "ec_min_age", 30*time.Minute, "minimum observed EC age before deletion")
+	flag.IntVar(&cfg.MaxCleanupPerCycle, "max_cleanup_per_cycle", 4, "maximum EC volumes deleted per cleanup cycle")
+	flag.DurationVar(&cfg.RequestTimeout, "request_timeout", 20*time.Second, "HTTP/gRPC request timeout")
+	flag.DurationVar(&cfg.MaxRuntime, "max_runtime", 0, "maximum run duration; 0 means run until interrupted")
+	flag.BoolVar(&cfg.DryRun, "dry_run", false, "log actions without deleting EC shards")
+	flag.Parse()
+
+	cfg.MasterAddresses = splitNonEmpty(masters)
+	cfg.FileSizeBytes = int64(*fileSizeMB) * 1024 * 1024
+
+	if len(cfg.MasterAddresses) == 0 {
+		return cfg, fmt.Errorf("at least one master is required")
+	}
+	if cfg.FileSizeBytes <= 0 {
+		return cfg, fmt.Errorf("file_size_mb must be positive")
+	}
+	if cfg.BatchSize <= 0 {
+		return cfg, fmt.Errorf("batch_size must be positive")
+	}
+	if cfg.WriteInterval <= 0 {
+		return cfg, fmt.Errorf("write_interval must be positive")
+	}
+	if cfg.CleanupInterval <= 0 {
+		return cfg, fmt.Errorf("cleanup_interval must be positive")
+	}
+	if cfg.EcMinAge < 0 {
+		return cfg, fmt.Errorf("ec_min_age must be zero or positive")
+	}
+	// Note: EcMinAge == 0 intentionally disables the age guard, making EC volumes eligible for cleanup immediately.
+	if cfg.MaxCleanupPerCycle <= 0 {
+		return cfg, fmt.Errorf("max_cleanup_per_cycle must be positive")
+	}
+	if cfg.RequestTimeout <= 0 {
+		return cfg, fmt.Errorf("request_timeout must be positive")
+	}
+
+	cfg.PathPrefix = ensureLeadingSlash(strings.TrimSpace(cfg.PathPrefix))
+	cfg.Collection = strings.TrimSpace(cfg.Collection)
+
+	cfg.FilerURL = strings.TrimRight(strings.TrimSpace(cfg.FilerURL), "/")
+	if cfg.FilerURL == "" {
+		return cfg, fmt.Errorf("filer URL is required")
+	}
+
+	if _, err := url.ParseRequestURI(cfg.FilerURL); err != nil {
+		return cfg, fmt.Errorf("invalid filer URL %q: %w", cfg.FilerURL, err)
+	}
+
+	return cfg, nil
+}
+
+func (r *runner) run(ctx context.Context) error {
+	writeTicker := time.NewTicker(r.cfg.WriteInterval)
+	defer writeTicker.Stop()
+
+	cleanupTicker := time.NewTicker(r.cfg.CleanupInterval)
+	defer cleanupTicker.Stop()
+
+	r.runWriteCycle(ctx)
+	r.runCleanupCycle(ctx)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-writeTicker.C:
+			r.runWriteCycle(ctx)
+		case <-cleanupTicker.C:
+			r.runCleanupCycle(ctx)
+		}
+	}
+}
+
+func (r *runner) runWriteCycle(ctx context.Context) {
+	for i := 0; i < r.cfg.BatchSize; i++ {
+		if ctx.Err() != nil {
+			return
+		}
+		if err := r.uploadOneFile(ctx); err != nil {
+			log.Printf("upload failed: %v", err)
+		}
+	}
+}
+
+func (r *runner) uploadOneFile(ctx context.Context) error {
+	sequence := r.nextSequence()
+	filePath := path.Join(r.cfg.PathPrefix, fmt.Sprintf("ec-stress-%d-%d.bin", time.Now().UnixNano(), sequence))
+	fileURL := r.cfg.FilerURL + filePath
+	if r.cfg.Collection != "" {
+		fileURL += "?collection=" + url.QueryEscape(r.cfg.Collection)
+	}
+
+	uploadCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
+	defer cancel()
+
+	body := io.LimitReader(r.rng, r.cfg.FileSizeBytes)
+	request, err := http.NewRequestWithContext(uploadCtx, http.MethodPut, fileURL, body)
+	if err != nil {
+		return err
+	}
+	request.ContentLength = r.cfg.FileSizeBytes
+	request.Header.Set("Content-Type", "application/octet-stream")
+
+	response, err := r.httpClient.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	io.Copy(io.Discard, response.Body)
+	if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
+		return fmt.Errorf("upload %s returned %s", filePath, response.Status)
+	}
+
+	log.Printf("uploaded %s size=%d", filePath, r.cfg.FileSizeBytes)
+	return nil
+}
+
+func (r *runner) runCleanupCycle(ctx context.Context) {
+	volumeList, err := r.fetchVolumeList(ctx)
+	if err != nil {
+		log.Printf("cleanup skipped: fetch volume list failed: %v", err)
+		return
+	}
+	if volumeList == nil || volumeList.TopologyInfo == nil {
+		log.Printf("cleanup skipped: topology is empty")
+		return
+	}
+
+	ecVolumes := collectEcVolumes(volumeList.TopologyInfo, r.cfg.Collection)
+	candidates := r.selectCleanupCandidates(ecVolumes)
+	if len(candidates) == 0 {
+		log.Printf("cleanup: no EC volume candidate aged >= %s in collection=%q", r.cfg.EcMinAge, r.cfg.Collection)
+		return
+	}
+
+	log.Printf("cleanup: deleting up to %d EC volumes (found=%d)", r.cfg.MaxCleanupPerCycle, len(candidates))
+	deleted := 0
+	for _, candidate := range candidates {
+		if ctx.Err() != nil {
+			return
+		}
+
+		if r.cfg.DryRun {
+			log.Printf(
+				"cleanup dry-run: would delete EC volume=%d collection=%q seen_for=%s nodes=%d",
+				candidate.VolumeID,
+				candidate.Info.Collection,
+				time.Since(candidate.FirstSeenAt).Round(time.Second),
+				len(candidate.Info.NodeShards),
+			)
+			continue
+		}
+
+		if err := r.deleteEcVolume(ctx, candidate.VolumeID, candidate.Info); err != nil {
+			log.Printf("cleanup volume=%d failed: %v", candidate.VolumeID, err)
+			continue
+		}
+
+		deleted++
+		r.mu.Lock()
+		delete(r.ecFirstSeenAt, candidate.VolumeID)
+		r.mu.Unlock()
+		log.Printf("cleanup volume=%d completed", candidate.VolumeID)
+	}
+
+	log.Printf("cleanup finished: deleted=%d attempted=%d", deleted, len(candidates))
+}
+
+func (r *runner) fetchVolumeList(ctx context.Context) (*master_pb.VolumeListResponse, error) {
+	var lastErr error
+	for _, master := range r.cfg.MasterAddresses {
+		masterAddress := strings.TrimSpace(master)
+		if masterAddress == "" {
+			continue
+		}
+
+		var response *master_pb.VolumeListResponse
+		err := pb.WithMasterClient(context.Background(), false, pb.ServerAddress(masterAddress), r.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
+			callCtx, cancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
+			defer cancel()
+
+			resp, callErr := client.VolumeList(callCtx, &master_pb.VolumeListRequest{})
+			if callErr != nil {
+				return callErr
+			}
+			response = resp
+			return nil
+		})
+		if err == nil {
+			return response, nil
+		}
+
+		lastErr = err
+	}
+
+	if lastErr == nil {
+		lastErr = fmt.Errorf("no valid master address")
+	}
+	return nil, lastErr
+}
+
+func collectEcVolumes(topology *master_pb.TopologyInfo, collection string) map[uint32]*ecVolumeInfo {
+	normalizedCollection := strings.TrimSpace(collection)
+	volumeShardSets := make(map[uint32]map[pb.ServerAddress]map[uint32]struct{})
+	volumeCollection := make(map[uint32]string)
+
+	for _, dc := range topology.GetDataCenterInfos() {
+		for _, rack := range dc.GetRackInfos() {
+			for _, node := range rack.GetDataNodeInfos() {
+				server := pb.NewServerAddressFromDataNode(node)
+				for _, disk := range node.GetDiskInfos() {
+					for _, shardInfo := range disk.GetEcShardInfos() {
+						if shardInfo == nil || shardInfo.Id == 0 {
+							continue
+						}
+						if normalizedCollection != "" && strings.TrimSpace(shardInfo.Collection) != normalizedCollection {
+							continue
+						}
+
+						shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).IdsUint32()
+						if len(shards) == 0 {
+							continue
+						}
+
+						perVolume := volumeShardSets[shardInfo.Id]
+						if perVolume == nil {
+							perVolume = make(map[pb.ServerAddress]map[uint32]struct{})
+							volumeShardSets[shardInfo.Id] = perVolume
+						}
+						perNode := perVolume[server]
+						if perNode == nil {
+							perNode = make(map[uint32]struct{})
+							perVolume[server] = perNode
+						}
+						for _, shardID := range shards {
+							perNode[shardID] = struct{}{}
+						}
+						volumeCollection[shardInfo.Id] = shardInfo.Collection
+					}
+				}
+			}
+		}
+	}
+
+	result := make(map[uint32]*ecVolumeInfo, len(volumeShardSets))
+	for volumeID, perNode := range volumeShardSets {
+		info := &ecVolumeInfo{
+			Collection: volumeCollection[volumeID],
+			NodeShards: make(map[pb.ServerAddress][]uint32, len(perNode)),
+		}
+		for server, shardSet := range perNode {
+			shardIDs := make([]uint32, 0, len(shardSet))
+			for shardID := range shardSet {
+				shardIDs = append(shardIDs, shardID)
+			}
+			sort.Slice(shardIDs, func(i, j int) bool { return shardIDs[i] < shardIDs[j] })
+			info.NodeShards[server] = shardIDs
+		}
+		result[volumeID] = info
+	}
+
+	return result
+}
+
+func (r *runner) selectCleanupCandidates(ecVolumes map[uint32]*ecVolumeInfo) []ecCleanupCandidate {
+	now := time.Now()
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	for volumeID := range ecVolumes {
+		if _, exists := r.ecFirstSeenAt[volumeID]; !exists {
+			r.ecFirstSeenAt[volumeID] = now
+		}
+	}
+	for volumeID := range r.ecFirstSeenAt {
+		if _, exists := ecVolumes[volumeID]; !exists {
+			delete(r.ecFirstSeenAt, volumeID)
+		}
+	}
+
+	candidates := make([]ecCleanupCandidate, 0, len(ecVolumes))
+	for volumeID, info := range ecVolumes {
+		firstSeenAt := r.ecFirstSeenAt[volumeID]
+		if r.cfg.EcMinAge > 0 && now.Sub(firstSeenAt) < r.cfg.EcMinAge {
+			continue
+		}
+		candidates = append(candidates, ecCleanupCandidate{
+			VolumeID:    volumeID,
+			FirstSeenAt: firstSeenAt,
+			Info:        info,
+		})
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		if candidates[i].FirstSeenAt.Equal(candidates[j].FirstSeenAt) {
+			return candidates[i].VolumeID < candidates[j].VolumeID
+		}
+		return candidates[i].FirstSeenAt.Before(candidates[j].FirstSeenAt)
+	})
+
+	if len(candidates) > r.cfg.MaxCleanupPerCycle {
+		candidates = candidates[:r.cfg.MaxCleanupPerCycle]
+	}
+	return candidates
+}
+
+func (r *runner) deleteEcVolume(ctx context.Context, volumeID uint32, info *ecVolumeInfo) error {
+	if info == nil {
+		return fmt.Errorf("ec volume %d has no topology info", volumeID)
+	}
+
+	failureCount := 0
+	for server, shardIDs := range info.NodeShards {
+		err := pb.WithVolumeServerClient(false, server, r.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+			unmountCtx, unmountCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
+			defer unmountCancel()
+			if _, err := client.VolumeEcShardsUnmount(unmountCtx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+				VolumeId: volumeID,
+				ShardIds: shardIDs,
+			}); err != nil {
+				log.Printf("volume %d ec shards unmount on %s failed: %v", volumeID, server, err)
+			}
+
+			if len(shardIDs) > 0 {
+				deleteCtx, deleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
+				defer deleteCancel()
+				if _, err := client.VolumeEcShardsDelete(deleteCtx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+					VolumeId:   volumeID,
+					Collection: r.cfg.Collection,
+					ShardIds:   shardIDs,
+				}); err != nil {
+					return err
+				}
+			}
+
+			finalDeleteCtx, finalDeleteCancel := context.WithTimeout(ctx, r.cfg.RequestTimeout)
+			defer finalDeleteCancel()
+			if _, err := client.VolumeDelete(finalDeleteCtx, &volume_server_pb.VolumeDeleteRequest{
+				VolumeId: volumeID,
+			}); err != nil {
+				log.Printf("volume %d delete on %s failed: %v", volumeID, server, err)
+			}
+			return nil
+		})
+
+		if err != nil {
+			failureCount++
+			log.Printf("cleanup volume=%d server=%s shards=%v failed: %v", volumeID, server, shardIDs, err)
+		}
+	}
+
+	if failureCount == len(info.NodeShards) && failureCount > 0 {
+		return fmt.Errorf("all shard deletions failed for volume %d", volumeID)
+	}
+	if failureCount > 0 {
+		return fmt.Errorf("partial shard deletion failure for volume %d", volumeID)
+	}
+	return nil
+}
+
+func (r *runner) nextSequence() int64 {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.sequence++
+	return r.sequence
+}
+
+func splitNonEmpty(value string) []string {
+	parts := strings.Split(value, ",")
+	result := make([]string, 0, len(parts))
+	for _, part := range parts {
+		trimmed := strings.TrimSpace(part)
+		if trimmed != "" {
+			result = append(result, trimmed)
+		}
+	}
+	return result
+}
+
+func ensureLeadingSlash(value string) string {
+	if value == "" {
+		return "/"
+	}
+	if strings.HasPrefix(value, "/") {
+		return value
+	}
+	return "/" + value
+}

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	size        = flag.Int("size", 1024, "file size")
+	concurrency = flag.Int("c", 4, "concurrent number of uploads")
+	times       = flag.Int("n", 1024, "repeated number of times")
+	fileCount   = flag.Int("fileCount", 1, "number of files to write")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	data := make([]byte, *size)
+	println("data len", len(data))
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func(x int) {
+			defer wg.Done()
+
+			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
+
+			for t := 0; t < *times; t++ {
+				for f := 0; f < *fileCount; f++ {
+					fn := r.Intn(*fileCount)
+					if size, err := uploadFileToFiler(data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					} else {
+						log.Fatalf("client %d upload %d times: %v", x, t, err)
+					}
+				}
+			}
+		}(x)
+	}
+
+	go func() {
+		ticker := time.NewTicker(1000 * time.Millisecond)
+		defer ticker.Stop()
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(data []byte, filename, destination string) (size int64, err error) {
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
+	}
+
+	part.Write(data)
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
+	}
+
+	uri := destination + filename
+
+	request, err := http.NewRequest(http.MethodPost, uri, body)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	// request.Close = true  // can not use this, which do not reuse http connection, impacting filer->volume also.
+
+	resp, err := util_http.GetGlobalHttpClient().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return int64(len(data)), nil
+}

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	dir         = flag.String("dir", ".", "upload files under this directory")
+	concurrency = flag.Int("c", 1, "concurrent number of uploads")
+	times       = flag.Int("n", 1, "repeated number of times")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	var fileNames []string
+
+	files, err := os.ReadDir(*dir)
+	if err != nil {
+		log.Fatalf("fail to read dir %v: %v", *dir, err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
+	}
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+			rand.Shuffle(len(fileNames), func(i, j int) {
+				fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
+			})
+			for t := 0; t < *times; t++ {
+				for _, filename := range fileNames {
+					if size, err := uploadFileToFiler(filename, *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(filename, destination string) (size int64, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", file.Name())
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
+	}
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	uri := destination + file.Name()
+
+	request, err := http.NewRequest(http.MethodPost, uri, body)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	}
+
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := util_http.GetGlobalHttpClient().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return fi.Size(), nil
+}

unmaintained/stress_filer_upload/write_files/write_files.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+)
+
+var (
+	minSize   = flag.Int("minSize", 1024, "min file size")
+	maxSize   = flag.Int("maxSize", 3*1024*1024, "max file size")
+	fileCount = flag.Int("n", 1, "number of files to write")
+	blockSize = flag.Int("blockSizeKB", 4, "write block size")
+	toDir     = flag.String("dir", ".", "destination directory")
+)
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+func main() {
+
+	flag.Parse()
+
+	block := make([]byte, *blockSize*1024)
+
+	for i := 0; i < *fileCount; i++ {
+
+		f, err := os.Create(fmt.Sprintf("%s/file%05d", *toDir, i))
+		check(err)
+
+		fileSize := *minSize + rand.Intn(*maxSize-*minSize)
+		startTime := time.Now()
+
+		fmt.Printf("write %s %d bytes: ", f.Name(), fileSize)
+
+		for x := 0; x < fileSize; {
+			rand.Read(block)
+			_, err = f.Write(block)
+			check(err)
+			x += len(block)
+		}
+
+		err = f.Close()
+		check(err)
+
+		fmt.Printf("%.02f MB/sec\n", float64(fileSize)*float64(time.Second)/float64(time.Now().Sub(startTime)*1024*1024))
+	}
+
+}

unmaintained/volume_tailer/volume_tailer.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"log"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	util2 "github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"golang.org/x/tools/godoc/util"
+)
+
+var (
+	master         = flag.String("master", "localhost:9333", "master server host and port")
+	volumeId       = flag.Int("volumeId", -1, "a volume id")
+	rewindDuration = flag.Duration("rewind", -1, "rewind back in time. -1 means from the first entry. 0 means from now.")
+	timeoutSeconds = flag.Int("timeoutSeconds", 0, "disconnect if no activity after these seconds")
+	showTextFile   = flag.Bool("showTextFile", false, "display textual file content")
+)
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	util2.LoadSecurityConfiguration()
+	grpcDialOption := security.LoadClientTLS(util2.GetViper(), "grpc.client")
+
+	vid := needle.VolumeId(*volumeId)
+
+	var sinceTimeNs int64
+	if *rewindDuration == 0 {
+		sinceTimeNs = time.Now().UnixNano()
+	} else if *rewindDuration == -1 {
+		sinceTimeNs = 0
+	} else if *rewindDuration > 0 {
+		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
+	}
+
+	err := operation.TailVolume(func(_ context.Context) pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+		if n.Size == 0 {
+			println("-", n.String())
+			return nil
+		} else {
+			println("+", n.String())
+		}
+
+		if *showTextFile {
+
+			data := n.Data
+			if n.IsCompressed() {
+				if data, err = util2.DecompressData(data); err != nil {
+					return err
+				}
+			}
+			if util.IsText(data) {
+				println(string(data))
+			}
+
+			println("-", n.String(), "compressed", n.IsCompressed(), "original size", len(data))
+		}
+		return nil
+	})
+
+	if err != nil {
+		log.Printf("Error VolumeTailSender volume %d: %v", vid, err)
+	}
+
+}