snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	size        = flag.Int("size", 1024, "file size")
+	concurrency = flag.Int("c", 4, "concurrent number of uploads")
+	times       = flag.Int("n", 1024, "repeated number of times")
+	fileCount   = flag.Int("fileCount", 1, "number of files to write")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	data := make([]byte, *size)
+	println("data len", len(data))
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func(x int) {
+			defer wg.Done()
+
+			r := rand.New(rand.NewSource(time.Now().UnixNano() + int64(x)))
+
+			for t := 0; t < *times; t++ {
+				for f := 0; f < *fileCount; f++ {
+					fn := r.Intn(*fileCount)
+					if size, err := uploadFileToFiler(data, fmt.Sprintf("file%04d", fn), *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					} else {
+						log.Fatalf("client %d upload %d times: %v", x, t, err)
+					}
+				}
+			}
+		}(x)
+	}
+
+	go func() {
+		ticker := time.NewTicker(1000 * time.Millisecond)
+		defer ticker.Stop()
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(data []byte, filename, destination string) (size int64, err error) {
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", filename)
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", filename, err)
+	}
+
+	part.Write(data)
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", filename, err)
+	}
+
+	uri := destination + filename
+
+	request, err := http.NewRequest(http.MethodPost, uri, body)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	}
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+	// request.Close = true  // can not use this, which do not reuse http connection, impacting filer->volume also.
+
+	resp, err := util_http.GetGlobalHttpClient().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return int64(len(data)), nil
+}

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+)
+
+var (
+	dir         = flag.String("dir", ".", "upload files under this directory")
+	concurrency = flag.Int("c", 1, "concurrent number of uploads")
+	times       = flag.Int("n", 1, "repeated number of times")
+	destination = flag.String("to", "http://localhost:8888/", "destination directory on filer")
+
+	statsChan = make(chan stat, 8)
+)
+
+type stat struct {
+	size int64
+}
+
+func main() {
+	flag.Parse()
+	util_http.InitGlobalHttpClient()
+
+	var fileNames []string
+
+	files, err := os.ReadDir(*dir)
+	if err != nil {
+		log.Fatalf("fail to read dir %v: %v", *dir, err)
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			continue
+		}
+		fileNames = append(fileNames, filepath.Join(*dir, file.Name()))
+	}
+
+	var wg sync.WaitGroup
+	for x := 0; x < *concurrency; x++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+			rand.Shuffle(len(fileNames), func(i, j int) {
+				fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
+			})
+			for t := 0; t < *times; t++ {
+				for _, filename := range fileNames {
+					if size, err := uploadFileToFiler(filename, *destination); err == nil {
+						statsChan <- stat{
+							size: size,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	go func() {
+		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
+
+		var lastTime time.Time
+		var counter, size int64
+		for {
+			select {
+			case stat := <-statsChan:
+				size += stat.size
+				counter++
+			case x := <-ticker.C:
+				if !lastTime.IsZero() {
+					elapsed := x.Sub(lastTime).Seconds()
+					fmt.Fprintf(os.Stdout, "%.2f files/s, %.2f MB/s\n",
+						float64(counter)/elapsed,
+						float64(size/1024/1024)/elapsed)
+				}
+				lastTime = x
+				size = 0
+				counter = 0
+			}
+		}
+	}()
+
+	wg.Wait()
+
+}
+
+func uploadFileToFiler(filename, destination string) (size int64, err error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		panic(err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+
+	if !strings.HasSuffix(destination, "/") {
+		destination = destination + "/"
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", file.Name())
+	if err != nil {
+		return 0, fmt.Errorf("fail to create form %v: %v", file.Name(), err)
+	}
+	_, err = io.Copy(part, file)
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		return 0, fmt.Errorf("fail to write part %v: %v", file.Name(), err)
+	}
+
+	uri := destination + file.Name()
+
+	request, err := http.NewRequest(http.MethodPost, uri, body)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	}
+
+	request.Header.Set("Content-Type", writer.FormDataContentType())
+
+	resp, err := util_http.GetGlobalHttpClient().Do(request)
+	if err != nil {
+		return 0, fmt.Errorf("http POST %s: %v", uri, err)
+	} else {
+		body := &bytes.Buffer{}
+		_, err := body.ReadFrom(resp.Body)
+		if err != nil {
+			return 0, fmt.Errorf("read http POST %s response: %v", uri, err)
+		}
+		io.Copy(io.Discard, resp.Body)
+		resp.Body.Close()
+	}
+
+	return fi.Size(), nil
+}

unmaintained/stress_filer_upload/write_files/write_files.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+)
+
+var (
+	minSize   = flag.Int("minSize", 1024, "min file size")
+	maxSize   = flag.Int("maxSize", 3*1024*1024, "max file size")
+	fileCount = flag.Int("n", 1, "number of files to write")
+	blockSize = flag.Int("blockSizeKB", 4, "write block size")
+	toDir     = flag.String("dir", ".", "destination directory")
+)
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+func main() {
+
+	flag.Parse()
+
+	block := make([]byte, *blockSize*1024)
+
+	for i := 0; i < *fileCount; i++ {
+
+		f, err := os.Create(fmt.Sprintf("%s/file%05d", *toDir, i))
+		check(err)
+
+		fileSize := *minSize + rand.Intn(*maxSize-*minSize)
+		startTime := time.Now()
+
+		fmt.Printf("write %s %d bytes: ", f.Name(), fileSize)
+
+		for x := 0; x < fileSize; {
+			rand.Read(block)
+			_, err = f.Write(block)
+			check(err)
+			x += len(block)
+		}
+
+		err = f.Close()
+		check(err)
+
+		fmt.Printf("%.02f MB/sec\n", float64(fileSize)*float64(time.Second)/float64(time.Now().Sub(startTime)*1024*1024))
+	}
+
+}