snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/erasure_coding/Makefile

@@ -0,0 +1,187 @@
+# Makefile for EC integration testing
+# Usage:
+#   make start      - Start the test cluster (master + 6 volume servers + filer)
+#   make stop       - Stop the test cluster
+#   make populate   - Populate test data (~300MB across 7 volumes)
+#   make shell      - Open weed shell connected to the test cluster
+#   make clean      - Stop cluster and remove all test data
+#   make setup      - Start cluster and populate data (one command)
+#
+# Requirements: curl, jq
+
+WEED_BINARY := $(shell pwd)/../../weed/weed
+TEST_DIR := /tmp/ec_manual_test
+# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
+MASTER_PORT := 29333
+FILER_PORT := 28888
+VOLUME_BASE_PORT := 28080
+NUM_VOLUME_SERVERS := 6
+VOLUME_SIZE_LIMIT_MB := 30
+MAX_VOLUMES_PER_SERVER := 10
+
+# Build weed binary if it doesn't exist
+$(WEED_BINARY):
+	cd ../../weed && go build -o weed .
+
+.PHONY: build
+build: $(WEED_BINARY)
+
+.PHONY: start
+start: build
+	@echo "=== Starting SeaweedFS test cluster ==="
+	@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
+	@# Create security.toml with JWT disabled
+	@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
+	@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@echo '' >> $(TEST_DIR)/security.toml
+	@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
+	@echo 'key = ""' >> $(TEST_DIR)/security.toml
+	@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
+	@# Create filer.toml with leveldb2
+	@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
+	@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
+	@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
+	@# Start master
+	@echo "Starting master on port $(MASTER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) master \
+		-port=$(MASTER_PORT) \
+		-mdir=$(TEST_DIR)/master \
+		-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
+	@sleep 3
+	@# Start volume servers (run from TEST_DIR to find security.toml)
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		port=$$(($(VOLUME_BASE_PORT) + $$i)); \
+		echo "Starting volume server $$i on port $$port (rack$$i)..."; \
+		cd $(TEST_DIR) && $(WEED_BINARY) volume \
+			-port=$$port \
+			-dir=$(TEST_DIR)/volume$$i \
+			-max=$(MAX_VOLUMES_PER_SERVER) \
+			-master=127.0.0.1:$(MASTER_PORT) \
+			-ip=127.0.0.1 \
+			-dataCenter=dc1 \
+			-rack=rack$$i \
+			> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
+	done
+	@sleep 3
+	@# Start filer (run from TEST_DIR to find security.toml)
+	@echo "Starting filer on port $(FILER_PORT)..."
+	@cd $(TEST_DIR) && $(WEED_BINARY) filer \
+		-port=$(FILER_PORT) \
+		-master=127.0.0.1:$(MASTER_PORT) \
+		-ip=127.0.0.1 \
+		> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
+	@sleep 3
+	@echo ""
+	@echo "=== Cluster started ==="
+	@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
+	@echo "Filer:  http://127.0.0.1:$(FILER_PORT)"
+	@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
+	@echo ""
+	@echo "Run 'make shell' to open weed shell"
+	@echo "Run 'make populate' to add test data"
+
+.PHONY: stop
+stop:
+	@echo "=== Stopping SeaweedFS test cluster ==="
+	@# Stop filer by PID
+	@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
+	@# Stop volume servers by PID
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
+	done
+	@# Stop master by PID
+	@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
+	@# Fallback: use pkill with specific patterns to ensure cleanup
+	@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
+	@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
+	@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
+	@echo "Cluster stopped."
+
+.PHONY: clean
+clean: stop
+	@echo "Removing test data..."
+	@rm -rf $(TEST_DIR)
+	@echo "Clean complete."
+
+.PHONY: populate
+populate:
+	@echo "=== Populating test data (~300MB) ==="
+	@# Create a 500KB test file template using mktemp for isolation
+	@tmpfile=$$(mktemp) && \
+	dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
+	uploaded=0; \
+	for i in $$(seq 1 600); do \
+		response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
+		fid=$$(echo $$response | jq -r '.fid'); \
+		url=$$(echo $$response | jq -r '.url'); \
+		if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
+			curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
+			uploaded=$$((uploaded + 1)); \
+		fi; \
+		if [ $$((i % 100)) -eq 0 ]; then \
+			echo "Uploaded $$uploaded files..."; \
+		fi; \
+	done; \
+	rm -f $$tmpfile; \
+	echo ""; \
+	echo "=== Data population complete ==="; \
+	echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
+	@echo ""
+	@echo "Volume status:"
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "  Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
+
+.PHONY: shell
+shell: build
+	@echo "Opening weed shell..."
+	@echo "Commands to try:"
+	@echo "  lock"
+	@echo "  volume.list"
+	@echo "  ec.encode -collection=ectest -quietFor=1s -force"
+	@echo "  ec.balance -collection=ectest"
+	@echo "  unlock"
+	@echo ""
+	@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
+
+.PHONY: setup
+setup: clean start
+	@sleep 2
+	@$(MAKE) populate
+
+.PHONY: status
+status:
+	@echo "=== Cluster Status ==="
+	@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
+		'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
+	@echo ""
+	@echo "=== EC Shards ==="
+	@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
+		count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
+		if [ "$$count" != "0" ]; then \
+			echo "  volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
+		fi; \
+	done
+
+.PHONY: help
+help:
+	@echo "EC Integration Test Makefile"
+	@echo ""
+	@echo "Targets:"
+	@echo "  make start     - Start test cluster (master + 6 volume servers + filer)"
+	@echo "  make stop      - Stop test cluster"
+	@echo "  make populate  - Populate ~300MB of test data"
+	@echo "  make shell     - Open weed shell"
+	@echo "  make setup     - Clean, start, and populate (all-in-one)"
+	@echo "  make status    - Show cluster and EC shard status"
+	@echo "  make clean     - Stop cluster and remove all test data"
+	@echo "  make help      - Show this help"
+	@echo ""
+	@echo "Quick start:"
+	@echo "  make setup     # Start cluster and populate data"
+	@echo "  make shell     # Open shell to run EC commands"
+

test/erasure_coding/README.md

@@ -0,0 +1,123 @@
+# Erasure Coding Integration Tests
+
+This directory contains integration tests for the EC (Erasure Coding) encoding volume location timing bug fix.
+
+## The Bug
+
+The bug caused **double storage usage** during EC encoding because:
+
+1. **Silent failure**: Functions returned `nil` instead of proper error messages
+2. **Timing race condition**: Volume locations were collected **AFTER** EC encoding when master metadata was already updated
+3. **Missing cleanup**: Original volumes weren't being deleted after EC encoding
+
+This resulted in both original `.dat` files AND EC `.ec00-.ec13` files coexisting, effectively **doubling storage usage**.
+
+## The Fix
+
+The fix addresses all three issues:
+
+1. **Fixed silent failures**: Updated `doDeleteVolumes()` and `doEcEncode()` to return proper errors
+2. **Fixed timing race condition**: Created `doDeleteVolumesWithLocations()` that uses pre-collected volume locations
+3. **Enhanced cleanup**: Volume locations are now collected **BEFORE** EC encoding, preventing the race condition
+
+## Integration Tests
+
+### TestECEncodingVolumeLocationTimingBug
+The main integration test that:
+- **Simulates master timing race condition**: Tests what happens when volume locations are read from master AFTER EC encoding has updated the metadata
+- **Verifies fix effectiveness**: Checks for the "Collecting volume locations...before EC encoding" message that proves the fix is working
+- **Tests multi-server distribution**: Runs EC encoding with 6 volume servers to test shard distribution
+- **Validates cleanup**: Ensures original volumes are properly cleaned up after EC encoding
+
+### TestECEncodingMasterTimingRaceCondition
+A focused test that specifically targets the **master metadata timing race condition**:
+- **Simulates the exact race condition**: Tests volume location collection timing relative to master metadata updates
+- **Detects timing fix**: Verifies that volume locations are collected BEFORE EC encoding starts
+- **Demonstrates bug impact**: Shows what happens when volume locations are unavailable after master metadata update
+
+### TestECEncodingRegressionPrevention
+Regression tests that ensure:
+- **Function signatures**: Fixed functions still exist and return proper errors
+- **Timing patterns**: Volume location collection happens in the correct order
+
+## Test Architecture
+
+The tests use:
+- **Real SeaweedFS cluster**: 1 master server + 6 volume servers
+- **Multi-server setup**: Tests realistic EC shard distribution across multiple servers
+- **Timing simulation**: Goroutines and delays to simulate race conditions
+- **Output validation**: Checks for specific log messages that prove the fix is working
+
+## Why Integration Tests Were Necessary
+
+Unit tests could not catch this bug because:
+1. **Race condition**: The bug only occurred in real-world timing scenarios
+2. **Master-volume server interaction**: Required actual master metadata updates
+3. **File system operations**: Needed real volume creation and EC shard generation
+4. **Cleanup timing**: Required testing the sequence of operations in correct order
+
+The integration tests successfully catch the timing bug by:
+- **Testing real command execution**: Uses actual `ec.encode` shell command
+- **Simulating race conditions**: Creates timing scenarios that expose the bug
+- **Validating output messages**: Checks for the key "Collecting volume locations...before EC encoding" message
+- **Monitoring cleanup behavior**: Ensures original volumes are properly deleted
+
+## Running the Tests
+
+```bash
+# Run all integration tests
+go test -v
+
+# Run only the main timing test
+go test -v -run TestECEncodingVolumeLocationTimingBug
+
+# Run only the race condition test
+go test -v -run TestECEncodingMasterTimingRaceCondition
+
+# Skip integration tests (short mode)
+go test -v -short
+```
+
+## Manual Testing with Makefile
+
+A Makefile is provided for manual EC testing.
+
+**Requirements:** `curl`, `jq` (command-line JSON processor)
+
+```bash
+# Quick start: start cluster and populate data
+make setup
+
+# Open weed shell to run EC commands
+make shell
+
+# Individual targets
+make start      # Start test cluster (master + 6 volume servers + filer)
+make stop       # Stop test cluster
+make populate   # Populate ~300MB of test data
+make status     # Show cluster and EC shard status
+make clean      # Stop cluster and remove all test data
+make help       # Show all targets
+```
+
+### EC Rebalance Limited Slots (Unit Test)
+
+The "no free ec shard slots" issue is tested with a **unit test** that works directly on
+topology data structures without requiring a running cluster.
+
+**Location**: `weed/shell/ec_rebalance_slots_test.go`
+
+Tests included:
+- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
+- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
+
+**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
+the rebalance step fails with "no free ec shard slots" instead of recognizing that
+moving shards frees slots on source servers.
+
+## Test Results
+
+**With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message
+**Without the fix**: No collection message, potential timing race condition
+
+The tests demonstrate that the fix prevents the volume location timing bug that caused double storage usage in EC encoding operations. 

test/erasure_coding/admin_dockertest/Makefile

@@ -0,0 +1,7 @@
+.PHONY: test clean
+
+test:
+	go test -v .
+
+clean:
+	rm -rf tmp

test/erasure_coding/admin_dockertest/ec_integration_test.go

@@ -0,0 +1,503 @@
+package admin_dockertest
+
+import (
+	"bytes"
+	crand "crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+const (
+	AdminUrl  = "http://localhost:23646"
+	MasterUrl = "http://localhost:9333"
+	FilerUrl  = "http://localhost:8888"
+)
+
+// Helper to run commands in background and track PIDs for cleanup. Guarded
+// by runningCmdsLock so parallel subprocess startup can append safely.
+var (
+	runningCmds     []*exec.Cmd
+	runningCmdsLock sync.Mutex
+)
+
+func cleanup() {
+	runningCmdsLock.Lock()
+	defer runningCmdsLock.Unlock()
+	for _, cmd := range runningCmds {
+		if cmd.Process != nil {
+			cmd.Process.Kill()
+		}
+	}
+}
+
+func startWeed(t *testing.T, name string, args ...string) *exec.Cmd {
+	cmd := exec.Command("./weed_bin", args...)
+
+	// Create logs dir in local ./tmp
+	wd, _ := os.Getwd()
+	logDir := filepath.Join(wd, "tmp", "logs")
+	os.MkdirAll(logDir, 0755)
+
+	logFile, err := os.Create(filepath.Join(logDir, name+".log"))
+	if err != nil {
+		t.Fatalf("Failed to create log file: %v", err)
+	}
+
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	// Set Cwd to test directory so it finds local ./tmp
+	cmd.Dir = wd
+
+	// assume "weed_bin" binary is in project root.
+	rootDir := filepath.Dir(filepath.Dir(filepath.Dir(wd)))
+	cmd.Path = filepath.Join(rootDir, "weed_bin")
+
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("Failed to start weed %v: %v", args, err)
+	}
+	runningCmdsLock.Lock()
+	runningCmds = append(runningCmds, cmd)
+	runningCmdsLock.Unlock()
+	return cmd
+}
+
+func stopWeed(t *testing.T, cmd *exec.Cmd) {
+	if cmd != nil && cmd.Process != nil {
+		t.Logf("Stopping process %d", cmd.Process.Pid)
+		cmd.Process.Kill()
+		cmd.Wait()
+
+		// Remove from runningCmds to avoid double kill in cleanup
+		for i, c := range runningCmds {
+			if c == cmd {
+				runningCmds = append(runningCmds[:i], runningCmds[i+1:]...)
+				break
+			}
+		}
+	}
+}
+
+func ensureEnvironment(t *testing.T) {
+	// 1. Build weed binary
+	wd, _ := os.Getwd()
+	rootDir := filepath.Dir(filepath.Dir(filepath.Dir(wd))) // Up 3 levels
+
+	buildCmd := exec.Command("go", "build", "-o", "weed_bin", "./weed")
+	buildCmd.Dir = rootDir
+	buildCmd.Stdout = os.Stdout
+	buildCmd.Stderr = os.Stderr
+	if err := buildCmd.Run(); err != nil {
+		t.Fatalf("Failed to build weed: %v", err)
+	}
+	t.Log("Successfully built weed binary")
+
+	// 2. Start Master
+	// Use local ./tmp/master
+	os.RemoveAll("tmp")
+	err := os.MkdirAll(filepath.Join("tmp", "master"), 0755)
+	if err != nil {
+		t.Fatalf("Failed to create tmp dir: %v", err)
+	}
+
+	startWeed(t, "master", "master", "-mdir=./tmp/master", "-port=9333", "-ip=localhost", "-peers=none", "-volumeSizeLimitMB=100")
+
+	// Wait for master
+	waitForUrl(t, MasterUrl+"/cluster/status", 10)
+
+	// 3. Start Volume Server (Worker)
+	// Start 14 volume servers to verify RS(10,4) default EC. Fork/exec in
+	// parallel because startWeed is non-blocking and the per-process fork +
+	// mkdir + log-file-open overhead stacks up sequentially on cold CI
+	// disks, eating most of the admin /health wait budget further down.
+	var volWg sync.WaitGroup
+	for i := 1; i <= 14; i++ {
+		volWg.Add(1)
+		go func(i int) {
+			defer volWg.Done()
+			volName := fmt.Sprintf("volume%d", i)
+			port := 8080 + i - 1
+			dir := filepath.Join("tmp", volName)
+			os.MkdirAll(dir, 0755)
+			startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
+		}(i)
+	}
+	volWg.Wait()
+
+	// 4. Start Filer
+	os.MkdirAll(filepath.Join("tmp", "filer"), 0755)
+	startWeed(t, "filer", "filer", "-defaultStoreDir=./tmp/filer", "-master=localhost:9333", "-port=8888", "-ip=localhost")
+	waitForUrl(t, FilerUrl+"/", 60)
+
+	// 5. Start Workers (Maintenance)
+	// We need workers to execute EC tasks
+	for i := 1; i <= 2; i++ {
+		workerName := fmt.Sprintf("worker%d", i)
+		metricsPort := 9327 + i - 1
+		debugPort := 6060 + i
+		dir, _ := filepath.Abs(filepath.Join("tmp", workerName))
+		os.MkdirAll(dir, 0755)
+		startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
+	}
+
+	// 6. Start Admin
+	os.RemoveAll(filepath.Join("tmp", "admin"))
+	os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
+	startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
+	// Admin is started after master, 14 volume servers, filer and 2 workers,
+	// so under cold CI conditions the wait here has to absorb the tail of
+	// every earlier subprocess coming up. 60s is too tight and has flaked;
+	// 180s gives comfortable headroom without meaningfully extending the
+	// fast path (the first successful /health usually hits well under 30s).
+	waitForUrl(t, AdminUrl+"/health", 180)
+
+	t.Log("Environment started successfully")
+}
+
+func waitForUrl(t *testing.T, url string, retries int) {
+	for i := 0; i < retries; i++ {
+		resp, err := http.Get(url)
+		if err == nil && resp.StatusCode == 200 {
+			resp.Body.Close()
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+	t.Fatalf("Timeout waiting for %s", url)
+}
+
+func fetchJSON(url string, out interface{}) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
+	}
+	return json.NewDecoder(resp.Body).Decode(out)
+}
+
+func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
+	if obj == nil {
+		return nil, false
+	}
+	if value, ok := obj[key]; ok {
+		return value, true
+	}
+	return nil, false
+}
+
+func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
+	for _, key := range keys {
+		if value, ok := mapField(obj, key); ok {
+			return value, true
+		}
+	}
+	return nil, false
+}
+
+func TestEcEndToEnd(t *testing.T) {
+	defer cleanup()
+	ensureEnvironment(t)
+
+	client := &http.Client{}
+
+	// 1. Configure plugin job types for fast EC detection/execution.
+	t.Log("Configuring plugin job types via API...")
+
+	// Disable volume balance to reduce interference for this EC-focused test.
+	balanceConfig := map[string]interface{}{
+		"job_type": "volume_balance",
+		"admin_runtime": map[string]interface{}{
+			"enabled": false,
+		},
+	}
+	jsonBody, err := json.Marshal(balanceConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal volume_balance config: %v", err)
+	}
+	req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create volume_balance config request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to update volume_balance config: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to update volume_balance config (status %d): %s", resp.StatusCode, string(body))
+	}
+	resp.Body.Close()
+
+	ecConfig := map[string]interface{}{
+		"job_type": "erasure_coding",
+		"admin_runtime": map[string]interface{}{
+			"enabled":                          true,
+			"detection_interval_minutes":       1,
+			"global_execution_concurrency":     4,
+			"per_worker_execution_concurrency": 4,
+			"max_jobs_per_detection":           100,
+		},
+		"worker_config_values": map[string]interface{}{
+			"quiet_for_seconds": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"min_interval_seconds": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"min_size_mb": map[string]interface{}{
+				"int64_value": "1",
+			},
+			"fullness_ratio": map[string]interface{}{
+				"double_value": 0.0001,
+			},
+		},
+	}
+	jsonBody, err = json.Marshal(ecConfig)
+	if err != nil {
+		t.Fatalf("Failed to marshal erasure_coding config: %v", err)
+	}
+	req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/erasure_coding/config", bytes.NewBuffer(jsonBody))
+	if err != nil {
+		t.Fatalf("Failed to create erasure_coding config request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err = client.Do(req)
+	if err != nil {
+		t.Fatalf("Failed to update erasure_coding config: %v", err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		t.Fatalf("Failed to update erasure_coding config (status %d): %s", resp.StatusCode, string(body))
+	}
+	resp.Body.Close()
+
+	// 2. Upload a file
+	fileSize := 5 * 1024 * 1024
+	data := make([]byte, fileSize)
+	crand.Read(data)
+	fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix())
+	t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName)
+	uploadUrl := FilerUrl + "/" + fileName
+
+	var uploadErr error
+	for i := 0; i < 10; i++ {
+		req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
+		if err != nil {
+			uploadErr = err
+			t.Logf("Upload attempt %d failed to create request: %v", i+1, err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		resp, err := client.Do(req)
+		if err == nil {
+			if resp.StatusCode == 201 {
+				resp.Body.Close()
+				uploadErr = nil
+				break
+			}
+			body, _ := io.ReadAll(resp.Body)
+			resp.Body.Close()
+			uploadErr = fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
+		} else {
+			uploadErr = err
+		}
+		t.Logf("Upload attempt %d failed: %v", i+1, uploadErr)
+		time.Sleep(2 * time.Second)
+	}
+
+	if uploadErr != nil {
+		t.Fatalf("Failed to upload file after retries: %v", uploadErr)
+	}
+	t.Log("Upload successful")
+
+	// 3. Verify EC Encoding
+	t.Log("Waiting for EC encoding (checking Master topology)...")
+	startTime := time.Now()
+	ecVerified := false
+	var lastBody []byte
+	debugTick := 0
+
+	for time.Since(startTime) < 300*time.Second {
+		// 3.1 Check Master Topology
+		resp, err := http.Get(MasterUrl + "/dir/status")
+		if err == nil {
+			lastBody, _ = io.ReadAll(resp.Body)
+			resp.Body.Close()
+
+			// Check total EC shards
+			reShards := regexp.MustCompile(`"EcShards":\s*(\d+)`)
+			matches := reShards.FindAllSubmatch(lastBody, -1)
+			totalShards := 0
+			for _, m := range matches {
+				var count int
+				fmt.Sscanf(string(m[1]), "%d", &count)
+				totalShards += count
+			}
+
+			if totalShards > 0 {
+				t.Logf("EC encoding verified (found %d total EcShards in topology) after %d seconds", totalShards, int(time.Since(startTime).Seconds()))
+				ecVerified = true
+				break
+			}
+		}
+
+		// 3.2 Debug: Check workers, jobs, and scheduler status
+		debugTick++
+
+		var workers []map[string]interface{}
+		workerCount := 0
+		ecDetectorCount := 0
+		ecExecutorCount := 0
+		if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
+			workerCount = len(workers)
+			for _, worker := range workers {
+				capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
+				if !ok {
+					continue
+				}
+				caps, ok := capsValue.(map[string]interface{})
+				if !ok {
+					continue
+				}
+				if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
+					if capValue["can_detect"] == true {
+						ecDetectorCount++
+					}
+					if capValue["can_execute"] == true {
+						ecExecutorCount++
+					}
+				}
+			}
+		}
+
+		var tasks []map[string]interface{}
+		taskCount := 0
+		ecTaskCount := 0
+		ecTaskStates := map[string]int{}
+		if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
+			taskCount = len(tasks)
+			for _, task := range tasks {
+				jobType, _ := task["job_type"].(string)
+				state, _ := task["state"].(string)
+				if jobType == "erasure_coding" {
+					ecTaskCount++
+					ecTaskStates[state]++
+				}
+			}
+		}
+
+		t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
+			workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
+
+		if debugTick%3 == 0 {
+			var pluginStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
+				t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
+					pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
+			}
+
+			var schedulerStatus map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
+				if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
+					t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
+						schedValue["current_job_type"], schedValue["current_phase"],
+						schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
+				} else {
+					t.Logf("Scheduler status: %v", schedulerStatus)
+				}
+			}
+
+			var schedulerStates []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
+				for _, state := range schedulerStates {
+					if state["job_type"] == "erasure_coding" {
+						t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
+							state["enabled"], state["detection_in_flight"], state["detector_available"],
+							state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
+							state["last_run_started_at"], state["last_run_completed_at"])
+						break
+					}
+				}
+			}
+
+			var jobTypes []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
+				var names []string
+				for _, jobType := range jobTypes {
+					if name, ok := jobType["job_type"].(string); ok && name != "" {
+						names = append(names, name)
+					}
+				}
+				t.Logf("Plugin job types: %v", names)
+			}
+
+			var activities []map[string]interface{}
+			if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
+				for i := len(activities) - 1; i >= 0; i-- {
+					act := activities[i]
+					t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
+				}
+			}
+		}
+
+		time.Sleep(10 * time.Second)
+	}
+
+	if !ecVerified {
+		dumpLogs(t)
+		t.Fatalf("Timed out waiting for EC encoding verified in Topology. Last body: %s", string(lastBody))
+	}
+
+	// 6. Verification: Read back the file
+	t.Log("Reading back file...")
+	resp, err = http.Get(uploadUrl)
+	if err != nil {
+		dumpLogs(t)
+		t.Fatalf("Failed to read back file: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		dumpLogs(t)
+		t.Fatalf("Read back failed status: %d", resp.StatusCode)
+	}
+	content, _ := io.ReadAll(resp.Body)
+	if len(content) != fileSize {
+		dumpLogs(t)
+		t.Fatalf("Read back size mismatch: got %d, want %d", len(content), fileSize)
+	}
+
+	// Verify byte-wise content equality
+	if !bytes.Equal(content, data) {
+		dumpLogs(t)
+		t.Fatalf("Read back content mismatch: uploaded and downloaded data differ")
+	}
+
+	t.Log("Test PASS: EC encoding and read back successful!")
+}
+
+func dumpLogs(t *testing.T) {
+	wd, _ := os.Getwd()
+	logDir := filepath.Join(wd, "tmp", "logs")
+	files, _ := os.ReadDir(logDir)
+	for _, f := range files {
+		if strings.HasSuffix(f.Name(), ".log") {
+			content, _ := os.ReadFile(filepath.Join(logDir, f.Name()))
+			t.Logf("--- LOG DUMP: %s ---\n%s\n--- END LOG ---", f.Name(), string(content))
+		}
+	}
+}

test/erasure_coding/multidisk_shardloss_test.go

@@ -0,0 +1,265 @@
+package erasure_coding
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+)
+
+// TestMultiDiskECBalanceNoShardLoss is the end-to-end regression for issue 9593.
+// It runs a real cluster of multi-disk volume servers (3 servers x 4 disks),
+// EC-encodes a volume, then runs ec.balance, asserting hard invariants the older
+// integration tests only logged:
+//
+//   - after encode the full set of 14 EC shards exists,
+//   - ec.balance never loses a shard (still 14 distinct shards afterwards),
+//   - shards end up spread across more than one disk per node, and
+//   - cluster.status counts physical disks (not one per node) and matches the
+//     real on-disk distribution.
+func TestMultiDiskECBalanceNoShardLoss(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping multi-disk EC integration test in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 240*time.Second)
+	defer cancel()
+
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	t.Log("waiting for multi-disk volume servers to register...")
+	time.Sleep(10 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	// Upload enough small files that the volume holds real data to encode.
+	var volumeId needle.VolumeId
+	for retry := 0; retry < 5; retry++ {
+		volumeId, err = uploadTestDataToMaster([]byte(strings.Repeat("multidisk-ec-9593 ", 64)), "127.0.0.1:9334")
+		if err == nil {
+			break
+		}
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "failed to upload test data")
+	for i := 0; i < 40; i++ {
+		if _, e := uploadTestDataToMaster([]byte(strings.Repeat("filler ", 128)), "127.0.0.1:9334"); e != nil {
+			break
+		}
+	}
+	t.Logf("using volume %d", volumeId)
+	time.Sleep(3 * time.Second)
+
+	// Populate every server's disks with volumes so the encode can see and target
+	// each physical disk. The master only enumerates disks that already hold a
+	// volume or EC shard — an empty disk leaves no trace in the topology (heartbeats
+	// aggregate capacity per disk type, not per physical disk). ec.encode therefore
+	// spreads a volume's shards only across the disks the master already knows hold
+	// data on each node; if a node's data sits on a single disk, all its shards land
+	// there and ec.balance cannot redistribute them (it has no within-node
+	// cross-disk move). So spreading must be set up before encoding.
+	//
+	// volume.grow only tops up toward a writable target and stops on the first
+	// allocation error, so a single -count grow can create far fewer volumes than
+	// asked and leave a node on one disk. Grow repeatedly on the nodes that have not
+	// spread yet (the volume server places each new volume on its least-loaded disk)
+	// until the master's topology shows every node holding volumes on at least two
+	// physical disks. This makes the multi-disk layout — and thus the post-encode
+	// disk spread — deterministic instead of racing volume-growth and heartbeat.
+	require.Eventually(t, func() bool {
+		spread := nodeVolumeDiskCounts(t, commandEnv)
+		if len(spread) == 3 && allAtLeast(spread, 2) {
+			return true
+		}
+		for i := 0; i < 3; i++ {
+			server := fmt.Sprintf("127.0.0.1:809%d", i)
+			if spread[server] < 2 {
+				captureCommandOutput(t, shell.Commands[findCommandIndex("volume.grow")],
+					[]string{"-collection", "test", "-dataNode", server, "-count", "4"}, commandEnv)
+			}
+		}
+		return false
+	}, 60*time.Second, 2*time.Second,
+		"volumes never spread across >=2 disks on all 3 nodes")
+
+	locked, unlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, locked, "could not acquire shell lock")
+	defer unlock()
+
+	// EC-encode the volume.
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("ec.encode")],
+		[]string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.encode output:\n%s", out)
+	require.NoError(t, err, "ec.encode failed")
+
+	// All 14 shards must exist after encoding.
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 30*time.Second, time.Second, "expected all %d EC shards after encode, got %v",
+		erasureShardCount, collectDistinctShardIDs(testDir, uint32(volumeId)))
+
+	beforeBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	t.Logf("after encode: %d distinct shards on %d disks", len(beforeBalance), disksWithShards(testDir, uint32(volumeId)))
+
+	// Run ec.balance.
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.balance")],
+		[]string{"-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.balance output:\n%s", out)
+	require.NoError(t, err, "ec.balance failed")
+	time.Sleep(3 * time.Second)
+
+	// The core regression: ec.balance must not lose any shard.
+	afterBalance := collectDistinctShardIDs(testDir, uint32(volumeId))
+	require.Equal(t, erasureShardCount, len(afterBalance),
+		"ec.balance lost shards on multi-disk nodes: had %v, now %v", sortedKeysOf(beforeBalance), sortedKeysOf(afterBalance))
+
+	// Shards must be spread across more than one physical disk per node overall.
+	usedDisks := disksWithShards(testDir, uint32(volumeId))
+	assert.Greater(t, usedDisks, 3, "EC shards should span more than one disk per node (got %d disks across 3 nodes)", usedDisks)
+
+	// cluster.status must count physical disks, not collapse to one per node: it
+	// must report at least the disks actually holding this volume's shards (which
+	// is already >3 across the 3 nodes). Before the fix it reported 3 (node count).
+	require.Eventually(t, func() bool {
+		n, ok := clusterStatusDiskCount(t, commandEnv)
+		return ok && n >= usedDisks
+	}, 30*time.Second, 2*time.Second, "cluster.status never reported the >=%d physical disks holding shards (multi-disk count)", usedDisks)
+
+	n, _ := clusterStatusDiskCount(t, commandEnv)
+	t.Logf("cluster.status reports %d physical disks (>= %d holding this volume's shards)", n, usedDisks)
+}
+
+const erasureShardCount = 14 // 10 data + 4 parity
+
+// collectDistinctShardIDs returns the set of EC shard ids present for a volume
+// across every disk of every server in the multi-disk test layout.
+func collectDistinctShardIDs(testDir string, volumeId uint32) map[int]bool {
+	ids := map[int]bool{}
+	for server := 0; server < 3; server++ {
+		for disk := 0; disk < 4; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			files, err := listECShardFiles(diskDir, volumeId)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				i := strings.LastIndex(f, ".ec")
+				if i < 0 {
+					continue
+				}
+				if n, err := strconv.Atoi(f[i+3:]); err == nil && n >= 0 && n < erasureShardCount {
+					ids[n] = true
+				}
+			}
+		}
+	}
+	return ids
+}
+
+// disksWithShards counts how many physical disks hold at least one shard.
+func disksWithShards(testDir string, volumeId uint32) int {
+	n := 0
+	for _, disks := range countShardsPerDisk(testDir, volumeId) {
+		for _, c := range disks {
+			if c > 0 {
+				n++
+			}
+		}
+	}
+	return n
+}
+
+// nodeVolumeDiskCounts returns, per volume server id, how many distinct physical
+// disks hold at least one volume according to the master's topology. The master
+// only enumerates disks that already hold a volume or EC shard (heartbeats
+// aggregate capacity per disk type, not per physical disk), so this reports the
+// disks ec.encode can actually spread a volume's shards across on each node.
+func nodeVolumeDiskCounts(t *testing.T, commandEnv *shell.CommandEnv) map[string]int {
+	t.Helper()
+	var resp *master_pb.VolumeListResponse
+	err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		var e error
+		resp, e = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
+		return e
+	})
+	counts := map[string]int{}
+	if err != nil || resp.GetTopologyInfo() == nil {
+		return counts
+	}
+	for _, dc := range resp.GetTopologyInfo().GetDataCenterInfos() {
+		for _, r := range dc.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				disks := map[uint32]bool{}
+				for _, di := range dn.GetDiskInfos() {
+					for _, vi := range di.GetVolumeInfos() {
+						disks[vi.GetDiskId()] = true
+					}
+				}
+				counts[dn.Id] = len(disks)
+			}
+		}
+	}
+	return counts
+}
+
+func allAtLeast(counts map[string]int, min int) bool {
+	for _, c := range counts {
+		if c < min {
+			return false
+		}
+	}
+	return true
+}
+
+var diskCountRe = regexp.MustCompile(`(\d+)\s+disks?`)
+
+// clusterStatusDiskCount runs cluster.status and parses the reported disk count.
+func clusterStatusDiskCount(t *testing.T, commandEnv *shell.CommandEnv) (int, bool) {
+	t.Helper()
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("cluster.status")], []string{}, commandEnv)
+	if err != nil {
+		return 0, false
+	}
+	m := diskCountRe.FindStringSubmatch(out)
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.Atoi(m[1])
+	return n, err == nil
+}
+
+func sortedKeysOf(m map[int]bool) []int {
+	out := make([]int, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	for i := 1; i < len(out); i++ {
+		for j := i; j > 0 && out[j-1] > out[j]; j-- {
+			out[j-1], out[j] = out[j], out[j-1]
+		}
+	}
+	return out
+}

test/erasure_coding/multidisk_shell_lifecycle_test.go

@@ -0,0 +1,410 @@
+package erasure_coding
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// TestMultiDiskECShellLifecycle drives the real shell commands — ec.encode,
+// ec.balance, ec.rebuild — against a live multi-disk cluster, through the
+// sequence a production repository goes through, with one invariant checked
+// after every step: the bytes a client stored come back identical. Shard
+// counting alone cannot tell a healthy volume from one that was rebuilt out
+// of the wrong inputs; reading the payload back can.
+//
+// The damage step reproduces a support case: shard files removed from disk
+// while the cluster still lists them, so the subsequent ec.rebuild sees copy
+// failures ("CopyFile not found ec volume id ...") for locations the master
+// believes exist, and must recover from the shards that are really there.
+func TestMultiDiskECShellLifecycle(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping multi-disk EC shell lifecycle test in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 360*time.Second)
+	defer cancel()
+
+	cluster, err := startMultiDiskCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer("127.0.0.1:9334", 30*time.Second))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	t.Log("waiting for multi-disk volume servers to register...")
+	time.Sleep(10 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr("127.0.0.1:9334"),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	// A payload with no structure, so a rebuild that assembled the wrong bytes
+	// cannot accidentally reproduce it.
+	payload := make([]byte, 8192)
+	_, err = rand.Read(payload)
+	require.NoError(t, err)
+
+	var volumeId needle.VolumeId
+	var fid string
+	for retry := 0; retry < 5; retry++ {
+		volumeId, fid, err = uploadPayload(payload)
+		if err == nil {
+			break
+		}
+		time.Sleep(3 * time.Second)
+	}
+	require.NoError(t, err, "failed to upload payload")
+	for i := 0; i < 40; i++ {
+		if _, _, e := uploadPayload([]byte(strings.Repeat("filler ", 128))); e != nil {
+			break
+		}
+	}
+	t.Logf("payload of %d bytes in volume %d as %s", len(payload), volumeId, fid)
+	time.Sleep(3 * time.Second)
+
+	requirePayload := func(step string) {
+		t.Helper()
+		require.Eventually(t, func() bool {
+			got, err := readFid(fid, uint32(volumeId))
+			return err == nil && bytes.Equal(got, payload)
+		}, 30*time.Second, time.Second, "payload not readable byte-identical after %s", step)
+		t.Logf("payload verified after %s", step)
+	}
+	requirePayload("upload")
+
+	// Spread volumes across each node's disks before encoding; the master only
+	// enumerates disks that already hold data, and shards only spread across
+	// enumerated disks (see TestMultiDiskECBalanceNoShardLoss for the details).
+	require.Eventually(t, func() bool {
+		spread := nodeVolumeDiskCounts(t, commandEnv)
+		if len(spread) == 3 && allAtLeast(spread, 2) {
+			return true
+		}
+		for i := 0; i < 3; i++ {
+			server := fmt.Sprintf("127.0.0.1:809%d", i)
+			if spread[server] < 2 {
+				captureCommandOutput(t, shell.Commands[findCommandIndex("volume.grow")],
+					[]string{"-collection", "test", "-dataNode", server, "-count", "4"}, commandEnv)
+			}
+		}
+		return false
+	}, 60*time.Second, 2*time.Second, "volumes never spread across >=2 disks on all 3 nodes")
+
+	locked, unlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, locked, "could not acquire shell lock")
+	defer unlock()
+
+	// ── ec.encode ──
+	out, err := captureCommandOutput(t, shell.Commands[findCommandIndex("ec.encode")],
+		[]string{"-volumeId", fmt.Sprintf("%d", volumeId), "-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.encode output:\n%s", out)
+	require.NoError(t, err, "ec.encode failed")
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 30*time.Second, time.Second, "expected all %d shards after encode", erasureShardCount)
+	requirePayload("ec.encode")
+
+	// ── ec.balance ──
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.balance")],
+		[]string{"-collection", "test", "-force"}, commandEnv)
+	t.Logf("ec.balance output:\n%s", out)
+	require.NoError(t, err, "ec.balance failed")
+	time.Sleep(3 * time.Second)
+	require.Len(t, collectDistinctShardIDs(testDir, uint32(volumeId)), erasureShardCount,
+		"ec.balance lost shards")
+	requirePayload("ec.balance")
+
+	// ── damage: two shard files vanish from disk ──
+	// Removed straight off disk, then the servers are restarted so the master
+	// relearns the shard set from disk rather than from stale heartbeat state.
+	// This is the recoverable form of the support case: the volume drops to 12
+	// of 14 shards, which is still above the 10 needed to read and rebuild.
+	removed := removeTwoShardFiles(t, testDir, uint32(volumeId))
+	t.Logf("removed shard files for shards %v", removed)
+	require.Len(t, collectDistinctShardIDs(testDir, uint32(volumeId)), erasureShardCount-2)
+
+	require.NoError(t, cluster.RestartVolumeServers(ctx))
+	for i := 0; i < 3; i++ {
+		require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:809%d", i), 30*time.Second))
+	}
+	requirePayload("shard loss + restart") // 12 of 14 remain; reads must survive
+
+	// The master must now see exactly the 12 shards on disk before any repair
+	// runs. ec.rebuild is driven by the master's view, so a repair planned
+	// while that view still lists the two dead shards would target the wrong
+	// set. Restart-then-relearn is what makes disk truth and the master agree.
+	require.Eventually(t, func() bool {
+		return len(masterEcShardIds(commandEnv, uint32(volumeId))) == erasureShardCount-2
+	}, 60*time.Second, 2*time.Second,
+		"master never relearned the reduced shard set (got %v)",
+		sortedKeysOf(masterEcShardIds(commandEnv, uint32(volumeId))))
+
+	// The shell lock is dropped by the restart's master disconnect; retake it.
+	relocked, reunlock := tryLockWithTimeout(t, commandEnv, 15*time.Second)
+	require.True(t, relocked, "could not reacquire shell lock after restart")
+	defer reunlock()
+
+	// ── ec.rebuild, the command from the support case ──
+	out, err = captureCommandOutput(t, shell.Commands[findCommandIndex("ec.rebuild")],
+		[]string{"-collection", "test", "-apply"}, commandEnv)
+	t.Logf("ec.rebuild output:\n%s", out)
+	require.NoError(t, err, "ec.rebuild failed")
+	require.Eventually(t, func() bool {
+		return len(collectDistinctShardIDs(testDir, uint32(volumeId))) == erasureShardCount
+	}, 60*time.Second, time.Second, "ec.rebuild did not restore all %d shards on disk", erasureShardCount)
+	requirePayload("ec.rebuild")
+
+	// The rebuilt shards are real: master's view and disk truth agree on all 14.
+	require.Eventually(t, func() bool {
+		registered := masterEcShardIds(commandEnv, uint32(volumeId))
+		onDisk := collectDistinctShardIDs(testDir, uint32(volumeId))
+		if len(registered) != len(onDisk) || len(onDisk) != erasureShardCount {
+			return false
+		}
+		for id := range onDisk {
+			if !registered[id] {
+				return false
+			}
+		}
+		return true
+	}, 60*time.Second, 2*time.Second,
+		"master's EC shard view diverged from disk truth after rebuild: master=%v disk=%v",
+		sortedKeysOf(masterEcShardIds(commandEnv, uint32(volumeId))), sortedKeysOf(collectDistinctShardIDs(testDir, uint32(volumeId))))
+}
+
+// RestartVolumeServers kills every volume server process and starts fresh ones
+// over the same directories, so registration is rebuilt purely from disk.
+func (c *MultiDiskCluster) RestartVolumeServers(ctx context.Context) error {
+	for _, cmd := range c.volumeServers {
+		if cmd != nil && cmd.Process != nil {
+			cmd.Process.Kill()
+			cmd.Wait()
+		}
+	}
+	c.volumeServers = nil
+	time.Sleep(2 * time.Second)
+	return c.startVolumeServers(ctx)
+}
+
+// startVolumeServers launches the standard 3-server x 4-disk layout over the
+// cluster's existing directories, mirroring startMultiDiskCluster's loop so a
+// restart brings servers back exactly as they first started.
+func (c *MultiDiskCluster) startVolumeServers(ctx context.Context) error {
+	weedBinary := findWeedBinary()
+	if weedBinary == "" {
+		return fmt.Errorf("weed binary not found")
+	}
+	const numServers = 3
+	const disksPerServer = 4
+	for i := 0; i < numServers; i++ {
+		var diskDirs []string
+		var maxVolumes []string
+		for d := 0; d < disksPerServer; d++ {
+			diskDirs = append(diskDirs, filepath.Join(c.testDir, fmt.Sprintf("server%d_disk%d", i, d)))
+			maxVolumes = append(maxVolumes, "5")
+		}
+		volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
+			"-port", fmt.Sprintf("809%d", i),
+			"-dir", strings.Join(diskDirs, ","),
+			"-max", strings.Join(maxVolumes, ","),
+			"-master", "127.0.0.1:9334",
+			"-ip", "127.0.0.1",
+			"-dataCenter", "dc1",
+			"-rack", fmt.Sprintf("rack%d", i),
+		)
+		logFile, err := os.OpenFile(filepath.Join(c.testDir, fmt.Sprintf("server%d_logs", i), "volume-restart.log"),
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+		if err != nil {
+			return fmt.Errorf("open restart log for server %d: %w", i, err)
+		}
+		c.logFiles = append(c.logFiles, logFile)
+		volumeCmd.Stdout = logFile
+		volumeCmd.Stderr = logFile
+		if err := volumeCmd.Start(); err != nil {
+			return fmt.Errorf("restart volume server %d: %w", i, err)
+		}
+		c.volumeServers = append(c.volumeServers, volumeCmd)
+	}
+	time.Sleep(8 * time.Second)
+	return nil
+}
+
+// uploadPayload stores data in collection "test" and returns the volume id and
+// fid so the same needle can be read back after every subsequent operation.
+func uploadPayload(data []byte) (needle.VolumeId, string, error) {
+	assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
+		return pb.ServerAddress("127.0.0.1:9334")
+	}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
+		Count:       1,
+		Collection:  "test",
+		Replication: "000",
+	})
+	if err != nil {
+		return 0, "", err
+	}
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return 0, "", err
+	}
+	uploadResult, err, _ := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
+		UploadUrl: "http://" + assignResult.Url + "/" + assignResult.Fid,
+		Filename:  "payload.bin",
+		MimeType:  "application/octet-stream",
+	})
+	if err != nil {
+		return 0, "", err
+	}
+	if uploadResult.Error != "" {
+		return 0, "", fmt.Errorf("upload error: %s", uploadResult.Error)
+	}
+	fidObj, err := needle.ParseFileIdFromString(assignResult.Fid)
+	if err != nil {
+		return 0, "", err
+	}
+	return fidObj.VolumeId, assignResult.Fid, nil
+}
+
+// readFid locates the volume via the master and fetches the needle bytes from
+// a currently-registered location — the same route a client read takes, for a
+// plain volume before encoding and for the EC read path after.
+func readFid(fid string, volumeId uint32) ([]byte, error) {
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:9334/dir/lookup?volumeId=%d", volumeId))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	var lookup struct {
+		Locations []struct {
+			Url string `json:"url"`
+		} `json:"locations"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&lookup); err != nil {
+		return nil, err
+	}
+	if len(lookup.Locations) == 0 {
+		return nil, fmt.Errorf("no locations for volume %d", volumeId)
+	}
+	var lastErr error
+	for _, loc := range lookup.Locations {
+		get, err := http.Get(fmt.Sprintf("http://%s/%s", loc.Url, fid))
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		body, err := io.ReadAll(get.Body)
+		get.Body.Close()
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if get.StatusCode != http.StatusOK {
+			lastErr = fmt.Errorf("GET %s from %s: %d", fid, loc.Url, get.StatusCode)
+			continue
+		}
+		return body, nil
+	}
+	return nil, lastErr
+}
+
+// removeTwoShardFiles deletes the files of two distinct shards straight off
+// the disks, without telling any server — the divergence between disk truth
+// and the master's view at the heart of the support case.
+func removeTwoShardFiles(t *testing.T, testDir string, volumeId uint32) []int {
+	t.Helper()
+	var removed []int
+	for server := 0; server < 3 && len(removed) < 2; server++ {
+		for disk := 0; disk < 4 && len(removed) < 2; disk++ {
+			diskDir := filepath.Join(testDir, fmt.Sprintf("server%d_disk%d", server, disk))
+			files, err := listECShardFiles(diskDir, volumeId)
+			if err != nil {
+				continue
+			}
+			for _, f := range files {
+				i := strings.LastIndex(f, ".ec")
+				if i < 0 {
+					continue
+				}
+				var id int
+				if _, err := fmt.Sscanf(f[i+3:], "%d", &id); err != nil {
+					continue
+				}
+				if err := os.Remove(filepath.Join(diskDir, f)); err != nil {
+					t.Fatalf("remove shard file %s: %v", f, err)
+				}
+				removed = append(removed, id)
+				break // at most one shard per disk, keep the loss spread out
+			}
+		}
+	}
+	require.Len(t, removed, 2, "could not find two shard files to remove")
+	return removed
+}
+
+// masterEcShardIds reports which shard ids the master currently believes exist
+// for the volume, across all nodes — the view every repair decision runs on.
+func masterEcShardIds(commandEnv *shell.CommandEnv, volumeId uint32) map[int]bool {
+	ids := map[int]bool{}
+	var resp *master_pb.VolumeListResponse
+	err := commandEnv.MasterClient.WithClient(context.Background(), false, func(client master_pb.SeaweedClient) error {
+		var e error
+		resp, e = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
+		return e
+	})
+	if err != nil || resp.GetTopologyInfo() == nil {
+		return ids
+	}
+	for _, dc := range resp.GetTopologyInfo().GetDataCenterInfos() {
+		for _, r := range dc.GetRackInfos() {
+			for _, dn := range r.GetDataNodeInfos() {
+				for _, di := range dn.GetDiskInfos() {
+					for _, eci := range di.GetEcShardInfos() {
+						if eci.GetId() != volumeId {
+							continue
+						}
+						for _, sid := range erasure_coding_ShardIds(eci.GetEcIndexBits()) {
+							ids[sid] = true
+						}
+					}
+				}
+			}
+		}
+	}
+	return ids
+}
+
+// erasure_coding_ShardIds expands an EC index bitmap into shard ids without
+// importing the storage package into this test's namespace twice.
+func erasure_coding_ShardIds(bits uint32) []int {
+	var out []int
+	for i := 0; i < 32; i++ {
+		if bits&(1<<uint(i)) != 0 {
+			out = append(out, i)
+		}
+	}
+	return out
+}

test/erasure_coding/systematic_interruption_test.go

@@ -0,0 +1,232 @@
+package erasure_coding
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+
+	"github.com/seaweedfs/seaweedfs/weed/shell"
+)
+
+// TestECInterruptionMatrix is the systematic counterpart of the randomized
+// chaos harness: instead of killing operations at random times, it enumerates
+// every phase of every interruptible EC operation and kills a real weed shell
+// exactly when that phase announces itself on the command's output. Each
+// (operation, phase) pair is one deterministic scenario: prepare the
+// precondition, kill at the phase marker, run the prescribed recovery, and
+// verify that every stored byte still reads back identical.
+//
+// The phase markers are the progress lines the pipelines print; each marker
+// below names the code that prints it. A marker is the START of its phase, so
+// killing on it lands the interruption inside that phase — deterministically
+// per phase, byte-exact timing within the phase left to the scheduler. Every
+// scenario REQUIRES its marker to appear: a marker that never prints means a
+// pipeline refactor renamed or dropped the progress line, and silently
+// degenerating into a no-interruption run would let CI pass without
+// exercising the boundary the scenario names. Update the marker with the
+// pipeline.
+//
+// Complementing this, weed/ec's TestECLifecycleModelExhaustive explores every
+// schedule of the same pipelines — crashes between every step, rollback
+// paths, restarts — exhaustively on a model whose steps mirror the code.
+func TestECInterruptionMatrix(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping EC interruption matrix in short mode")
+	}
+
+	testDir := t.TempDir()
+	ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute)
+	defer cancel()
+
+	cluster, err := startChaosCluster(ctx, testDir)
+	require.NoError(t, err)
+	defer cluster.Stop()
+
+	require.NoError(t, waitForServer(chaosMasterAddr, 30*time.Second))
+	for i := 0; i < chaosServerCount; i++ {
+		require.NoError(t, waitForServer("127.0.0.1:"+chaosVolumePort(i), 30*time.Second))
+	}
+	time.Sleep(8 * time.Second)
+
+	commandEnv := shell.NewCommandEnv(&shell.ShellOptions{
+		Masters:        stringPtr(chaosMasterAddr),
+		GrpcDialOption: grpc.WithInsecure(),
+		FilerGroup:     stringPtr("default"),
+	})
+	connectToMasterAndSync(ctx, t, commandEnv)
+
+	r := newChaosRun(t, ctx, cluster, commandEnv, testDir, 1)
+	r.relock()
+	defer r.unlockIfHeld()
+	r.seedAndSpread()
+	r.verify("seeding")
+
+	// One scenario per (operation, phase marker). The preparation puts a
+	// dedicated volume into the operation's precondition state, so scenarios
+	// stay independent even though they share the cluster.
+	type scenario struct {
+		op      string
+		marker  string // printed by ↓
+		printed string // the code that prints it, for the reader
+	}
+	encodeScenarios := []scenario{
+		{"ec.encode", "markVolumeReadonly ", "weed/ec markVolumeReplicaWritable"},
+		{"ec.encode", "generateEcShards ", "weed/ec generateEcShards"},
+		{"ec.encode", "mount ", "weed/ec MountEcShards"},
+		{"ec.encode", "Deleting original volumes", "weed/ec ProcessEcEncodeBatch, the commit boundary"},
+	}
+	decodeScenarios := []scenario{
+		{"ec.decode", " shard locations:", "weed/ec DoEcDecode, before collect"},
+		{"ec.decode", "generateNormalVolume", "weed/ec generateNormalVolume"},
+		{"ec.decode", "unmount ec volume", "weed/ec unmountAndDeleteEcShards, teardown begin"},
+		{"ec.decode", "delete ec volume", "weed/ec unmountAndDeleteEcShards, shard deletion"},
+	}
+
+	for _, sc := range encodeScenarios {
+		sc := sc
+		t.Run(fmt.Sprintf("encode@%s", trimMarker(sc.marker)), func(t *testing.T) {
+			vid := r.ensureRegularVolume()
+			r.unlockIfHeld()
+			out, hit := runShellKillAtMarker(t, r.ctx,
+				fmt.Sprintf("ec.encode -volumeId %d -collection %s -force", vid, chaosCollection), sc.marker)
+			r.t.Logf("killed %s at %q (marker hit: %v); output:\n%s", sc.op, sc.marker, hit, out)
+			r.relock()
+			require.True(t, hit, "marker %q never appeared (printed by %s); update the marker with the pipeline", sc.marker, sc.printed)
+			r.recoverInterruptedEncode(vid)
+			r.verify(fmt.Sprintf("encode killed at %q", sc.marker))
+		})
+	}
+	for _, sc := range decodeScenarios {
+		sc := sc
+		t.Run(fmt.Sprintf("decode@%s", trimMarker(sc.marker)), func(t *testing.T) {
+			vid := r.ensureEncodedVolume()
+			r.unlockIfHeld()
+			out, hit := runShellKillAtMarker(t, r.ctx,
+				fmt.Sprintf("ec.decode -volumeId %d -collection %s -checkMinFreeSpace=false", vid, chaosCollection), sc.marker)
+			r.t.Logf("killed %s at %q (marker hit: %v); output:\n%s", sc.op, sc.marker, hit, out)
+			r.relock()
+			require.True(t, hit, "marker %q never appeared (printed by %s); update the marker with the pipeline", sc.marker, sc.printed)
+			r.recoverInterruptedDecode(vid)
+			r.verify(fmt.Sprintf("decode killed at %q", sc.marker))
+		})
+	}
+
+	// Balance: encode without rebalancing first, so the standalone balance is
+	// guaranteed to plan moves and the marker is guaranteed to print.
+	t.Run("balance@moves", func(t *testing.T) {
+		vid := r.ensureRegularVolume()
+		out, err := r.shellCommand("ec.encode",
+			"-volumeId", fmt.Sprintf("%d", vid), "-collection", chaosCollection, "-force", "-rebalance=false")
+		r.t.Logf("clumped encode v%d output:\n%s", vid, out)
+		require.NoError(t, err, "encode without rebalance")
+		r.volumes[vid].encoded = true
+
+		r.unlockIfHeld()
+		killOut, hit := runShellKillAtMarker(t, r.ctx,
+			fmt.Sprintf("ec.balance -collection %s -apply", chaosCollection), "moves ec shard")
+		r.t.Logf("killed ec.balance at move (marker hit: %v); output:\n%s", hit, killOut)
+		r.relock()
+		require.True(t, hit, "the balance never printed a move; the -rebalance=false setup should have guaranteed one")
+		r.recoverInterruptedBalance()
+		r.verify("balance killed at move")
+	})
+
+	r.verify("final")
+	t.Logf("interruption matrix done: %d payloads live, %d deleted, %d volumes tracked",
+		len(r.payloads), len(r.deleted), len(r.volumes))
+}
+
+func trimMarker(m string) string {
+	out := make([]rune, 0, len(m))
+	for _, c := range m {
+		if c == ' ' || c == ':' {
+			continue
+		}
+		out = append(out, c)
+	}
+	return string(out)
+}
+
+// runShellKillAtMarker feeds "lock" plus the command to a weed shell
+// subprocess, scans its combined output live, and kills the process the
+// moment a line containing marker appears. Returns the captured output and
+// whether the marker was seen before the shell exited on its own.
+func runShellKillAtMarker(t *testing.T, ctx context.Context, command, marker string) (string, bool) {
+	t.Helper()
+	weedBinary := findWeedBinary()
+	require.NotEmpty(t, weedBinary, "weed binary not found")
+	cmd := exec.CommandContext(ctx, weedBinary, "shell", "-master="+chaosMasterAddr)
+
+	pr, pw, err := os.Pipe()
+	require.NoError(t, err)
+	cmd.Stdout, cmd.Stderr = pw, pw
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+	require.NoError(t, cmd.Start())
+	pw.Close() // the child keeps its dup; ours would hold the reader open
+
+	var buf bytes.Buffer
+	var mu sync.Mutex
+	markerSeen := make(chan struct{})
+	scanDone := make(chan struct{})
+	go func() {
+		defer close(scanDone)
+		signaled := false
+		scanner := bufio.NewScanner(pr)
+		scanner.Buffer(make([]byte, 0, 256*1024), 1024*1024)
+		for scanner.Scan() {
+			line := scanner.Text()
+			mu.Lock()
+			buf.WriteString(line)
+			buf.WriteByte('\n')
+			mu.Unlock()
+			if !signaled && bytesContains(line, marker) {
+				signaled = true
+				close(markerSeen)
+			}
+		}
+	}()
+
+	fmt.Fprintf(stdin, "lock\n%s\n", command)
+	// Stdin stays open: the shell must die by kill, never by graceful EOF —
+	// a graceful exit would release the cluster lock cleanly and dodge the
+	// dead-holder cleanup path this test also exercises.
+	hit := false
+	select {
+	case <-markerSeen:
+		hit = true
+	case <-scanDone: // shell exited before printing the marker
+	case <-time.After(90 * time.Second):
+	}
+	if !hit {
+		// A shell that printed the marker and exited immediately can make both
+		// channels ready at once, and select picks between ready cases at
+		// random; a printed marker must never be reported as missed.
+		select {
+		case <-markerSeen:
+			hit = true
+		default:
+		}
+	}
+	cmd.Process.Kill()
+	cmd.Wait()
+	pr.Close()
+	<-scanDone
+	mu.Lock()
+	defer mu.Unlock()
+	return buf.String(), hit
+}
+
+func bytesContains(line, marker string) bool {
+	return len(marker) > 0 && strings.Contains(line, marker)
+}