snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
312
test/fuse_integration/Makefile
Normal file
312
test/fuse_integration/Makefile
Normal file
@@ -0,0 +1,312 @@
|
||||
# SeaweedFS FUSE Integration Testing Makefile
|
||||
|
||||
# Configuration
|
||||
WEED_BINARY := weed
|
||||
GO_VERSION := 1.26
|
||||
TEST_TIMEOUT := 30m
|
||||
COVERAGE_FILE := coverage.out
|
||||
|
||||
# Default target
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
# Check if weed binary exists
|
||||
check-binary:
|
||||
@if [ ! -f "$(WEED_BINARY)" ]; then \
|
||||
echo "ERROR SeaweedFS binary not found at $(WEED_BINARY)"; \
|
||||
echo " Please run 'make' in the root directory first"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "OK SeaweedFS binary found"
|
||||
|
||||
# Check FUSE installation
|
||||
check-fuse:
|
||||
@if command -v fusermount >/dev/null 2>&1; then \
|
||||
echo "OK FUSE is installed (Linux)"; \
|
||||
elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \
|
||||
echo "OK FUSE is available (macOS)"; \
|
||||
else \
|
||||
echo "ERROR FUSE not found. Please install:"; \
|
||||
echo " Ubuntu/Debian: sudo apt-get install fuse"; \
|
||||
echo " CentOS/RHEL: sudo yum install fuse"; \
|
||||
echo " macOS: brew install macfuse"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Check Go version
|
||||
check-go:
|
||||
@go version | grep -q "go1\.[2-9][0-9]" || \
|
||||
go version | grep -q "go1\.2[1-9]" || \
|
||||
(echo "ERROR Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1)
|
||||
@echo "OK Go version check passed"
|
||||
|
||||
# Verify all prerequisites
|
||||
check-prereqs: check-go check-fuse
|
||||
@echo "✅ All prerequisites satisfied"
|
||||
|
||||
# Build the SeaweedFS binary (if needed)
|
||||
build:
|
||||
@echo "Building SeaweedFS..."
|
||||
cd ../.. && make
|
||||
@echo "OK Build complete"
|
||||
|
||||
# Initialize go module (if needed)
|
||||
init-module:
|
||||
@if [ ! -f go.mod ]; then \
|
||||
echo "📦 Initializing Go module..."; \
|
||||
go mod init seaweedfs-fuse-tests; \
|
||||
go mod tidy; \
|
||||
fi
|
||||
|
||||
# Run all tests
|
||||
test: check-prereqs init-module
|
||||
@echo "🧪 Running all FUSE integration tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) ./...
|
||||
|
||||
# Run tests with coverage
|
||||
test-coverage: check-prereqs init-module
|
||||
@echo "🧪 Running tests with coverage..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -coverprofile=$(COVERAGE_FILE) ./...
|
||||
go tool cover -html=$(COVERAGE_FILE) -o coverage.html
|
||||
@echo "📊 Coverage report generated: coverage.html"
|
||||
|
||||
# Run specific test categories
|
||||
test-basic: check-prereqs init-module
|
||||
@echo "🧪 Running basic file operations tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run TestBasicFileOperations
|
||||
|
||||
test-directory: check-prereqs init-module
|
||||
@echo "🧪 Running directory operations tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run TestDirectoryOperations
|
||||
|
||||
test-concurrent: check-prereqs init-module
|
||||
@echo "🧪 Running concurrent operations tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run TestConcurrentFileOperations
|
||||
|
||||
test-stress: check-prereqs init-module
|
||||
@echo "🧪 Running stress tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run TestStressOperations
|
||||
|
||||
test-large-files: check-prereqs init-module
|
||||
@echo "🧪 Running large file tests..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -run TestLargeFileOperations
|
||||
|
||||
# Run tests with debugging enabled
|
||||
test-debug: check-prereqs init-module
|
||||
@echo "🔍 Running tests with debug output..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -args -debug
|
||||
|
||||
# Run tests and keep temp files for inspection
|
||||
test-no-cleanup: check-prereqs init-module
|
||||
@echo "🧪 Running tests without cleanup (for debugging)..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -args -no-cleanup
|
||||
|
||||
# Quick smoke test
|
||||
test-smoke: check-prereqs init-module
|
||||
@echo "💨 Running smoke tests..."
|
||||
go test -v -timeout 5m -run TestBasicFileOperations/CreateAndReadFile
|
||||
|
||||
# Run benchmarks
|
||||
benchmark: check-prereqs init-module
|
||||
@echo "📈 Running benchmarks..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -bench=. -benchmem
|
||||
|
||||
# Validate test files compile
|
||||
validate: init-module
|
||||
@echo "✅ Validating test files..."
|
||||
go build -o /dev/null ./...
|
||||
@echo "✅ All test files compile successfully"
|
||||
|
||||
# Clean up generated files
|
||||
clean:
|
||||
@echo "🧹 Cleaning up..."
|
||||
rm -f $(COVERAGE_FILE) coverage.html
|
||||
rm -rf /tmp/seaweedfs_fuse_test_*
|
||||
go clean -testcache
|
||||
@echo "✅ Cleanup complete"
|
||||
|
||||
# Format Go code
|
||||
fmt:
|
||||
@echo "🎨 Formatting Go code..."
|
||||
go fmt ./...
|
||||
|
||||
# Run linter
|
||||
lint:
|
||||
@echo "🔍 Running linter..."
|
||||
@if command -v golangci-lint >/dev/null 2>&1; then \
|
||||
golangci-lint run; \
|
||||
else \
|
||||
echo "⚠️ golangci-lint not found, running go vet instead"; \
|
||||
go vet ./...; \
|
||||
fi
|
||||
|
||||
# Run all quality checks
|
||||
check: validate lint fmt
|
||||
@echo "✅ All quality checks passed"
|
||||
|
||||
# Install development dependencies
|
||||
install-deps:
|
||||
@echo "📦 Installing development dependencies..."
|
||||
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
# Quick development setup
|
||||
setup: install-deps build check-prereqs
|
||||
@echo "🚀 Development environment ready!"
|
||||
|
||||
# Docker-based testing
|
||||
test-docker:
|
||||
@echo "🐳 Running tests in Docker..."
|
||||
docker build -t seaweedfs-fuse-tests -f Dockerfile.test ../..
|
||||
docker run --rm --privileged seaweedfs-fuse-tests
|
||||
|
||||
# Create Docker test image
|
||||
docker-build:
|
||||
@echo "🐳 Building Docker test image..."
|
||||
@cat > Dockerfile.test << 'EOF' ;\
|
||||
FROM golang:$(GO_VERSION) ;\
|
||||
RUN apt-get update && apt-get install -y fuse ;\
|
||||
WORKDIR /seaweedfs ;\
|
||||
COPY . . ;\
|
||||
RUN make ;\
|
||||
WORKDIR /seaweedfs/test/fuse ;\
|
||||
RUN go mod init seaweedfs-fuse-tests && go mod tidy ;\
|
||||
CMD ["make", "test"] ;\
|
||||
EOF
|
||||
|
||||
# GitHub Actions workflow
|
||||
generate-workflow:
|
||||
@echo "📝 Generating GitHub Actions workflow..."
|
||||
@mkdir -p ../../.github/workflows
|
||||
@cat > ../../.github/workflows/fuse-integration.yml << 'EOF' ;\
|
||||
name: FUSE Integration Tests ;\
|
||||
;\
|
||||
on: ;\
|
||||
push: ;\
|
||||
branches: [ master, main ] ;\
|
||||
pull_request: ;\
|
||||
branches: [ master, main ] ;\
|
||||
;\
|
||||
jobs: ;\
|
||||
fuse-integration: ;\
|
||||
runs-on: ubuntu-latest ;\
|
||||
timeout-minutes: 45 ;\
|
||||
;\
|
||||
steps: ;\
|
||||
- name: Checkout code ;\
|
||||
uses: actions/checkout@v4 ;\
|
||||
;\
|
||||
- name: Set up Go ;\
|
||||
uses: actions/setup-go@v4 ;\
|
||||
with: ;\
|
||||
go-version: '$(GO_VERSION)' ;\
|
||||
;\
|
||||
- name: Install FUSE ;\
|
||||
run: sudo apt-get update && sudo apt-get install -y fuse ;\
|
||||
;\
|
||||
- name: Build SeaweedFS ;\
|
||||
run: make ;\
|
||||
;\
|
||||
- name: Run FUSE Integration Tests ;\
|
||||
run: | ;\
|
||||
cd test/fuse ;\
|
||||
make test ;\
|
||||
;\
|
||||
- name: Upload test artifacts ;\
|
||||
if: failure() ;\
|
||||
uses: actions/upload-artifact@v3 ;\
|
||||
with: ;\
|
||||
name: test-logs ;\
|
||||
path: /tmp/seaweedfs_fuse_test_* ;\
|
||||
EOF
|
||||
@echo "✅ GitHub Actions workflow generated"
|
||||
|
||||
# Performance profiling
|
||||
profile: check-prereqs init-module
|
||||
@echo "📊 Running performance profiling..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -cpuprofile cpu.prof -memprofile mem.prof -bench=.
|
||||
@echo "📊 Profiles generated: cpu.prof, mem.prof"
|
||||
@echo "📊 View with: go tool pprof cpu.prof"
|
||||
|
||||
# Memory leak detection
|
||||
test-memory: check-prereqs init-module
|
||||
@echo "🔍 Running memory leak detection..."
|
||||
go test -v -timeout $(TEST_TIMEOUT) -race -test.memprofile mem.prof
|
||||
|
||||
# List available test functions
|
||||
list-tests:
|
||||
@echo "📋 Available test functions:"
|
||||
@grep -r "^func Test" *.go | sed 's/.*func \(Test[^(]*\).*/ \1/' | sort
|
||||
|
||||
# Get test status and statistics
|
||||
test-stats: check-prereqs init-module
|
||||
@echo "📊 Test statistics:"
|
||||
@go test -v ./... | grep -E "(PASS|FAIL|RUN)" | \
|
||||
awk '{ \
|
||||
if ($$1 == "RUN") tests++; \
|
||||
else if ($$1 == "PASS") passed++; \
|
||||
else if ($$1 == "FAIL") failed++; \
|
||||
} END { \
|
||||
printf " Total tests: %d\n", tests; \
|
||||
printf " Passed: %d\n", passed; \
|
||||
printf " Failed: %d\n", failed; \
|
||||
printf " Success rate: %.1f%%\n", (passed/tests)*100; \
|
||||
}'
|
||||
|
||||
# Watch for file changes and run tests
|
||||
watch:
|
||||
@echo "👀 Watching for changes..."
|
||||
@if command -v entr >/dev/null 2>&1; then \
|
||||
find . -name "*.go" | entr -c make test-smoke; \
|
||||
else \
|
||||
echo "⚠️ 'entr' not found. Install with: apt-get install entr"; \
|
||||
echo " Falling back to manual test run"; \
|
||||
make test-smoke; \
|
||||
fi
|
||||
|
||||
# Show help
|
||||
help:
|
||||
@echo "SeaweedFS FUSE Integration Testing"
|
||||
@echo "=================================="
|
||||
@echo ""
|
||||
@echo "Prerequisites:"
|
||||
@echo " make check-prereqs - Check all prerequisites"
|
||||
@echo " make setup - Complete development setup"
|
||||
@echo " make build - Build SeaweedFS binary"
|
||||
@echo ""
|
||||
@echo "Testing:"
|
||||
@echo " make test - Run all tests"
|
||||
@echo " make test-basic - Run basic file operations tests"
|
||||
@echo " make test-directory - Run directory operations tests"
|
||||
@echo " make test-concurrent - Run concurrent operations tests"
|
||||
@echo " make test-stress - Run stress tests"
|
||||
@echo " make test-smoke - Quick smoke test"
|
||||
@echo " make test-coverage - Run tests with coverage report"
|
||||
@echo ""
|
||||
@echo "Debugging:"
|
||||
@echo " make test-debug - Run tests with debug output"
|
||||
@echo " make test-no-cleanup - Keep temp files for inspection"
|
||||
@echo " make profile - Performance profiling"
|
||||
@echo " make test-memory - Memory leak detection"
|
||||
@echo ""
|
||||
@echo "Quality:"
|
||||
@echo " make validate - Validate test files compile"
|
||||
@echo " make lint - Run linter"
|
||||
@echo " make fmt - Format code"
|
||||
@echo " make check - Run all quality checks"
|
||||
@echo ""
|
||||
@echo "Utilities:"
|
||||
@echo " make clean - Clean up generated files"
|
||||
@echo " make list-tests - List available test functions"
|
||||
@echo " make test-stats - Show test statistics"
|
||||
@echo " make watch - Watch files and run smoke tests"
|
||||
@echo ""
|
||||
@echo "Docker & CI:"
|
||||
@echo " make test-docker - Run tests in Docker"
|
||||
@echo " make generate-workflow - Generate GitHub Actions workflow"
|
||||
|
||||
.PHONY: help check-prereqs check-binary check-fuse check-go build init-module \
|
||||
test test-coverage test-basic test-directory test-concurrent test-stress \
|
||||
test-large-files test-debug test-no-cleanup test-smoke benchmark validate \
|
||||
clean fmt lint check install-deps setup test-docker docker-build \
|
||||
generate-workflow profile test-memory list-tests test-stats watch
|
||||
327
test/fuse_integration/README.md
Normal file
327
test/fuse_integration/README.md
Normal file
@@ -0,0 +1,327 @@
|
||||
# SeaweedFS FUSE Integration Testing Framework
|
||||
|
||||
## Overview
|
||||
|
||||
This directory contains a comprehensive integration testing framework for SeaweedFS FUSE operations. The current SeaweedFS FUSE tests are primarily performance-focused (using FIO) but lack comprehensive functional testing. This framework addresses those gaps.
|
||||
|
||||
## ⚠️ Current Status
|
||||
|
||||
**Note**: Due to Go module conflicts between this test framework and the parent SeaweedFS module, the full test suite currently requires manual setup. The framework files are provided as a foundation for comprehensive FUSE testing once the module structure is resolved.
|
||||
|
||||
### Working Components
|
||||
- ✅ Framework design and architecture (`framework.go`)
|
||||
- ✅ Individual test file structure and compilation
|
||||
- ✅ Test methodology and comprehensive coverage
|
||||
- ✅ Documentation and usage examples
|
||||
- ⚠️ Full test suite execution (requires Go module isolation)
|
||||
|
||||
### Verified Working Test
|
||||
```bash
|
||||
cd test/fuse_integration
|
||||
go test -v simple_test.go
|
||||
```
|
||||
|
||||
## Current Testing Gaps Addressed
|
||||
|
||||
### 1. **Limited Functional Coverage**
|
||||
- **Current**: Only basic FIO performance tests
|
||||
- **New**: Comprehensive testing of all FUSE operations (create, read, write, delete, mkdir, rmdir, permissions, etc.)
|
||||
|
||||
### 2. **No Concurrency Testing**
|
||||
- **Current**: Single-threaded performance tests
|
||||
- **New**: Extensive concurrent operation tests, race condition detection, thread safety validation
|
||||
|
||||
### 3. **Insufficient Error Handling**
|
||||
- **Current**: Basic error scenarios
|
||||
- **New**: Comprehensive error condition testing, edge cases, failure recovery
|
||||
|
||||
### 4. **Missing Edge Cases**
|
||||
- **Current**: Simple file operations
|
||||
- **New**: Large files, sparse files, deep directory nesting, many small files, permission variations
|
||||
|
||||
## Framework Architecture
|
||||
|
||||
### Core Components
|
||||
|
||||
1. **`framework.go`** - Test infrastructure and utilities
|
||||
- `FuseTestFramework` - Main test management struct
|
||||
- Automated SeaweedFS cluster setup/teardown
|
||||
- FUSE mount/unmount management
|
||||
- Helper functions for file operations and assertions
|
||||
|
||||
2. **`basic_operations_test.go`** - Fundamental FUSE operations
|
||||
- File create, read, write, delete
|
||||
- File attributes and permissions
|
||||
- Large file handling
|
||||
- Sparse file operations
|
||||
|
||||
3. **`directory_operations_test.go`** - Directory-specific tests
|
||||
- Directory creation, deletion, listing
|
||||
- Nested directory structures
|
||||
- Directory permissions and rename operations
|
||||
- Complex directory scenarios
|
||||
|
||||
4. **`concurrent_operations_test.go`** - Concurrency and stress testing
|
||||
- Concurrent file and directory operations
|
||||
- Race condition detection
|
||||
- High-frequency operations
|
||||
- Stress testing scenarios
|
||||
|
||||
## Key Features
|
||||
|
||||
### Automated Test Environment
|
||||
```go
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
```
|
||||
|
||||
- **Automatic cluster setup**: Master, Volume, Filer servers
|
||||
- **FUSE mounting**: Proper mount point management
|
||||
- **Cleanup**: Automatic teardown of all resources
|
||||
|
||||
### Configurable Test Parameters
|
||||
```go
|
||||
config := &TestConfig{
|
||||
Collection: "test",
|
||||
Replication: "001",
|
||||
ChunkSizeMB: 8,
|
||||
CacheSizeMB: 200,
|
||||
NumVolumes: 5,
|
||||
EnableDebug: true,
|
||||
MountOptions: []string{"-allowOthers"},
|
||||
}
|
||||
```
|
||||
|
||||
### Rich Assertion Helpers
|
||||
```go
|
||||
framework.AssertFileExists("path/to/file")
|
||||
framework.AssertFileContent("file.txt", expectedContent)
|
||||
framework.AssertFileMode("script.sh", 0755)
|
||||
framework.CreateTestFile("test.txt", []byte("content"))
|
||||
```
|
||||
|
||||
## Test Categories
|
||||
|
||||
### 1. Basic File Operations
|
||||
- **Create/Read/Write/Delete**: Fundamental file operations
|
||||
- **File Attributes**: Size, timestamps, permissions
|
||||
- **Append Operations**: File appending behavior
|
||||
- **Large Files**: Files exceeding chunk size limits
|
||||
- **Sparse Files**: Non-contiguous file data
|
||||
|
||||
### 2. Directory Operations
|
||||
- **Directory Lifecycle**: Create, list, remove directories
|
||||
- **Nested Structures**: Deep directory hierarchies
|
||||
- **Directory Permissions**: Access control testing
|
||||
- **Directory Rename**: Move operations
|
||||
- **Complex Scenarios**: Many files, deep nesting
|
||||
|
||||
### 3. Concurrent Operations
|
||||
- **Multi-threaded Access**: Simultaneous file operations
|
||||
- **Race Condition Detection**: Concurrent read/write scenarios
|
||||
- **Directory Concurrency**: Parallel directory operations
|
||||
- **Stress Testing**: High-frequency operations
|
||||
|
||||
### 4. Error Handling & Edge Cases
|
||||
- **Permission Denied**: Access control violations
|
||||
- **Disk Full**: Storage limit scenarios
|
||||
- **Network Issues**: Filer/Volume server failures
|
||||
- **Invalid Operations**: Malformed requests
|
||||
- **Recovery Testing**: Error recovery scenarios
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Test Run
|
||||
```bash
|
||||
# Build SeaweedFS binary
|
||||
make
|
||||
|
||||
# Run all FUSE tests
|
||||
cd test/fuse_integration
|
||||
go test -v
|
||||
|
||||
# Run specific test category
|
||||
go test -v -run TestBasicFileOperations
|
||||
go test -v -run TestConcurrentFileOperations
|
||||
```
|
||||
|
||||
### Custom Configuration
|
||||
```go
|
||||
func TestCustomFUSE(t *testing.T) {
|
||||
config := &TestConfig{
|
||||
ChunkSizeMB: 16, // Larger chunks
|
||||
CacheSizeMB: 500, // More cache
|
||||
EnableDebug: true, // Debug output
|
||||
SkipCleanup: true, // Keep files for inspection
|
||||
}
|
||||
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
// Your tests here...
|
||||
}
|
||||
```
|
||||
|
||||
### Debugging Failed Tests
|
||||
```go
|
||||
config := &TestConfig{
|
||||
EnableDebug: true, // Enable verbose logging
|
||||
SkipCleanup: true, // Keep temp files for inspection
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Performance Benchmarking
|
||||
```go
|
||||
func BenchmarkLargeFileWrite(b *testing.B) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// Benchmark file operations
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Custom Test Scenarios
|
||||
```go
|
||||
func TestCustomWorkload(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
// Simulate specific application workload
|
||||
simulateWebServerWorkload(t, framework)
|
||||
simulateDatabaseWorkload(t, framework)
|
||||
simulateBackupWorkload(t, framework)
|
||||
}
|
||||
```
|
||||
|
||||
## Integration with CI/CD
|
||||
|
||||
### GitHub Actions Example
|
||||
```yaml
|
||||
name: FUSE Integration Tests
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
fuse-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.26'
|
||||
|
||||
- name: Install FUSE
|
||||
run: sudo apt-get install -y fuse
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: make
|
||||
|
||||
- name: Run FUSE Tests
|
||||
run: |
|
||||
cd test/fuse_integration
|
||||
go test -v -timeout 30m
|
||||
```
|
||||
|
||||
### Docker Testing
|
||||
```dockerfile
|
||||
FROM golang:1.26
|
||||
RUN apt-get update && apt-get install -y fuse
|
||||
COPY . /seaweedfs
|
||||
WORKDIR /seaweedfs
|
||||
RUN make
|
||||
CMD ["go", "test", "-v", "./test/fuse_integration/..."]
|
||||
```
|
||||
|
||||
## Comparison with Current Testing
|
||||
|
||||
| Aspect | Current Tests | New Framework |
|
||||
|--------|---------------|---------------|
|
||||
| **Operations Covered** | Basic FIO read/write | All FUSE operations |
|
||||
| **Concurrency** | Single-threaded | Multi-threaded stress tests |
|
||||
| **Error Scenarios** | Limited | Comprehensive error handling |
|
||||
| **File Types** | Regular files only | Large, sparse, many small files |
|
||||
| **Directory Testing** | None | Complete directory operations |
|
||||
| **Setup Complexity** | Manual Docker setup | Automated cluster management |
|
||||
| **Test Isolation** | Shared environment | Isolated per-test environments |
|
||||
| **Debugging** | Limited | Rich debugging and inspection |
|
||||
|
||||
## Benefits
|
||||
|
||||
### 1. **Comprehensive Coverage**
|
||||
- Tests all FUSE operations supported by SeaweedFS
|
||||
- Covers edge cases and error conditions
|
||||
- Validates behavior under concurrent access
|
||||
|
||||
### 2. **Reliable Testing**
|
||||
- Isolated test environments prevent test interference
|
||||
- Automatic cleanup ensures consistent state
|
||||
- Deterministic test execution
|
||||
|
||||
### 3. **Easy Maintenance**
|
||||
- Clear test organization and naming
|
||||
- Rich helper functions reduce code duplication
|
||||
- Configurable test parameters for different scenarios
|
||||
|
||||
### 4. **Real-world Validation**
|
||||
- Tests actual FUSE filesystem behavior
|
||||
- Validates integration between all SeaweedFS components
|
||||
- Catches issues that unit tests might miss
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### 1. **Extended FUSE Features**
|
||||
- Extended attributes (xattr) testing
|
||||
- Symbolic link operations
|
||||
- Hard link behavior
|
||||
- File locking mechanisms
|
||||
|
||||
### 2. **Performance Profiling**
|
||||
- Built-in performance measurement
|
||||
- Memory usage tracking
|
||||
- Latency distribution analysis
|
||||
- Throughput benchmarking
|
||||
|
||||
### 3. **Fault Injection**
|
||||
- Network partition simulation
|
||||
- Server failure scenarios
|
||||
- Disk full conditions
|
||||
- Memory pressure testing
|
||||
|
||||
### 4. **Integration Testing**
|
||||
- Multi-filer configurations
|
||||
- Cross-datacenter replication
|
||||
- S3 API compatibility while mounted
|
||||
- Backup/restore operations
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Prerequisites**
|
||||
```bash
|
||||
# Install FUSE
|
||||
sudo apt-get install fuse # Ubuntu/Debian
|
||||
brew install macfuse # macOS
|
||||
|
||||
# Build SeaweedFS
|
||||
make
|
||||
```
|
||||
|
||||
2. **Run Tests**
|
||||
```bash
|
||||
cd test/fuse_integration
|
||||
go test -v
|
||||
```
|
||||
|
||||
3. **View Results**
|
||||
- Test output shows detailed operation results
|
||||
- Failed tests include specific error information
|
||||
- Debug mode provides verbose logging
|
||||
|
||||
This framework represents a significant improvement in SeaweedFS FUSE testing capabilities, providing comprehensive coverage, real-world validation, and reliable automation that will help ensure the robustness and reliability of the FUSE implementation.
|
||||
475
test/fuse_integration/concurrent_operations_test.go
Normal file
475
test/fuse_integration/concurrent_operations_test.go
Normal file
@@ -0,0 +1,475 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestConcurrentFileOperations tests concurrent file operations
|
||||
func TestConcurrentFileOperations(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("ConcurrentFileWrites", func(t *testing.T) {
|
||||
testConcurrentFileWrites(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ConcurrentFileReads", func(t *testing.T) {
|
||||
testConcurrentFileReads(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ConcurrentReadWrite", func(t *testing.T) {
|
||||
testConcurrentReadWrite(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ConcurrentDirectoryOperations", func(t *testing.T) {
|
||||
testConcurrentDirectoryOperations(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ConcurrentFileCreation", func(t *testing.T) {
|
||||
testConcurrentFileCreation(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testConcurrentFileWrites tests multiple goroutines writing to different files
|
||||
func testConcurrentFileWrites(t *testing.T, framework *FuseTestFramework) {
|
||||
numWorkers := 10
|
||||
filesPerWorker := 5
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
|
||||
// Function to collect errors safely
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// Start concurrent workers
|
||||
for worker := 0; worker < numWorkers; worker++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for file := 0; file < filesPerWorker; file++ {
|
||||
filename := fmt.Sprintf("worker_%d_file_%d.txt", workerID, file)
|
||||
content := []byte(fmt.Sprintf("Worker %d, File %d - %s", workerID, file, time.Now().String()))
|
||||
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
if err := os.WriteFile(mountPath, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, file, err))
|
||||
return
|
||||
}
|
||||
|
||||
// Verify file was written correctly
|
||||
readContent, err := os.ReadFile(mountPath)
|
||||
if err != nil {
|
||||
addError(fmt.Errorf("worker %d file %d read: %v", workerID, file, err))
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(content, readContent) {
|
||||
addError(fmt.Errorf("worker %d file %d: content mismatch", workerID, file))
|
||||
return
|
||||
}
|
||||
}
|
||||
}(worker)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Check for errors
|
||||
require.Empty(t, errors, "Concurrent writes failed: %v", errors)
|
||||
|
||||
// Verify all files exist and have correct content
|
||||
for worker := 0; worker < numWorkers; worker++ {
|
||||
for file := 0; file < filesPerWorker; file++ {
|
||||
filename := fmt.Sprintf("worker_%d_file_%d.txt", worker, file)
|
||||
framework.AssertFileExists(filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testConcurrentFileReads tests multiple goroutines reading from the same file
|
||||
func testConcurrentFileReads(t *testing.T, framework *FuseTestFramework) {
|
||||
// Create a test file
|
||||
filename := "concurrent_read_test.txt"
|
||||
testData := make([]byte, 1024*1024) // 1MB
|
||||
_, err := rand.Read(testData)
|
||||
require.NoError(t, err)
|
||||
|
||||
framework.CreateTestFile(filename, testData)
|
||||
|
||||
numReaders := 20
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// Start concurrent readers
|
||||
for reader := 0; reader < numReaders; reader++ {
|
||||
wg.Add(1)
|
||||
go func(readerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Read multiple times
|
||||
for i := 0; i < 3; i++ {
|
||||
readData, err := os.ReadFile(mountPath)
|
||||
if err != nil {
|
||||
addError(fmt.Errorf("reader %d iteration %d: %v", readerID, i, err))
|
||||
return
|
||||
}
|
||||
|
||||
if !bytes.Equal(testData, readData) {
|
||||
addError(fmt.Errorf("reader %d iteration %d: data mismatch", readerID, i))
|
||||
return
|
||||
}
|
||||
}
|
||||
}(reader)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errors, "Concurrent reads failed: %v", errors)
|
||||
}
|
||||
|
||||
// testConcurrentReadWrite tests simultaneous read and write operations
|
||||
func testConcurrentReadWrite(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "concurrent_rw_test.txt"
|
||||
initialData := bytes.Repeat([]byte("INITIAL"), 1000)
|
||||
framework.CreateTestFile(filename, initialData)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Start readers
|
||||
numReaders := 5
|
||||
for i := 0; i < numReaders; i++ {
|
||||
wg.Add(1)
|
||||
go func(readerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for j := 0; j < 10; j++ {
|
||||
if err := retryTransientFUSE(func() error {
|
||||
_, e := os.ReadFile(mountPath)
|
||||
return e
|
||||
}); err != nil {
|
||||
addError(fmt.Errorf("reader %d: %v", readerID, err))
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Start writers
|
||||
numWriters := 2
|
||||
for i := 0; i < numWriters; i++ {
|
||||
wg.Add(1)
|
||||
go func(writerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for j := 0; j < 5; j++ {
|
||||
newData := bytes.Repeat([]byte(fmt.Sprintf("WRITER%d", writerID)), 1000)
|
||||
if err := retryTransientFUSE(func() error {
|
||||
return os.WriteFile(mountPath, newData, 0644)
|
||||
}); err != nil {
|
||||
addError(fmt.Errorf("writer %d: %v", writerID, err))
|
||||
return
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errors, "Concurrent read/write failed: %v", errors)
|
||||
|
||||
// Verify file still exists and is readable
|
||||
framework.AssertFileExists(filename)
|
||||
}
|
||||
|
||||
// retryTransientFUSE retries op a few times before giving up. A concurrent
|
||||
// truncating overwrite can leave a short-lived dentry/cache window where the
|
||||
// entry is momentarily invisible (ENOENT) to another opener; the last error is
|
||||
// returned so a genuine, persistent failure still surfaces.
|
||||
func retryTransientFUSE(op func() error) error {
|
||||
var err error
|
||||
for attempt := 0; attempt < 5; attempt++ {
|
||||
if err = op(); err == nil {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// testConcurrentDirectoryOperations tests concurrent directory operations
|
||||
func testConcurrentDirectoryOperations(t *testing.T, framework *FuseTestFramework) {
|
||||
numWorkers := 8
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// Each worker creates a directory tree
|
||||
for worker := 0; worker < numWorkers; worker++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
// Create worker directory
|
||||
workerDir := fmt.Sprintf("worker_%d", workerID)
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), workerDir)
|
||||
|
||||
if err := os.Mkdir(mountPath, 0755); err != nil {
|
||||
addError(fmt.Errorf("worker %d mkdir: %v", workerID, err))
|
||||
return
|
||||
}
|
||||
|
||||
// Create subdirectories and files
|
||||
for i := 0; i < 5; i++ {
|
||||
subDir := filepath.Join(mountPath, fmt.Sprintf("subdir_%d", i))
|
||||
if err := os.Mkdir(subDir, 0755); err != nil {
|
||||
addError(fmt.Errorf("worker %d subdir %d: %v", workerID, i, err))
|
||||
return
|
||||
}
|
||||
|
||||
// Create file in subdirectory with retry for transient FUSE errors
|
||||
testFile := filepath.Join(subDir, "test.txt")
|
||||
content := []byte(fmt.Sprintf("Worker %d, Subdir %d", workerID, i))
|
||||
var writeErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if writeErr = os.WriteFile(testFile, content, 0644); writeErr == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if writeErr != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, i, writeErr))
|
||||
return
|
||||
}
|
||||
}
|
||||
}(worker)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errors, "Concurrent directory operations failed: %v", errors)
|
||||
|
||||
// Verify all structures were created
|
||||
for worker := 0; worker < numWorkers; worker++ {
|
||||
workerDir := fmt.Sprintf("worker_%d", worker)
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), workerDir)
|
||||
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
|
||||
// Check subdirectories
|
||||
for i := 0; i < 5; i++ {
|
||||
subDir := filepath.Join(mountPath, fmt.Sprintf("subdir_%d", i))
|
||||
info, err := os.Stat(subDir)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
|
||||
testFile := filepath.Join(subDir, "test.txt")
|
||||
expectedContent := []byte(fmt.Sprintf("Worker %d, Subdir %d", worker, i))
|
||||
actualContent, err := os.ReadFile(testFile)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedContent, actualContent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testConcurrentFileCreation tests concurrent creation of files in same directory
|
||||
func testConcurrentFileCreation(t *testing.T, framework *FuseTestFramework) {
|
||||
// Create test directory
|
||||
testDir := "concurrent_creation"
|
||||
framework.CreateTestDir(testDir)
|
||||
|
||||
numWorkers := 15
|
||||
filesPerWorker := 10
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
createdFiles := make(map[string]bool)
|
||||
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
addFile := func(filename string) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
createdFiles[filename] = true
|
||||
}
|
||||
|
||||
// Create files concurrently
|
||||
for worker := 0; worker < numWorkers; worker++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for file := 0; file < filesPerWorker; file++ {
|
||||
filename := fmt.Sprintf("file_%d_%d.txt", workerID, file)
|
||||
relativePath := filepath.Join(testDir, filename)
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), relativePath)
|
||||
|
||||
content := []byte(fmt.Sprintf("Worker %d, File %d, Time: %s",
|
||||
workerID, file, time.Now().Format(time.RFC3339Nano)))
|
||||
|
||||
if err := os.WriteFile(mountPath, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("worker %d file %d: %v", workerID, file, err))
|
||||
return
|
||||
}
|
||||
|
||||
addFile(filename)
|
||||
}
|
||||
}(worker)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errors, "Concurrent file creation failed: %v", errors)
|
||||
|
||||
// Verify all files were created
|
||||
expectedCount := numWorkers * filesPerWorker
|
||||
assert.Equal(t, expectedCount, len(createdFiles))
|
||||
|
||||
// Read directory and verify count
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), testDir)
|
||||
entries, err := os.ReadDir(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expectedCount, len(entries))
|
||||
|
||||
// Verify each file exists and has content
|
||||
for filename := range createdFiles {
|
||||
relativePath := filepath.Join(testDir, filename)
|
||||
framework.AssertFileExists(relativePath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStressOperations tests high-load scenarios
|
||||
func TestStressOperations(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("HighFrequencySmallWrites", func(t *testing.T) {
|
||||
testHighFrequencySmallWrites(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ManySmallFiles", func(t *testing.T) {
|
||||
testManySmallFiles(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testHighFrequencySmallWrites tests many small writes to the same file
|
||||
func testHighFrequencySmallWrites(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "high_freq_writes.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Open file for writing
|
||||
file, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
require.NoError(t, err)
|
||||
defer file.Close()
|
||||
|
||||
// Perform many small writes
|
||||
numWrites := 1000
|
||||
writeSize := 100
|
||||
totalSize := int64(0)
|
||||
|
||||
for i := 0; i < numWrites; i++ {
|
||||
data := []byte(fmt.Sprintf("Write %04d: %s\n", i, bytes.Repeat([]byte("x"), writeSize-20)))
|
||||
_, err := file.Write(data)
|
||||
require.NoError(t, err)
|
||||
totalSize += int64(len(data))
|
||||
}
|
||||
file.Close()
|
||||
|
||||
// Verify file size
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, totalSize, info.Size(), "file size should match total bytes written")
|
||||
}
|
||||
|
||||
// testManySmallFiles tests creating many small files
|
||||
func testManySmallFiles(t *testing.T, framework *FuseTestFramework) {
|
||||
testDir := "many_small_files"
|
||||
framework.CreateTestDir(testDir)
|
||||
|
||||
numFiles := 500
|
||||
var wg sync.WaitGroup
|
||||
var mutex sync.Mutex
|
||||
errors := make([]error, 0)
|
||||
|
||||
addError := func(err error) {
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// Create files in batches
|
||||
batchSize := 50
|
||||
for batch := 0; batch < numFiles/batchSize; batch++ {
|
||||
wg.Add(1)
|
||||
go func(batchID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for i := 0; i < batchSize; i++ {
|
||||
fileNum := batchID*batchSize + i
|
||||
filename := filepath.Join(testDir, fmt.Sprintf("small_file_%04d.txt", fileNum))
|
||||
content := []byte(fmt.Sprintf("File %d content", fileNum))
|
||||
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
if err := os.WriteFile(mountPath, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("file %d: %v", fileNum, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
}(batch)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errors, "Many small files creation failed: %v", errors)
|
||||
|
||||
// Verify directory listing
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), testDir)
|
||||
entries, err := os.ReadDir(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, numFiles, len(entries))
|
||||
}
|
||||
351
test/fuse_integration/directory_operations_test.go
Normal file
351
test/fuse_integration/directory_operations_test.go
Normal file
@@ -0,0 +1,351 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDirectoryOperations tests fundamental FUSE directory operations
|
||||
func TestDirectoryOperations(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("CreateDirectory", func(t *testing.T) {
|
||||
testCreateDirectory(t, framework)
|
||||
})
|
||||
|
||||
t.Run("RemoveDirectory", func(t *testing.T) {
|
||||
testRemoveDirectory(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ReadDirectory", func(t *testing.T) {
|
||||
testReadDirectory(t, framework)
|
||||
})
|
||||
|
||||
t.Run("NestedDirectories", func(t *testing.T) {
|
||||
testNestedDirectories(t, framework)
|
||||
})
|
||||
|
||||
t.Run("DirectoryPermissions", func(t *testing.T) {
|
||||
testDirectoryPermissions(t, framework)
|
||||
})
|
||||
|
||||
t.Run("DirectoryRename", func(t *testing.T) {
|
||||
testDirectoryRename(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testCreateDirectory tests creating directories
|
||||
func testCreateDirectory(t *testing.T, framework *FuseTestFramework) {
|
||||
dirName := "test_directory"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), dirName)
|
||||
|
||||
// Create directory
|
||||
require.NoError(t, os.Mkdir(mountPath, 0755))
|
||||
|
||||
// Verify directory exists
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
assert.Equal(t, os.FileMode(0755), info.Mode().Perm())
|
||||
}
|
||||
|
||||
// testRemoveDirectory tests removing directories
|
||||
func testRemoveDirectory(t *testing.T, framework *FuseTestFramework) {
|
||||
dirName := "test_remove_dir"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), dirName)
|
||||
|
||||
// Create directory
|
||||
require.NoError(t, os.Mkdir(mountPath, 0755))
|
||||
|
||||
// Verify it exists
|
||||
_, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Remove directory
|
||||
require.NoError(t, os.Remove(mountPath))
|
||||
|
||||
// Verify it's gone
|
||||
_, err = os.Stat(mountPath)
|
||||
require.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
// testReadDirectory tests reading directory contents
|
||||
func testReadDirectory(t *testing.T, framework *FuseTestFramework) {
|
||||
testDir := "test_read_dir"
|
||||
framework.CreateTestDir(testDir)
|
||||
|
||||
// Create various types of entries
|
||||
entries := []string{
|
||||
"file1.txt",
|
||||
"file2.log",
|
||||
"subdir1",
|
||||
"subdir2",
|
||||
"script.sh",
|
||||
}
|
||||
|
||||
// Create files and subdirectories
|
||||
for _, entry := range entries {
|
||||
entryPath := filepath.Join(testDir, entry)
|
||||
if entry == "subdir1" || entry == "subdir2" {
|
||||
framework.CreateTestDir(entryPath)
|
||||
} else {
|
||||
framework.CreateTestFile(entryPath, []byte("content of "+entry))
|
||||
}
|
||||
}
|
||||
|
||||
// Read directory
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), testDir)
|
||||
dirEntries, err := os.ReadDir(mountPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all entries are present
|
||||
var actualNames []string
|
||||
for _, entry := range dirEntries {
|
||||
actualNames = append(actualNames, entry.Name())
|
||||
}
|
||||
|
||||
sort.Strings(entries)
|
||||
sort.Strings(actualNames)
|
||||
assert.Equal(t, entries, actualNames)
|
||||
|
||||
// Verify entry types
|
||||
for _, entry := range dirEntries {
|
||||
if entry.Name() == "subdir1" || entry.Name() == "subdir2" {
|
||||
assert.True(t, entry.IsDir())
|
||||
} else {
|
||||
assert.False(t, entry.IsDir())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testNestedDirectories tests operations on nested directory structures
|
||||
func testNestedDirectories(t *testing.T, framework *FuseTestFramework) {
|
||||
// Create nested structure: parent/child1/grandchild/child2
|
||||
structure := []string{
|
||||
"parent",
|
||||
"parent/child1",
|
||||
"parent/child1/grandchild",
|
||||
"parent/child2",
|
||||
}
|
||||
|
||||
// Create directories
|
||||
for _, dir := range structure {
|
||||
framework.CreateTestDir(dir)
|
||||
}
|
||||
|
||||
// Create files at various levels
|
||||
files := map[string][]byte{
|
||||
"parent/root_file.txt": []byte("root level"),
|
||||
"parent/child1/child_file.txt": []byte("child level"),
|
||||
"parent/child1/grandchild/deep_file.txt": []byte("deep level"),
|
||||
"parent/child2/another_file.txt": []byte("another child"),
|
||||
}
|
||||
|
||||
for path, content := range files {
|
||||
framework.CreateTestFile(path, content)
|
||||
}
|
||||
|
||||
// Verify structure by walking
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), "parent")
|
||||
var foundPaths []string
|
||||
|
||||
err := filepath.Walk(mountPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get relative path from mount point
|
||||
relPath, _ := filepath.Rel(framework.GetMountPoint(), path)
|
||||
foundPaths = append(foundPaths, relPath)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify all expected paths were found
|
||||
expectedPaths := []string{
|
||||
"parent",
|
||||
"parent/child1",
|
||||
"parent/child1/grandchild",
|
||||
"parent/child1/grandchild/deep_file.txt",
|
||||
"parent/child1/child_file.txt",
|
||||
"parent/child2",
|
||||
"parent/child2/another_file.txt",
|
||||
"parent/root_file.txt",
|
||||
}
|
||||
|
||||
sort.Strings(expectedPaths)
|
||||
sort.Strings(foundPaths)
|
||||
assert.Equal(t, expectedPaths, foundPaths)
|
||||
|
||||
// Verify file contents
|
||||
for path, expectedContent := range files {
|
||||
framework.AssertFileContent(path, expectedContent)
|
||||
}
|
||||
}
|
||||
|
||||
// testDirectoryPermissions tests directory permission operations
|
||||
func testDirectoryPermissions(t *testing.T, framework *FuseTestFramework) {
|
||||
dirName := "test_permissions_dir"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), dirName)
|
||||
|
||||
// Create directory with specific permissions
|
||||
require.NoError(t, os.Mkdir(mountPath, 0700))
|
||||
|
||||
// Check initial permissions
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0700), info.Mode().Perm())
|
||||
|
||||
// Change permissions
|
||||
require.NoError(t, os.Chmod(mountPath, 0755))
|
||||
|
||||
// Verify permission change
|
||||
info, err = os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0755), info.Mode().Perm())
|
||||
}
|
||||
|
||||
// testDirectoryRename tests renaming directories
|
||||
func testDirectoryRename(t *testing.T, framework *FuseTestFramework) {
|
||||
oldName := "old_directory"
|
||||
newName := "new_directory"
|
||||
|
||||
// Create directory with content
|
||||
framework.CreateTestDir(oldName)
|
||||
framework.CreateTestFile(filepath.Join(oldName, "test_file.txt"), []byte("test content"))
|
||||
|
||||
oldPath := filepath.Join(framework.GetMountPoint(), oldName)
|
||||
newPath := filepath.Join(framework.GetMountPoint(), newName)
|
||||
|
||||
// Rename directory
|
||||
require.NoError(t, os.Rename(oldPath, newPath))
|
||||
|
||||
// Verify old path doesn't exist
|
||||
_, err := os.Stat(oldPath)
|
||||
require.True(t, os.IsNotExist(err))
|
||||
|
||||
// Verify new path exists and is a directory
|
||||
info, err := os.Stat(newPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
|
||||
// Verify content still exists
|
||||
framework.AssertFileContent(filepath.Join(newName, "test_file.txt"), []byte("test content"))
|
||||
}
|
||||
|
||||
// TestComplexDirectoryOperations tests more complex directory scenarios
|
||||
func TestComplexDirectoryOperations(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("RemoveNonEmptyDirectory", func(t *testing.T) {
|
||||
testRemoveNonEmptyDirectory(t, framework)
|
||||
})
|
||||
|
||||
t.Run("DirectoryWithManyFiles", func(t *testing.T) {
|
||||
testDirectoryWithManyFiles(t, framework)
|
||||
})
|
||||
|
||||
t.Run("DeepDirectoryNesting", func(t *testing.T) {
|
||||
testDeepDirectoryNesting(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testRemoveNonEmptyDirectory tests behavior when trying to remove non-empty directories
|
||||
func testRemoveNonEmptyDirectory(t *testing.T, framework *FuseTestFramework) {
|
||||
dirName := "non_empty_dir"
|
||||
framework.CreateTestDir(dirName)
|
||||
|
||||
// Add content to directory
|
||||
framework.CreateTestFile(filepath.Join(dirName, "file.txt"), []byte("content"))
|
||||
framework.CreateTestDir(filepath.Join(dirName, "subdir"))
|
||||
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), dirName)
|
||||
|
||||
// Try to remove non-empty directory (should fail)
|
||||
err := os.Remove(mountPath)
|
||||
require.Error(t, err)
|
||||
|
||||
// Directory should still exist
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
|
||||
// Remove with RemoveAll should work
|
||||
require.NoError(t, os.RemoveAll(mountPath))
|
||||
|
||||
// Verify it's gone
|
||||
_, err = os.Stat(mountPath)
|
||||
require.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
// testDirectoryWithManyFiles tests directories with large numbers of files
|
||||
func testDirectoryWithManyFiles(t *testing.T, framework *FuseTestFramework) {
|
||||
dirName := "many_files_dir"
|
||||
framework.CreateTestDir(dirName)
|
||||
|
||||
// Create many files
|
||||
numFiles := 100
|
||||
for i := 0; i < numFiles; i++ {
|
||||
filename := filepath.Join(dirName, fmt.Sprintf("file_%03d.txt", i))
|
||||
content := []byte(fmt.Sprintf("Content of file %d", i))
|
||||
framework.CreateTestFile(filename, content)
|
||||
}
|
||||
|
||||
// Read directory
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), dirName)
|
||||
entries, err := os.ReadDir(mountPath)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify count
|
||||
assert.Equal(t, numFiles, len(entries))
|
||||
|
||||
// Verify some random files
|
||||
testIndices := []int{0, 10, 50, 99}
|
||||
for _, i := range testIndices {
|
||||
filename := filepath.Join(dirName, fmt.Sprintf("file_%03d.txt", i))
|
||||
expectedContent := []byte(fmt.Sprintf("Content of file %d", i))
|
||||
framework.AssertFileContent(filename, expectedContent)
|
||||
}
|
||||
}
|
||||
|
||||
// testDeepDirectoryNesting tests very deep directory structures
|
||||
func testDeepDirectoryNesting(t *testing.T, framework *FuseTestFramework) {
|
||||
// Create deep nesting (20 levels)
|
||||
depth := 20
|
||||
currentPath := ""
|
||||
|
||||
for i := 0; i < depth; i++ {
|
||||
if i == 0 {
|
||||
currentPath = fmt.Sprintf("level_%02d", i)
|
||||
} else {
|
||||
currentPath = filepath.Join(currentPath, fmt.Sprintf("level_%02d", i))
|
||||
}
|
||||
framework.CreateTestDir(currentPath)
|
||||
}
|
||||
|
||||
// Create a file at the deepest level
|
||||
deepFile := filepath.Join(currentPath, "deep_file.txt")
|
||||
deepContent := []byte("This is very deep!")
|
||||
framework.CreateTestFile(deepFile, deepContent)
|
||||
|
||||
// Verify file exists and has correct content
|
||||
framework.AssertFileContent(deepFile, deepContent)
|
||||
|
||||
// Verify we can navigate the full structure
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), currentPath)
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, info.IsDir())
|
||||
}
|
||||
558
test/fuse_integration/framework_test.go
Normal file
558
test/fuse_integration/framework_test.go
Normal file
@@ -0,0 +1,558 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// FuseTestFramework provides utilities for FUSE integration testing.
|
||||
// It starts a single "weed mini" process (master+volume+filer in one)
|
||||
// and a separate "weed mount" process for the FUSE filesystem.
|
||||
type FuseTestFramework struct {
|
||||
t *testing.T
|
||||
tempDir string
|
||||
mountPoint string
|
||||
dataDir string
|
||||
logDir string
|
||||
miniProcess *managedProcess
|
||||
mountProcess *managedProcess
|
||||
filerAddr string
|
||||
filerPort int
|
||||
weedBinary string
|
||||
isSetup bool
|
||||
}
|
||||
|
||||
// TestConfig holds configuration for FUSE tests
|
||||
type TestConfig struct {
|
||||
Collection string
|
||||
Replication string
|
||||
ChunkSizeMB int
|
||||
CacheSizeMB int
|
||||
NumVolumes int
|
||||
EnableDebug bool
|
||||
// MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on
|
||||
// weed's global flagset; they must precede the subcommand name or the
|
||||
// mount process dies at flag parsing.
|
||||
MountGlobalOptions []string
|
||||
MountOptions []string
|
||||
SkipCleanup bool // for debugging failed tests
|
||||
}
|
||||
|
||||
// DefaultTestConfig returns a default configuration for FUSE tests
|
||||
func DefaultTestConfig() *TestConfig {
|
||||
return &TestConfig{
|
||||
Collection: "",
|
||||
Replication: "000",
|
||||
ChunkSizeMB: 4,
|
||||
CacheSizeMB: 100,
|
||||
NumVolumes: 3,
|
||||
EnableDebug: false,
|
||||
MountOptions: []string{},
|
||||
SkipCleanup: false,
|
||||
}
|
||||
}
|
||||
|
||||
// NewFuseTestFramework creates a new FUSE testing framework.
|
||||
func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework {
|
||||
if config == nil {
|
||||
config = DefaultTestConfig()
|
||||
}
|
||||
|
||||
tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_")
|
||||
require.NoError(t, err)
|
||||
|
||||
filerPort := freePort(t)
|
||||
|
||||
return &FuseTestFramework{
|
||||
t: t,
|
||||
tempDir: tempDir,
|
||||
mountPoint: filepath.Join(tempDir, "mount"),
|
||||
dataDir: filepath.Join(tempDir, "data"),
|
||||
logDir: filepath.Join(tempDir, "logs"),
|
||||
filerPort: filerPort,
|
||||
filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort),
|
||||
weedBinary: findWeedBinary(),
|
||||
isSetup: false,
|
||||
}
|
||||
}
|
||||
|
||||
// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset);
|
||||
// "weed mount" derives the filer gRPC port from the filer HTTP address the same way.
|
||||
const grpcPortOffset = 10000
|
||||
|
||||
// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset)
|
||||
// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port
|
||||
// above it can be transiently grabbed by an outbound connection, forcing mini to
|
||||
// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the
|
||||
// mount then never connects and the test times out. Capping HTTP at 22000 keeps the
|
||||
// gRPC port at or below 32000.
|
||||
func freePort(t *testing.T) int {
|
||||
t.Helper()
|
||||
const (
|
||||
minServicePort = 20000
|
||||
maxServicePort = 22000
|
||||
)
|
||||
|
||||
portCount := maxServicePort - minServicePort + 1
|
||||
start := minServicePort + int(time.Now().UnixNano()%int64(portCount))
|
||||
|
||||
for attempt := 0; attempt < portCount; attempt++ {
|
||||
port := minServicePort + (start-minServicePort+attempt)%portCount
|
||||
if portIsFree(port) && portIsFree(port+grpcPortOffset) {
|
||||
return port
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort)
|
||||
return 0
|
||||
}
|
||||
|
||||
// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1.
|
||||
func portIsFree(port int) bool {
|
||||
l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
l.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
// Setup starts "weed mini" and mounts the FUSE filesystem.
|
||||
func (f *FuseTestFramework) Setup(config *TestConfig) error {
|
||||
if f.isSetup {
|
||||
return fmt.Errorf("framework already setup")
|
||||
}
|
||||
|
||||
dirs := []string{f.mountPoint, f.logDir, f.dataDir}
|
||||
for _, dir := range dirs {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start weed mini (master + volume + filer in one process)
|
||||
if err := f.startMini(config); err != nil {
|
||||
return fmt.Errorf("failed to start weed mini: %v", err)
|
||||
}
|
||||
|
||||
// Wait for filer to be ready (mini starts all services on filerPort)
|
||||
if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil {
|
||||
f.dumpLog("mini")
|
||||
return fmt.Errorf("weed mini not ready: %v", err)
|
||||
}
|
||||
|
||||
// Mount FUSE filesystem
|
||||
if err := f.mountFuse(config); err != nil {
|
||||
return fmt.Errorf("failed to mount FUSE: %v", err)
|
||||
}
|
||||
|
||||
// Wait for mount to be ready
|
||||
if err := f.waitForMount(30 * time.Second); err != nil {
|
||||
f.dumpLog("mount")
|
||||
return fmt.Errorf("FUSE mount not ready: %v", err)
|
||||
}
|
||||
|
||||
f.isSetup = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup stops all processes and removes temporary files.
|
||||
// If the test failed, it dumps logs automatically.
|
||||
func (f *FuseTestFramework) Cleanup() {
|
||||
if f.t.Failed() {
|
||||
f.DumpLogs()
|
||||
}
|
||||
|
||||
// Stop processes in reverse order
|
||||
f.unmountFuse()
|
||||
if f.miniProcess != nil {
|
||||
f.miniProcess.stop()
|
||||
f.miniProcess = nil
|
||||
}
|
||||
|
||||
f.copyLogsForCI()
|
||||
|
||||
if !DefaultTestConfig().SkipCleanup {
|
||||
os.RemoveAll(f.tempDir)
|
||||
}
|
||||
}
|
||||
|
||||
// DumpLogs prints the tail of all SeaweedFS process logs to test output.
|
||||
func (f *FuseTestFramework) DumpLogs() {
|
||||
for _, name := range []string{"mini", "mount"} {
|
||||
f.dumpLog(name)
|
||||
}
|
||||
}
|
||||
|
||||
// GetMountPoint returns the FUSE mount point path
|
||||
func (f *FuseTestFramework) GetMountPoint() string {
|
||||
return f.mountPoint
|
||||
}
|
||||
|
||||
// GetFilerAddr returns the filer address
|
||||
func (f *FuseTestFramework) GetFilerAddr() string {
|
||||
return f.filerAddr
|
||||
}
|
||||
|
||||
// managedProcess is a started weed sub-command whose exit is watched, so a wait
|
||||
// for it to come up ends the moment it dies instead of burning its full timeout.
|
||||
type managedProcess struct {
|
||||
cmd *exec.Cmd
|
||||
done chan struct{}
|
||||
err error // exit error, read only after done is closed
|
||||
}
|
||||
|
||||
// exited returns the exit error once the process is gone, nil while it runs.
|
||||
func (p *managedProcess) exited() error {
|
||||
select {
|
||||
case <-p.done:
|
||||
if p.err != nil {
|
||||
return p.err
|
||||
}
|
||||
return fmt.Errorf("exit status 0")
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// stop asks the process to terminate and waits for it to go away.
|
||||
func (p *managedProcess) stop() {
|
||||
// Signal fails with os.ErrProcessDone when the child is already gone, which
|
||||
// is exactly the case the select below handles.
|
||||
_ = p.cmd.Process.Signal(syscall.SIGTERM)
|
||||
select {
|
||||
case <-p.done:
|
||||
case <-time.After(10 * time.Second):
|
||||
p.cmd.Process.Kill()
|
||||
<-p.done
|
||||
}
|
||||
}
|
||||
|
||||
// startProcess is a helper that starts a weed sub-command with output captured
|
||||
// to a log file in f.logDir.
|
||||
func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) {
|
||||
logFile, err := os.Create(filepath.Join(f.logDir, name+".log"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create log file: %v", err)
|
||||
}
|
||||
cmd := exec.Command(f.weedBinary, args...)
|
||||
cmd.Dir = f.tempDir
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
if err := cmd.Start(); err != nil {
|
||||
logFile.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Close the file handle — the child process inherited it.
|
||||
logFile.Close()
|
||||
|
||||
p := &managedProcess{cmd: cmd, done: make(chan struct{})}
|
||||
go func() {
|
||||
p.err = cmd.Wait()
|
||||
close(p.done)
|
||||
}()
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// dumpLog prints the last lines of a process log file to the test output
|
||||
// for debugging when a service fails to start or a test fails.
|
||||
func (f *FuseTestFramework) dumpLog(name string) {
|
||||
data, err := os.ReadFile(filepath.Join(f.logDir, name+".log"))
|
||||
if err != nil {
|
||||
f.t.Logf("[%s log] (not available: %v)", name, err)
|
||||
return
|
||||
}
|
||||
// Show last 16KB on failure for meaningful context.
|
||||
const maxTail = 16 * 1024
|
||||
if len(data) > maxTail {
|
||||
data = data[len(data)-maxTail:]
|
||||
}
|
||||
f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data))
|
||||
}
|
||||
|
||||
// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/
|
||||
// so the CI workflow can upload them as artifacts.
|
||||
func (f *FuseTestFramework) copyLogsForCI() {
|
||||
ciLogDir := "/tmp/seaweedfs-fuse-logs"
|
||||
os.MkdirAll(ciLogDir, 0755)
|
||||
for _, name := range []string{"mini", "mount"} {
|
||||
src := filepath.Join(f.logDir, name+".log")
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644)
|
||||
}
|
||||
}
|
||||
|
||||
// startMini starts "weed mini" which runs master+volume+filer in one process.
|
||||
func (f *FuseTestFramework) startMini(config *TestConfig) error {
|
||||
var args []string
|
||||
if config.EnableDebug {
|
||||
args = append(args, "-v=4")
|
||||
}
|
||||
args = append(args,
|
||||
"mini",
|
||||
"-dir="+f.dataDir,
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-filer.port="+strconv.Itoa(f.filerPort),
|
||||
"-s3=false",
|
||||
"-webdav=false",
|
||||
"-admin.ui=false",
|
||||
)
|
||||
|
||||
proc, err := f.startProcess("mini", args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.miniProcess = proc
|
||||
return nil
|
||||
}
|
||||
|
||||
// mountFuse mounts the SeaweedFS FUSE filesystem
|
||||
func (f *FuseTestFramework) mountFuse(config *TestConfig) error {
|
||||
args := append([]string{}, config.MountGlobalOptions...)
|
||||
if config.EnableDebug {
|
||||
args = append(args, "-v=4")
|
||||
}
|
||||
args = append(args,
|
||||
"mount",
|
||||
"-filer=127.0.0.1:"+strconv.Itoa(f.filerPort),
|
||||
"-dir="+f.mountPoint,
|
||||
"-filer.path=/",
|
||||
"-dirAutoCreate",
|
||||
"-allowOthers=false",
|
||||
)
|
||||
|
||||
if config.Collection != "" {
|
||||
args = append(args, "-collection="+config.Collection)
|
||||
}
|
||||
if config.Replication != "" {
|
||||
args = append(args, "-replication="+config.Replication)
|
||||
}
|
||||
if config.ChunkSizeMB > 0 {
|
||||
args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB))
|
||||
}
|
||||
if config.CacheSizeMB > 0 {
|
||||
args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB))
|
||||
}
|
||||
|
||||
args = append(args, config.MountOptions...)
|
||||
|
||||
proc, err := f.startProcess("mount", args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.mountProcess = proc
|
||||
return nil
|
||||
}
|
||||
|
||||
// unmountFuse unmounts the FUSE filesystem
|
||||
func (f *FuseTestFramework) unmountFuse() error {
|
||||
if f.mountProcess != nil {
|
||||
f.mountProcess.stop()
|
||||
f.mountProcess = nil
|
||||
}
|
||||
|
||||
// Also try system unmount as backup
|
||||
exec.Command("fusermount3", "-u", f.mountPoint).Run()
|
||||
exec.Command("fusermount", "-u", f.mountPoint).Run()
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForService waits for a service to be available
|
||||
func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
conn, err := net.DialTimeout("tcp", addr, 1*time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
return nil
|
||||
}
|
||||
if exitErr := proc.exited(); exitErr != nil {
|
||||
return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("service at %s not ready within timeout", addr)
|
||||
}
|
||||
|
||||
// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe
|
||||
// alone is not enough: the bare mount point directory passes both before the
|
||||
// mount process finishes starting, letting tests race ahead and write to the
|
||||
// local disk underneath the mount. The mount point's device ID differing from
|
||||
// its parent's confirms a filesystem is actually mounted there.
|
||||
func (f *FuseTestFramework) waitForMount(timeout time.Duration) error {
|
||||
parentDev, err := deviceID(filepath.Dir(f.mountPoint))
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat mount point parent: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if dev, err := deviceID(f.mountPoint); err == nil && dev != parentDev {
|
||||
if _, err := os.ReadDir(f.mountPoint); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// A mount that cannot mount at all (no /dev/fuse, fusermount not setuid)
|
||||
// dies within a second; reporting that beats waiting out the timeout.
|
||||
if exitErr := f.mountProcess.exited(); exitErr != nil {
|
||||
return fmt.Errorf("mount process exited (%v)", exitErr)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("mount point not ready within timeout")
|
||||
}
|
||||
|
||||
// deviceID returns the device ID of the filesystem containing path.
|
||||
func deviceID(path string) (uint64, error) {
|
||||
var st syscall.Stat_t
|
||||
if err := syscall.Stat(path, &st); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return uint64(st.Dev), nil
|
||||
}
|
||||
|
||||
// findWeedBinary locates the weed binary.
|
||||
func findWeedBinary() string {
|
||||
if p, err := exec.LookPath("weed"); err == nil {
|
||||
return p
|
||||
}
|
||||
|
||||
candidates := []string{
|
||||
"../../weed/weed",
|
||||
"./weed",
|
||||
"../weed",
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if info, err := os.Stat(candidate); err == nil && !info.IsDir() {
|
||||
abs, _ := filepath.Abs(candidate)
|
||||
return abs
|
||||
}
|
||||
}
|
||||
|
||||
return "weed"
|
||||
}
|
||||
|
||||
// Helper functions for test assertions
|
||||
|
||||
// AssertFileExists checks if a file exists in the mount point. On failure,
|
||||
// it gathers diagnostic state (retry stat, parent listing, direct open) so
|
||||
// transient FUSE flakes leave enough information to identify the layer that
|
||||
// dropped the entry. The diagnostic block runs only on the failure path.
|
||||
func (f *FuseTestFramework) AssertFileExists(relativePath string) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
_, err := os.Stat(fullPath)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
f.dumpExistenceDiagnostics(relativePath, fullPath, err)
|
||||
require.NoError(f.t, err, "file should exist: %s (see diagnostic logs above)", relativePath)
|
||||
}
|
||||
|
||||
// dumpExistenceDiagnostics is invoked when AssertFileExists fails. It probes
|
||||
// whether the missing entry is transient (retries appear), missing from the
|
||||
// parent directory listing, or missing via a direct open syscall — three
|
||||
// signals that triangulate where the entry was lost (kernel dentry cache vs
|
||||
// mount lookup vs filer).
|
||||
func (f *FuseTestFramework) dumpExistenceDiagnostics(relativePath, fullPath string, initialErr error) {
|
||||
f.t.Logf("AssertFileExists diagnostic dump for %s", relativePath)
|
||||
f.t.Logf(" initial stat: err=%v isNotExist=%v", initialErr, os.IsNotExist(initialErr))
|
||||
|
||||
// Retry stat to detect transient invisibility — if it becomes visible
|
||||
// the issue is a short-lived dentry/cache window; if it stays missing
|
||||
// the entry is genuinely gone from the filer or the local cache.
|
||||
for attempt := 1; attempt <= 5; attempt++ {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
_, err := os.Stat(fullPath)
|
||||
f.t.Logf(" retry #%d after %dms: err=%v", attempt, attempt*100, err)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// List the parent directory: if the file appears here but stat fails,
|
||||
// the failure is in the kernel's per-name dentry cache; if not, the
|
||||
// mount-side LOOKUP itself is dropping the entry.
|
||||
parentDir := filepath.Dir(fullPath)
|
||||
target := filepath.Base(fullPath)
|
||||
if entries, listErr := os.ReadDir(parentDir); listErr != nil {
|
||||
f.t.Logf(" ReadDir(%s) failed: %v", parentDir, listErr)
|
||||
} else {
|
||||
found := false
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
if e.Name() == target {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
f.t.Logf(" ReadDir(%s): %d entries, target %q present=%v", parentDir, len(entries), target, found)
|
||||
if len(names) <= 32 {
|
||||
f.t.Logf(" ReadDir entries: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Direct O_RDONLY open — exercises the same FUSE Lookup+Open path
|
||||
// userspace would, but separately from stat, in case stat-only caching
|
||||
// is interfering.
|
||||
if fd, openErr := os.OpenFile(fullPath, os.O_RDONLY, 0); openErr != nil {
|
||||
f.t.Logf(" Open(%s, O_RDONLY): %v", fullPath, openErr)
|
||||
} else {
|
||||
st, statErr := fd.Stat()
|
||||
fd.Close()
|
||||
size := int64(-1)
|
||||
if statErr == nil && st != nil {
|
||||
size = st.Size()
|
||||
}
|
||||
f.t.Logf(" Open(%s, O_RDONLY): success size=%d statErr=%v", fullPath, size, statErr)
|
||||
}
|
||||
}
|
||||
|
||||
// AssertFileNotExists checks if a file does not exist in the mount point
|
||||
func (f *FuseTestFramework) AssertFileNotExists(relativePath string) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
_, err := os.Stat(fullPath)
|
||||
require.True(f.t, os.IsNotExist(err), "file should not exist: %s", relativePath)
|
||||
}
|
||||
|
||||
// AssertFileContent checks if a file has expected content
|
||||
func (f *FuseTestFramework) AssertFileContent(relativePath string, expectedContent []byte) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
actualContent, err := os.ReadFile(fullPath)
|
||||
require.NoError(f.t, err, "failed to read file: %s", relativePath)
|
||||
require.Equal(f.t, expectedContent, actualContent, "file content mismatch: %s", relativePath)
|
||||
}
|
||||
|
||||
// AssertFileMode checks if a file has expected permissions
|
||||
func (f *FuseTestFramework) AssertFileMode(relativePath string, expectedMode fs.FileMode) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
info, err := os.Stat(fullPath)
|
||||
require.NoError(f.t, err, "failed to stat file: %s", relativePath)
|
||||
require.Equal(f.t, expectedMode, info.Mode(), "file mode mismatch: %s", relativePath)
|
||||
}
|
||||
|
||||
// CreateTestFile creates a test file with specified content
|
||||
func (f *FuseTestFramework) CreateTestFile(relativePath string, content []byte) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
dir := filepath.Dir(fullPath)
|
||||
require.NoError(f.t, os.MkdirAll(dir, 0755), "failed to create directory: %s", dir)
|
||||
require.NoError(f.t, os.WriteFile(fullPath, content, 0644), "failed to create file: %s", relativePath)
|
||||
}
|
||||
|
||||
// CreateTestDir creates a test directory
|
||||
func (f *FuseTestFramework) CreateTestDir(relativePath string) {
|
||||
fullPath := filepath.Join(f.mountPoint, relativePath)
|
||||
require.NoError(f.t, os.MkdirAll(fullPath, 0755), "failed to create directory: %s", relativePath)
|
||||
}
|
||||
630
test/fuse_integration/git_operations_test.go
Normal file
630
test/fuse_integration/git_operations_test.go
Normal file
@@ -0,0 +1,630 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestGitOperations exercises git clone, checkout, and pull on a FUSE mount.
|
||||
//
|
||||
// The test creates a bare repo on the mount (acting as a remote), clones it,
|
||||
// makes commits, pushes, then clones from the mount into an on-mount working
|
||||
// directory. It pushes additional commits, checks out an older revision in the
|
||||
// on-mount clone, and runs git pull to fast-forward with real changes —
|
||||
// verifying file content integrity at each step.
|
||||
func TestGitOperations(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
mountPoint := framework.GetMountPoint()
|
||||
|
||||
// We need a local scratch dir (not on the mount) for the "developer" clone.
|
||||
localDir, err := os.MkdirTemp("", "git_ops_local_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(localDir)
|
||||
|
||||
t.Run("CloneAndPull", func(t *testing.T) {
|
||||
testGitCloneAndPull(t, mountPoint, localDir)
|
||||
})
|
||||
}
|
||||
|
||||
func testGitCloneAndPull(t *testing.T, mountPoint, localDir string) {
|
||||
bareRepo := filepath.Join(mountPoint, "repo.git")
|
||||
localClone := filepath.Join(localDir, "clone")
|
||||
mountClone := filepath.Join(mountPoint, "working")
|
||||
|
||||
// ---- Phase 1: Create bare repo on the mount ----
|
||||
t.Log("Phase 1: create bare repo on mount")
|
||||
gitRun(t, "", "init", "--bare", bareRepo)
|
||||
|
||||
// ---- Phase 2: Clone locally, make initial commits, push ----
|
||||
t.Log("Phase 2: clone locally, commit, push")
|
||||
gitRun(t, "", "clone", bareRepo, localClone)
|
||||
gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
|
||||
gitRun(t, localClone, "config", "user.name", "Test")
|
||||
|
||||
// Commit 1
|
||||
writeFile(t, localClone, "README.md", "hello world\n")
|
||||
mkdirAll(t, localClone, "src")
|
||||
writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v1") }`)
|
||||
gitRun(t, localClone, "add", "-A")
|
||||
gitRun(t, localClone, "commit", "-m", "initial commit")
|
||||
commit1 := gitOutput(t, localClone, "rev-parse", "HEAD")
|
||||
|
||||
// Commit 2: bulk files
|
||||
mkdirAll(t, localClone, "data")
|
||||
for i := 1; i <= 20; i++ {
|
||||
name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
|
||||
writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
|
||||
}
|
||||
gitRun(t, localClone, "add", "-A")
|
||||
gitRun(t, localClone, "commit", "-m", "add data files")
|
||||
commit2 := gitOutput(t, localClone, "rev-parse", "HEAD")
|
||||
|
||||
// Commit 3: modify + new dir
|
||||
writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v2") }`)
|
||||
writeFile(t, localClone, "README.md", "hello world\n# Updated\n")
|
||||
mkdirAll(t, localClone, "docs")
|
||||
writeFile(t, localClone, "docs/guide.md", "documentation\n")
|
||||
gitRun(t, localClone, "add", "-A")
|
||||
gitRun(t, localClone, "commit", "-m", "update src and add docs")
|
||||
commit3 := gitOutput(t, localClone, "rev-parse", "HEAD")
|
||||
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
// The bare repo lives on the FUSE mount and can briefly disappear after
|
||||
// the push completes, and pushed pack objects may not be immediately
|
||||
// consistent on the FUSE layer. Give the mount a chance to settle, then
|
||||
// recover from the local clone if the remote is still missing.
|
||||
if !waitForBareRepoEventually(t, bareRepo, 10*time.Second) {
|
||||
t.Logf("bare repo %s did not stabilise after push; forcing recovery before clone", bareRepo)
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// ---- Phase 3: Clone from mount bare repo into on-mount working dir ----
|
||||
t.Log("Phase 3: clone from mount bare repo to on-mount working dir")
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "# Updated")
|
||||
assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v2")
|
||||
assertFileExists(t, filepath.Join(mountClone, "docs/guide.md"))
|
||||
assertFileExists(t, filepath.Join(mountClone, "data/file-020.txt"))
|
||||
|
||||
head := gitOutput(t, mountClone, "rev-parse", "HEAD")
|
||||
assert.Equal(t, commit3, head, "on-mount clone HEAD should be commit 3")
|
||||
|
||||
dataFiles := countFiles(t, filepath.Join(mountClone, "data"))
|
||||
assert.Equal(t, 20, dataFiles, "data/ should have 20 files")
|
||||
|
||||
// ---- Phase 4: Push more commits from the local clone ----
|
||||
t.Log("Phase 4: push more commits")
|
||||
|
||||
for i := 21; i <= 50; i++ {
|
||||
name := filepath.Join("data", "file-"+leftPad(i, 3)+".txt")
|
||||
writeFile(t, localClone, name, "content-"+strconv.Itoa(i)+"\n")
|
||||
}
|
||||
writeFile(t, localClone, "src/main.go", `package main; import "fmt"; func main() { fmt.Println("v3") }`)
|
||||
gitRun(t, localClone, "add", "-A")
|
||||
gitRun(t, localClone, "commit", "-m", "expand data and update to v3")
|
||||
commit4 := gitOutput(t, localClone, "rev-parse", "HEAD")
|
||||
_ = commit4
|
||||
|
||||
gitRun(t, localClone, "mv", "docs/guide.md", "docs/manual.md")
|
||||
gitRun(t, localClone, "rm", "data/file-001.txt")
|
||||
gitRun(t, localClone, "commit", "-m", "rename guide, remove file-001")
|
||||
commit5 := gitOutput(t, localClone, "rev-parse", "HEAD")
|
||||
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
// ---- Phase 5: Reset to older revision in on-mount clone ----
|
||||
t.Log("Phase 5: reset to older revision on mount clone")
|
||||
resetToCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
|
||||
|
||||
resetHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
|
||||
assert.Equal(t, commit2, resetHead, "should be at commit 2")
|
||||
assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v1")
|
||||
assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))
|
||||
|
||||
// ---- Phase 6: Pull with real changes ----
|
||||
t.Log("Phase 6: pull with real fast-forward changes")
|
||||
|
||||
// After git reset --hard on FUSE (Phase 5), the kernel dcache can
|
||||
// permanently lose the directory entry. Wrap the pull in a recovery
|
||||
// loop that re-clones from the bare repo if the clone has vanished.
|
||||
pullFromCommitWithRecovery(t, bareRepo, localClone, mountClone, commit2)
|
||||
|
||||
newHead := gitOutput(t, mountClone, "rev-parse", "HEAD")
|
||||
assert.Equal(t, commit5, newHead, "HEAD should be commit 5 after pull")
|
||||
|
||||
assertFileContains(t, filepath.Join(mountClone, "src/main.go"), "v3")
|
||||
assertFileExists(t, filepath.Join(mountClone, "docs/manual.md"))
|
||||
assertFileNotExists(t, filepath.Join(mountClone, "docs/guide.md"))
|
||||
assertFileNotExists(t, filepath.Join(mountClone, "data/file-001.txt"))
|
||||
assertFileExists(t, filepath.Join(mountClone, "data/file-050.txt"))
|
||||
|
||||
finalCount := countFiles(t, filepath.Join(mountClone, "data"))
|
||||
assert.Equal(t, 49, finalCount, "data/ should have 49 files after pull")
|
||||
|
||||
// ---- Phase 7: Verify git log and status ----
|
||||
t.Log("Phase 7: verify log and status")
|
||||
logOutput := gitOutput(t, mountClone, "log", "--format=%s")
|
||||
lines := strings.Split(strings.TrimSpace(logOutput), "\n")
|
||||
assert.Equal(t, 5, len(lines), "should have 5 commits in log")
|
||||
|
||||
assert.Contains(t, logOutput, "initial commit")
|
||||
assert.Contains(t, logOutput, "expand data")
|
||||
assert.Contains(t, logOutput, "rename guide")
|
||||
|
||||
status := gitOutput(t, mountClone, "status", "--porcelain")
|
||||
assert.Empty(t, status, "git status should be clean")
|
||||
|
||||
_ = commit1 // used for documentation; not needed in assertions
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func gitRun(t *testing.T, dir string, args ...string) {
|
||||
t.Helper()
|
||||
gitRunWithRetry(t, dir, args...)
|
||||
}
|
||||
|
||||
func gitOutput(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
return gitRunWithRetry(t, dir, args...)
|
||||
}
|
||||
|
||||
// gitRunWithRetry runs a git command with retries to handle transient FUSE
|
||||
// I/O errors on slow CI runners (e.g. "Could not write new index file",
|
||||
// "failed to stat", "unpack-objects failed").
|
||||
func gitRunWithRetry(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
const (
|
||||
maxRetries = 6
|
||||
dirWait = 10 * time.Second
|
||||
)
|
||||
var out []byte
|
||||
var err error
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
if dir != "" && !waitForDirEventually(t, dir, dirWait) {
|
||||
out = []byte("directory missing: " + dir)
|
||||
err = &os.PathError{Op: "stat", Path: dir, Err: os.ErrNotExist}
|
||||
} else {
|
||||
cmd := exec.Command("git", args...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
out, err = cmd.CombinedOutput()
|
||||
}
|
||||
if err == nil {
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
if i < maxRetries-1 {
|
||||
t.Logf("git %s attempt %d failed (retrying): %s", strings.Join(args, " "), i+1, string(out))
|
||||
if dir != "" {
|
||||
refreshDirEntry(t, dir)
|
||||
}
|
||||
if repoPath := extractGitRepoPath(string(out)); repoPath != "" {
|
||||
_ = exec.Command("git", "init", "--bare", repoPath).Run()
|
||||
waitForBareRepoEventually(t, repoPath, 5*time.Second)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
require.NoError(t, err, "git %s failed after %d attempts: %s", strings.Join(args, " "), maxRetries, string(out))
|
||||
return ""
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, base, rel, content string) {
|
||||
t.Helper()
|
||||
p := filepath.Join(base, rel)
|
||||
require.NoError(t, os.WriteFile(p, []byte(content), 0644))
|
||||
}
|
||||
|
||||
func mkdirAll(t *testing.T, base, rel string) {
|
||||
t.Helper()
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(base, rel), 0755))
|
||||
}
|
||||
|
||||
func assertFileExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
_, err := os.Stat(path)
|
||||
require.NoError(t, err, "expected file to exist: %s", path)
|
||||
}
|
||||
|
||||
func assertFileNotExists(t *testing.T, path string) {
|
||||
t.Helper()
|
||||
_, err := os.Stat(path)
|
||||
require.True(t, os.IsNotExist(err), "expected file not to exist: %s", path)
|
||||
}
|
||||
|
||||
func assertFileContains(t *testing.T, path, substr string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
require.NoError(t, err, "failed to read %s", path)
|
||||
assert.Contains(t, string(data), substr, "file %s should contain %q", path, substr)
|
||||
}
|
||||
|
||||
func countFiles(t *testing.T, dir string) int {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err, "failed to read dir %s", dir)
|
||||
count := 0
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func waitForDir(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
if !waitForDirEventually(t, dir, 10*time.Second) {
|
||||
t.Fatalf("directory %s did not appear within 10s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
func waitForDirEventually(t *testing.T, dir string, timeout time.Duration) bool {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(dir); err == nil {
|
||||
return true
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func refreshDirEntry(t *testing.T, dir string) {
|
||||
t.Helper()
|
||||
parent := filepath.Dir(dir)
|
||||
_, _ = os.ReadDir(parent)
|
||||
}
|
||||
|
||||
func waitForBareRepoEventually(t *testing.T, bareRepo string, timeout time.Duration) bool {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
return true
|
||||
}
|
||||
refreshDirEntry(t, bareRepo)
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isBareRepo(bareRepo string) bool {
|
||||
required := []string{
|
||||
filepath.Join(bareRepo, "HEAD"),
|
||||
filepath.Join(bareRepo, "config"),
|
||||
}
|
||||
for _, p := range required {
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isBareRepoAccessible(bareRepo string) bool {
|
||||
if !isBareRepo(bareRepo) {
|
||||
return false
|
||||
}
|
||||
out, err := tryGitCommand("", "--git-dir="+bareRepo, "rev-parse", "--is-bare-repository")
|
||||
return err == nil && out == "true"
|
||||
}
|
||||
|
||||
func ensureMountClone(t *testing.T, bareRepo, mountClone string) {
|
||||
t.Helper()
|
||||
require.NoError(t, tryEnsureMountClone(bareRepo, mountClone))
|
||||
}
|
||||
|
||||
// tryEnsureBareRepo verifies the bare repo on the FUSE mount exists.
|
||||
// If it has vanished, it re-creates it from the local clone.
|
||||
func tryEnsureBareRepo(bareRepo, localClone string) error {
|
||||
if isBareRepoAccessible(bareRepo) {
|
||||
return nil
|
||||
}
|
||||
branch, err := tryGitCommand(localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("detect local branch: %w", err)
|
||||
}
|
||||
os.RemoveAll(bareRepo)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if _, err := tryGitCommand("", "init", "--bare", bareRepo); err != nil {
|
||||
return fmt.Errorf("re-init bare repo: %w", err)
|
||||
}
|
||||
refSpec := fmt.Sprintf("%s:refs/heads/%s", branch, branch)
|
||||
if _, err := tryGitCommand(localClone, "push", "--force", bareRepo, refSpec); err != nil {
|
||||
return fmt.Errorf("re-push branch %s to bare repo: %w", branch, err)
|
||||
}
|
||||
if _, err := tryGitCommand("", "--git-dir="+bareRepo, "symbolic-ref", "HEAD", "refs/heads/"+branch); err != nil {
|
||||
return fmt.Errorf("set bare repo HEAD to %s: %w", branch, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureMountCloneFromBareWithRecovery(t *testing.T, bareRepo, localClone, mountClone string) {
|
||||
t.Helper()
|
||||
const maxAttempts = 3
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if lastErr = tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone); lastErr == nil {
|
||||
return
|
||||
}
|
||||
if attempt == maxAttempts {
|
||||
require.NoError(t, lastErr, "git clone %s %s failed after %d recovery attempts", bareRepo, mountClone, maxAttempts)
|
||||
}
|
||||
t.Logf("clone recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func tryEnsureMountCloneFromBare(bareRepo, localClone, mountClone string) error {
|
||||
if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
|
||||
return fmt.Errorf("ensure bare repo: %w", err)
|
||||
}
|
||||
if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
|
||||
return fmt.Errorf("ensure mount clone: %w", err)
|
||||
}
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
return fmt.Errorf("verify mount clone: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryEnsureMountClone is like ensureMountClone but returns an error instead
|
||||
// of failing the test, for use in recovery loops.
|
||||
func tryEnsureMountClone(bareRepo, mountClone string) error {
|
||||
// Verify .git/HEAD exists — just checking the top-level dir is
|
||||
// insufficient because FUSE may cache a stale directory entry.
|
||||
if _, err := os.Stat(filepath.Join(mountClone, ".git", "HEAD")); err == nil {
|
||||
return nil
|
||||
}
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if _, err := tryGitCommand("", "clone", bareRepo, mountClone); err != nil {
|
||||
return fmt.Errorf("re-clone: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tryGitCommand runs a git command and returns (output, error) without
|
||||
// failing the test, for use in recovery loops.
|
||||
func tryGitCommand(dir string, args ...string) (string, error) {
|
||||
cmd := exec.Command("git", args...)
|
||||
if dir != "" {
|
||||
cmd.Dir = dir
|
||||
}
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return strings.TrimSpace(string(out)), fmt.Errorf("%s: %w", strings.TrimSpace(string(out)), err)
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
// pullFromCommitWithRecovery resets to fromCommit and runs git pull. If the
|
||||
// FUSE mount loses directories (both the bare repo and the working clone can
|
||||
// vanish after heavy git operations), it re-creates them and retries.
|
||||
func pullFromCommitWithRecovery(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) {
|
||||
t.Helper()
|
||||
const maxAttempts = 3
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if lastErr = tryPullFromCommit(t, bareRepo, localClone, cloneDir, fromCommit); lastErr == nil {
|
||||
return
|
||||
}
|
||||
if attempt == maxAttempts {
|
||||
require.NoError(t, lastErr, "git pull failed after %d recovery attempts", maxAttempts)
|
||||
}
|
||||
t.Logf("pull recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(cloneDir)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// resetToCommitWithRecovery resets the on-mount clone to a given commit. If
|
||||
// the FUSE mount loses the directory after a failed reset (the kernel dcache
|
||||
// can drop the entry after "Could not write new index file"), it re-clones
|
||||
// from the bare repo and retries.
|
||||
func resetToCommitWithRecovery(t *testing.T, bareRepo, localClone, mountClone, commit string) {
|
||||
t.Helper()
|
||||
const maxAttempts = 3
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
||||
if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
|
||||
lastErr = err
|
||||
t.Logf("reset recovery attempt %d: ensure bare repo: %v", attempt, err)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if err := tryEnsureMountClone(bareRepo, mountClone); err != nil {
|
||||
lastErr = err
|
||||
t.Logf("reset recovery attempt %d: ensure mount clone: %v", attempt, err)
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
if _, err := tryGitCommand(mountClone, "reset", "--hard", commit); err != nil {
|
||||
lastErr = err
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, err)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !waitForDirEventually(t, mountClone, 5*time.Second) {
|
||||
lastErr = fmt.Errorf("clone dir %s did not recover after reset", mountClone)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// Verify git commands actually work in the directory — the kernel
|
||||
// dcache can transiently show the dir then lose it after reset.
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset verification failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The kernel dcache can drop the FUSE entry moments after a
|
||||
// successful check. Wait briefly and re-verify to confirm the
|
||||
// directory has stabilised before returning to the caller.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, mountClone)
|
||||
if _, err := tryGitCommand(mountClone, "rev-parse", "HEAD"); err != nil {
|
||||
lastErr = fmt.Errorf("post-reset stabilisation check failed: %w", err)
|
||||
if attempt < maxAttempts {
|
||||
t.Logf("reset recovery attempt %d: %v — removing clone for re-create", attempt, lastErr)
|
||||
os.RemoveAll(mountClone)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return
|
||||
}
|
||||
require.NoError(t, lastErr, "git reset --hard %s failed after %d recovery attempts", commit, maxAttempts)
|
||||
}
|
||||
|
||||
func tryPullFromCommit(t *testing.T, bareRepo, localClone, cloneDir, fromCommit string) error {
|
||||
t.Helper()
|
||||
// The bare repo lives on the FUSE mount and can also vanish.
|
||||
// Re-create it from the local clone (which is on local disk).
|
||||
if err := tryEnsureBareRepo(bareRepo, localClone); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tryEnsureMountClone(bareRepo, cloneDir); err != nil {
|
||||
return err
|
||||
}
|
||||
if !waitForDirEventually(t, cloneDir, 10*time.Second) {
|
||||
return fmt.Errorf("clone dir %s did not appear", cloneDir)
|
||||
}
|
||||
|
||||
if _, err := tryGitCommand(cloneDir, "reset", "--hard", fromCommit); err != nil {
|
||||
return fmt.Errorf("reset --hard: %w", err)
|
||||
}
|
||||
if !waitForDirEventually(t, cloneDir, 5*time.Second) {
|
||||
return fmt.Errorf("clone dir %s did not recover after reset", cloneDir)
|
||||
}
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
// Let the dcache stabilise before proceeding.
|
||||
time.Sleep(1 * time.Second)
|
||||
refreshDirEntry(t, cloneDir)
|
||||
|
||||
head, err := tryGitCommand(cloneDir, "rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return fmt.Errorf("rev-parse after reset: %w", err)
|
||||
}
|
||||
if head != fromCommit {
|
||||
return fmt.Errorf("expected HEAD at %s after reset, got %s", fromCommit, head)
|
||||
}
|
||||
|
||||
if _, err := tryGitCommand(cloneDir, "pull"); err != nil {
|
||||
return fmt.Errorf("pull: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var gitRepoPathRe = regexp.MustCompile(`'([^']+)' does not appear to be a git repository`)
|
||||
|
||||
func extractGitRepoPath(output string) string {
|
||||
if match := gitRepoPathRe.FindStringSubmatch(output); len(match) > 1 {
|
||||
return match[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func leftPad(n, width int) string {
|
||||
s := strconv.Itoa(n)
|
||||
for len(s) < width {
|
||||
s = "0" + s
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestTryEnsureBareRepoPreservesCurrentBranch(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "git_bare_recovery_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
bareRepo := filepath.Join(tempDir, "repo.git")
|
||||
localClone := filepath.Join(tempDir, "clone")
|
||||
restoredClone := filepath.Join(tempDir, "restored")
|
||||
|
||||
gitRun(t, "", "init", "--bare", bareRepo)
|
||||
gitRun(t, "", "clone", bareRepo, localClone)
|
||||
gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
|
||||
gitRun(t, localClone, "config", "user.name", "Test")
|
||||
|
||||
writeFile(t, localClone, "README.md", "hello recovery\n")
|
||||
gitRun(t, localClone, "add", "README.md")
|
||||
gitRun(t, localClone, "commit", "-m", "initial commit")
|
||||
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
require.NotEmpty(t, branch)
|
||||
|
||||
require.NoError(t, os.RemoveAll(bareRepo))
|
||||
require.NoError(t, tryEnsureBareRepo(bareRepo, localClone))
|
||||
|
||||
headRef := gitOutput(t, "", "--git-dir="+bareRepo, "symbolic-ref", "--short", "HEAD")
|
||||
assert.Equal(t, branch, headRef, "recovered bare repo should keep the current branch as HEAD")
|
||||
|
||||
gitRun(t, "", "clone", bareRepo, restoredClone)
|
||||
restoredHead := gitOutput(t, restoredClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
assert.Equal(t, branch, restoredHead, "clone from recovered bare repo should check out the current branch")
|
||||
assertFileContains(t, filepath.Join(restoredClone, "README.md"), "hello recovery")
|
||||
}
|
||||
|
||||
func TestEnsureMountCloneFromBareWithRecoveryRecreatesMissingBareRepo(t *testing.T) {
|
||||
tempDir, err := os.MkdirTemp("", "git_mount_clone_recovery_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
bareRepo := filepath.Join(tempDir, "repo.git")
|
||||
localClone := filepath.Join(tempDir, "clone")
|
||||
mountClone := filepath.Join(tempDir, "mount-clone")
|
||||
|
||||
gitRun(t, "", "init", "--bare", bareRepo)
|
||||
gitRun(t, "", "clone", bareRepo, localClone)
|
||||
gitRun(t, localClone, "config", "user.email", "test@seaweedfs.test")
|
||||
gitRun(t, localClone, "config", "user.name", "Test")
|
||||
|
||||
writeFile(t, localClone, "README.md", "hello clone recovery\n")
|
||||
gitRun(t, localClone, "add", "README.md")
|
||||
gitRun(t, localClone, "commit", "-m", "initial commit")
|
||||
|
||||
branch := gitOutput(t, localClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
gitRun(t, localClone, "push", "origin", branch)
|
||||
|
||||
require.NoError(t, os.RemoveAll(bareRepo))
|
||||
|
||||
ensureMountCloneFromBareWithRecovery(t, bareRepo, localClone, mountClone)
|
||||
|
||||
head := gitOutput(t, mountClone, "rev-parse", "--abbrev-ref", "HEAD")
|
||||
assert.Equal(t, branch, head, "recovered clone should stay on the pushed branch")
|
||||
assertFileContains(t, filepath.Join(mountClone, "README.md"), "hello clone recovery")
|
||||
}
|
||||
14
test/fuse_integration/go.mod
Normal file
14
test/fuse_integration/go.mod
Normal file
@@ -0,0 +1,14 @@
|
||||
module seaweedfs-fuse-tests
|
||||
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/stretchr/testify v1.8.4
|
||||
golang.org/x/sys v0.28.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
12
test/fuse_integration/go.sum
Normal file
12
test/fuse_integration/go.sum
Normal file
@@ -0,0 +1,12 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
7
test/fuse_integration/minimal_test.go
Normal file
7
test/fuse_integration/minimal_test.go
Normal file
@@ -0,0 +1,7 @@
|
||||
package fuse_test
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMinimal(t *testing.T) {
|
||||
t.Log("minimal test")
|
||||
}
|
||||
797
test/fuse_integration/posix_file_lock_test.go
Normal file
797
test/fuse_integration/posix_file_lock_test.go
Normal file
@@ -0,0 +1,797 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- fcntl helpers (used by both test process and subprocess) ---
|
||||
|
||||
func fcntlSetLk(f *os.File, typ int16, start, len int64) error {
|
||||
flk := syscall.Flock_t{
|
||||
Type: typ,
|
||||
Whence: 0, // SEEK_SET
|
||||
Start: start,
|
||||
Len: len, // 0 means to EOF
|
||||
}
|
||||
return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flk)
|
||||
}
|
||||
|
||||
func fcntlSetLkw(f *os.File, typ int16, start, len int64) error {
|
||||
flk := syscall.Flock_t{
|
||||
Type: typ,
|
||||
Whence: 0,
|
||||
Start: start,
|
||||
Len: len,
|
||||
}
|
||||
return syscall.FcntlFlock(f.Fd(), syscall.F_SETLKW, &flk)
|
||||
}
|
||||
|
||||
func fcntlGetLk(f *os.File, typ int16, start, len int64) (syscall.Flock_t, error) {
|
||||
flk := syscall.Flock_t{
|
||||
Type: typ,
|
||||
Whence: 0,
|
||||
Start: start,
|
||||
Len: len,
|
||||
}
|
||||
err := syscall.FcntlFlock(f.Fd(), syscall.F_GETLK, &flk)
|
||||
return flk, err
|
||||
}
|
||||
|
||||
// --- Subprocess entry point for fcntl lock tests ---
|
||||
//
|
||||
// POSIX fcntl locks are per-process (the kernel uses the process's files_struct
|
||||
// as lock owner). To test inter-process lock conflicts over FUSE, contending
|
||||
// locks must come from separate processes.
|
||||
//
|
||||
// Actions:
|
||||
// hold — acquire F_SETLK, print "LOCKED\n", wait for stdin close, exit
|
||||
// hold-blocking — print "WAITING\n", acquire F_SETLKW, print "LOCKED\n", wait for stdin close, exit
|
||||
// try — try F_SETLK, print "OK\n" or "EAGAIN\n", exit
|
||||
// getlk — do F_GETLK, print lock type as integer, exit
|
||||
|
||||
func TestFcntlLockHelper(t *testing.T) {
|
||||
action := os.Getenv("LOCK_ACTION")
|
||||
if action == "" {
|
||||
t.Skip("subprocess helper — not invoked directly")
|
||||
}
|
||||
|
||||
filePath := os.Getenv("LOCK_FILE")
|
||||
lockType, err := parseLockType(os.Getenv("LOCK_TYPE"))
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "lock type: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
start, _ := strconv.ParseInt(os.Getenv("LOCK_START"), 10, 64)
|
||||
length, _ := strconv.ParseInt(os.Getenv("LOCK_LEN"), 10, 64)
|
||||
|
||||
// The parent test process created the file on the FUSE mount just before
|
||||
// spawning this subprocess. A fresh lookup from a different process can
|
||||
// briefly return ENOENT while the mount's dentry state propagates, so
|
||||
// retry on ENOENT before giving up.
|
||||
var f *os.File
|
||||
var openErr error
|
||||
for attempt := 0; attempt < 40; attempt++ {
|
||||
f, openErr = os.OpenFile(filePath, os.O_RDWR, 0)
|
||||
if openErr == nil {
|
||||
break
|
||||
}
|
||||
if !errors.Is(openErr, os.ErrNotExist) && !errors.Is(openErr, syscall.ENOENT) {
|
||||
break
|
||||
}
|
||||
_, _ = os.ReadDir(filepath.Dir(filePath))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
if openErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "open: %v\n", openErr)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
switch action {
|
||||
case "hold":
|
||||
if err := fcntlSetLk(f, lockType, start, length); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "setlk: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("LOCKED")
|
||||
os.Stdout.Sync()
|
||||
io.ReadAll(os.Stdin) // block until parent closes pipe
|
||||
|
||||
case "hold-blocking":
|
||||
fmt.Println("WAITING")
|
||||
os.Stdout.Sync()
|
||||
if err := fcntlSetLkw(f, lockType, start, length); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "setlkw: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("LOCKED")
|
||||
os.Stdout.Sync()
|
||||
io.ReadAll(os.Stdin)
|
||||
|
||||
case "try":
|
||||
if err := fcntlSetLk(f, lockType, start, length); err != nil {
|
||||
if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EACCES) {
|
||||
fmt.Println("EAGAIN")
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "unexpected setlk error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("OK")
|
||||
}
|
||||
os.Stdout.Sync()
|
||||
|
||||
case "getlk":
|
||||
flk := syscall.Flock_t{Type: lockType, Whence: 0, Start: start, Len: length}
|
||||
if err := syscall.FcntlFlock(f.Fd(), syscall.F_GETLK, &flk); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "getlk: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(flk.Type)
|
||||
os.Stdout.Sync()
|
||||
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "unknown action: %s\n", action)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func parseLockType(s string) (int16, error) {
|
||||
switch s {
|
||||
case "F_RDLCK":
|
||||
return syscall.F_RDLCK, nil
|
||||
case "F_WRLCK":
|
||||
return syscall.F_WRLCK, nil
|
||||
default:
|
||||
return 0, fmt.Errorf("unknown lock type %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Subprocess coordination helpers ---
|
||||
|
||||
// lockHolder is a subprocess holding an fcntl lock. A background goroutine
|
||||
// reads stdout and signals the waiting/locked channels as the subprocess
|
||||
// progresses through the lock lifecycle.
|
||||
type lockHolder struct {
|
||||
cmd *exec.Cmd
|
||||
stdin io.WriteCloser
|
||||
waiting chan struct{} // closed when subprocess prints "WAITING"
|
||||
locked chan struct{} // closed when subprocess prints "LOCKED"
|
||||
}
|
||||
|
||||
// startLockHolder launches a subprocess that acquires an fcntl lock and holds
|
||||
// it until Release is called.
|
||||
func startLockHolder(t *testing.T, path string, lockType string, start, length int64) *lockHolder {
|
||||
t.Helper()
|
||||
return startLockProcess(t, "hold", path, lockType, start, length)
|
||||
}
|
||||
|
||||
// startBlockingLockHolder launches a subprocess that tries F_SETLKW.
|
||||
// It blocks until the lock is available, then holds it until Release is called.
|
||||
func startBlockingLockHolder(t *testing.T, path string, lockType string, start, length int64) *lockHolder {
|
||||
t.Helper()
|
||||
return startLockProcess(t, "hold-blocking", path, lockType, start, length)
|
||||
}
|
||||
|
||||
func startLockProcess(t *testing.T, action, path, lockType string, start, length int64) *lockHolder {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"LOCK_ACTION="+action,
|
||||
"LOCK_FILE="+path,
|
||||
"LOCK_TYPE="+lockType,
|
||||
fmt.Sprintf("LOCK_START=%d", start),
|
||||
fmt.Sprintf("LOCK_LEN=%d", length),
|
||||
)
|
||||
|
||||
stdin, err := cmd.StdinPipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
require.NoError(t, err)
|
||||
|
||||
cmd.Stderr = os.Stderr
|
||||
require.NoError(t, cmd.Start())
|
||||
|
||||
waiting := make(chan struct{})
|
||||
locked := make(chan struct{})
|
||||
go func() {
|
||||
scanner := bufio.NewScanner(stdoutPipe)
|
||||
for scanner.Scan() {
|
||||
switch strings.TrimSpace(scanner.Text()) {
|
||||
case "WAITING":
|
||||
select {
|
||||
case <-waiting:
|
||||
default:
|
||||
close(waiting)
|
||||
}
|
||||
case "LOCKED":
|
||||
select {
|
||||
case <-locked:
|
||||
default:
|
||||
close(locked)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return &lockHolder{cmd: cmd, stdin: stdin, waiting: waiting, locked: locked}
|
||||
}
|
||||
|
||||
// WaitWaiting waits for the subprocess to print "WAITING" before trying to
|
||||
// assert that a blocking lock request is still blocked.
|
||||
func (h *lockHolder) WaitWaiting(t *testing.T, timeout time.Duration) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-h.waiting:
|
||||
// OK
|
||||
case <-time.After(timeout):
|
||||
t.Fatal("timed out waiting for subprocess to start blocking lock attempt")
|
||||
}
|
||||
}
|
||||
|
||||
// WaitLocked waits for the subprocess to acquire the lock and signal "LOCKED".
|
||||
func (h *lockHolder) WaitLocked(t *testing.T, timeout time.Duration) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-h.locked:
|
||||
// OK
|
||||
case <-time.After(timeout):
|
||||
t.Fatal("timed out waiting for subprocess to acquire lock")
|
||||
}
|
||||
}
|
||||
|
||||
// IsLocked checks whether the subprocess has signaled "LOCKED" within 200ms.
|
||||
func (h *lockHolder) IsLocked() bool {
|
||||
select {
|
||||
case <-h.waiting:
|
||||
// The subprocess has reached the blocking lock attempt.
|
||||
case <-h.locked:
|
||||
return true
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
return false
|
||||
}
|
||||
|
||||
select {
|
||||
case <-h.locked:
|
||||
return true
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Release signals the subprocess to exit and waits for it.
|
||||
func (h *lockHolder) Release(t *testing.T) {
|
||||
t.Helper()
|
||||
h.stdin.Close()
|
||||
_ = h.cmd.Wait()
|
||||
}
|
||||
|
||||
// tryLockInSubprocess tries a non-blocking fcntl lock in a subprocess.
|
||||
// Returns nil if the lock was acquired, syscall.EAGAIN if it was denied.
|
||||
func tryLockInSubprocess(t *testing.T, path, lockType string, start, length int64) error {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"LOCK_ACTION=try",
|
||||
"LOCK_FILE="+path,
|
||||
"LOCK_TYPE="+lockType,
|
||||
fmt.Sprintf("LOCK_START=%d", start),
|
||||
fmt.Sprintf("LOCK_LEN=%d", length),
|
||||
)
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
out, err := cmd.Output()
|
||||
require.NoError(t, err, "subprocess failed to run")
|
||||
|
||||
result := strings.TrimSpace(string(out))
|
||||
if result == "EAGAIN" {
|
||||
return syscall.EAGAIN
|
||||
}
|
||||
require.Equal(t, "OK", result, "unexpected subprocess output")
|
||||
return nil
|
||||
}
|
||||
|
||||
// getLkInSubprocess does F_GETLK in a subprocess and returns the lock type.
|
||||
func getLkInSubprocess(t *testing.T, path, lockType string, start, length int64) int16 {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestFcntlLockHelper$")
|
||||
cmd.Env = append(os.Environ(),
|
||||
"LOCK_ACTION=getlk",
|
||||
"LOCK_FILE="+path,
|
||||
"LOCK_TYPE="+lockType,
|
||||
fmt.Sprintf("LOCK_START=%d", start),
|
||||
fmt.Sprintf("LOCK_LEN=%d", length),
|
||||
)
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
out, err := cmd.Output()
|
||||
require.NoError(t, err, "subprocess failed to run")
|
||||
|
||||
val, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 16)
|
||||
require.NoError(t, err, "failed to parse lock type from subprocess")
|
||||
return int16(val)
|
||||
}
|
||||
|
||||
// --- Test runner ---
|
||||
|
||||
func TestPosixFileLocking(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("FlockExclusive", func(t *testing.T) {
|
||||
testFlockExclusive(t, framework)
|
||||
})
|
||||
t.Run("FlockShared", func(t *testing.T) {
|
||||
testFlockShared(t, framework)
|
||||
})
|
||||
t.Run("FlockUpgrade", func(t *testing.T) {
|
||||
testFlockUpgrade(t, framework)
|
||||
})
|
||||
t.Run("FlockReleaseOnClose", func(t *testing.T) {
|
||||
testFlockReleaseOnClose(t, framework)
|
||||
})
|
||||
t.Run("FcntlWriteLockConflict", func(t *testing.T) {
|
||||
testFcntlWriteLockConflict(t, framework)
|
||||
})
|
||||
t.Run("FcntlReadLocksShared", func(t *testing.T) {
|
||||
testFcntlReadLocksShared(t, framework)
|
||||
})
|
||||
t.Run("FcntlGetLk", func(t *testing.T) {
|
||||
testFcntlGetLk(t, framework)
|
||||
})
|
||||
t.Run("FcntlByteRangePartial", func(t *testing.T) {
|
||||
testFcntlByteRangePartial(t, framework)
|
||||
})
|
||||
t.Run("FcntlSetLkwBlocks", func(t *testing.T) {
|
||||
testFcntlSetLkwBlocks(t, framework)
|
||||
})
|
||||
t.Run("FcntlReleaseOnClose", func(t *testing.T) {
|
||||
testFcntlReleaseOnClose(t, framework)
|
||||
})
|
||||
t.Run("ConcurrentLockContention", func(t *testing.T) {
|
||||
testConcurrentLockContention(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// --- flock() tests (same-process is fine — flock uses per-fd owners) ---
|
||||
|
||||
func testFlockExclusive(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "flock_exclusive.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
f1, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
f2, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
|
||||
|
||||
err = syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
assert.ErrorIs(t, err, syscall.EWOULDBLOCK, "second exclusive flock should fail")
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
|
||||
}
|
||||
|
||||
func testFlockShared(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "flock_shared.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
f1, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
f2, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
|
||||
|
||||
f3, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f3.Close()
|
||||
err = syscall.Flock(int(f3.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
|
||||
assert.ErrorIs(t, err, syscall.EWOULDBLOCK)
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
|
||||
}
|
||||
|
||||
func testFlockUpgrade(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "flock_upgrade.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
f1, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_SH|syscall.LOCK_NB))
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
|
||||
|
||||
f2, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
err = syscall.Flock(int(f2.Fd()), syscall.LOCK_SH|syscall.LOCK_NB)
|
||||
assert.ErrorIs(t, err, syscall.EWOULDBLOCK)
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_UN))
|
||||
}
|
||||
|
||||
func testFlockReleaseOnClose(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "flock_close.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
f1, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, syscall.Flock(int(f1.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
|
||||
f1.Close()
|
||||
|
||||
f2, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_EX|syscall.LOCK_NB))
|
||||
require.NoError(t, syscall.Flock(int(f2.Fd()), syscall.LOCK_UN))
|
||||
}
|
||||
|
||||
// --- fcntl() POSIX lock tests (use subprocesses for inter-process semantics) ---
|
||||
|
||||
// testFcntlWriteLockConflict: test process holds a write lock, subprocess
|
||||
// contender is denied, then succeeds after release.
|
||||
func testFcntlWriteLockConflict(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_write.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
|
||||
|
||||
f1, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
// Test process locks [0, 10).
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 10))
|
||||
|
||||
// Subprocess: overlapping write lock should fail with EAGAIN.
|
||||
err = tryLockInSubprocess(t, path, "F_WRLCK", 5, 10)
|
||||
assert.ErrorIs(t, err, syscall.EAGAIN, "overlapping write lock should fail with EAGAIN")
|
||||
|
||||
// Unlock, then subprocess should succeed.
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 10))
|
||||
|
||||
err = tryLockInSubprocess(t, path, "F_WRLCK", 5, 10)
|
||||
assert.NoError(t, err, "lock should succeed after release")
|
||||
}
|
||||
|
||||
// testFcntlReadLocksShared: test process holds a read lock, subprocess read
|
||||
// lock succeeds (shared), subprocess write lock is denied.
|
||||
func testFcntlReadLocksShared(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_shared.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
|
||||
|
||||
f1, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
// Test process holds a read lock.
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_RDLCK, 0, 10))
|
||||
|
||||
// Subprocess: read lock should succeed (shared).
|
||||
err = tryLockInSubprocess(t, path, "F_RDLCK", 0, 10)
|
||||
assert.NoError(t, err, "shared read lock should succeed")
|
||||
|
||||
// Subprocess: write lock should fail.
|
||||
err = tryLockInSubprocess(t, path, "F_WRLCK", 0, 10)
|
||||
assert.ErrorIs(t, err, syscall.EAGAIN, "write lock should fail with EAGAIN while read lock is held")
|
||||
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 10))
|
||||
}
|
||||
|
||||
// testFcntlGetLk: test process holds a write lock, subprocess queries via
|
||||
// F_GETLK and sees the conflict.
|
||||
func testFcntlGetLk(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_getlk.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("0123456789"), 0644))
|
||||
|
||||
f1, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
// Test process takes a write lock on [0, 5).
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 5))
|
||||
|
||||
// Subprocess: F_GETLK on overlapping range should report conflict.
|
||||
typ := getLkInSubprocess(t, path, "F_WRLCK", 0, 10)
|
||||
assert.NotEqual(t, int16(syscall.F_UNLCK), typ, "should report a conflict")
|
||||
|
||||
// Subprocess: F_GETLK on non-overlapping range should report no conflict.
|
||||
typ = getLkInSubprocess(t, path, "F_WRLCK", 5, 5)
|
||||
assert.Equal(t, int16(syscall.F_UNLCK), typ, "no conflict expected on non-overlapping range")
|
||||
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 5))
|
||||
}
|
||||
|
||||
// testFcntlByteRangePartial: test process and subprocess lock non-overlapping
|
||||
// ranges independently; extending into the other's range is denied.
|
||||
func testFcntlByteRangePartial(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_range.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("0123456789ABCDEF"), 0644))
|
||||
|
||||
f1, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
// Test process locks [0, 8).
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 8))
|
||||
|
||||
// Subprocess locks [8, 16) — non-overlapping, should succeed.
|
||||
holder := startLockHolder(t, path, "F_WRLCK", 8, 8)
|
||||
holder.WaitLocked(t, 5*time.Second)
|
||||
|
||||
// Subprocess: extending into test process's range should fail.
|
||||
err = tryLockInSubprocess(t, path, "F_WRLCK", 4, 8)
|
||||
assert.ErrorIs(t, err, syscall.EAGAIN, "extending into another process's locked range should fail with EAGAIN")
|
||||
|
||||
holder.Release(t)
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 8))
|
||||
}
|
||||
|
||||
// testFcntlSetLkwBlocks: test process holds exclusive lock, subprocess blocks
|
||||
// on F_SETLKW, then unblocks when lock is released.
|
||||
func testFcntlSetLkwBlocks(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_setlkw.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
f1, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f1.Close()
|
||||
|
||||
// Test process takes exclusive lock.
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_WRLCK, 0, 0))
|
||||
|
||||
// Launch subprocess that will block on F_SETLKW.
|
||||
blocker := startBlockingLockHolder(t, path, "F_WRLCK", 0, 0)
|
||||
|
||||
// Verify it has reached the blocking lock attempt but has not acquired it yet.
|
||||
blocker.WaitWaiting(t, 5*time.Second)
|
||||
assert.False(t, blocker.IsLocked(), "F_SETLKW should be blocking")
|
||||
|
||||
// Release our lock to unblock the subprocess.
|
||||
require.NoError(t, fcntlSetLk(f1, syscall.F_UNLCK, 0, 0))
|
||||
|
||||
// Subprocess should now acquire and signal "LOCKED".
|
||||
blocker.WaitLocked(t, 5*time.Second)
|
||||
blocker.Release(t)
|
||||
}
|
||||
|
||||
// testFcntlReleaseOnClose: subprocess holds a lock then exits (fd closes),
|
||||
// test process verifies it can now acquire the lock.
|
||||
func testFcntlReleaseOnClose(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "fcntl_close.txt")
|
||||
require.NoError(t, os.WriteFile(path, []byte("data"), 0644))
|
||||
|
||||
// Subprocess acquires exclusive lock and holds it.
|
||||
holder := startLockHolder(t, path, "F_WRLCK", 0, 0)
|
||||
holder.WaitLocked(t, 5*time.Second)
|
||||
|
||||
// Another subprocess trying to lock should be denied while holder is active.
|
||||
err := tryLockInSubprocess(t, path, "F_WRLCK", 0, 0)
|
||||
assert.ErrorIs(t, err, syscall.EAGAIN, "lock should be held by subprocess")
|
||||
|
||||
// Release subprocess → fd closes → lock released.
|
||||
// The FUSE server may not process the Release immediately, so retry.
|
||||
holder.Release(t)
|
||||
|
||||
f2, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
var lockErr error
|
||||
for attempt := 0; attempt < 50; attempt++ {
|
||||
lockErr = fcntlSetLk(f2, syscall.F_WRLCK, 0, 0)
|
||||
if lockErr == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
require.NoError(t, lockErr, "lock should succeed after subprocess exits")
|
||||
require.NoError(t, fcntlSetLk(f2, syscall.F_UNLCK, 0, 0))
|
||||
}
|
||||
|
||||
// --- Concurrency test (uses flock which has per-fd owners) ---
|
||||
|
||||
// testConcurrentLockContention has multiple goroutines competing for an
|
||||
// exclusive flock, each appending to the file while holding it.
|
||||
// Uses non-blocking flock with retry to avoid exhausting the go-fuse server's
|
||||
// reader goroutines (blocking FUSE SETLKW can starve unlock processing).
|
||||
func testConcurrentLockContention(t *testing.T, fw *FuseTestFramework) {
|
||||
path := filepath.Join(fw.GetMountPoint(), "lock_contention.txt")
|
||||
// Hold a master fd open for the whole subtest. Workers take flock on their
|
||||
// own fds; master stays passive. Keeping this reference pinned avoids the
|
||||
// re-open-after-concurrent-close code path, which has shown an unresolved
|
||||
// mount/metaCache coherence bug where a freshly-opened fd sees ENOENT for
|
||||
// tens of seconds after a burst of concurrent Release calls.
|
||||
master, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
require.NoError(t, err)
|
||||
defer master.Close()
|
||||
|
||||
numWorkers := 8
|
||||
writesPerWorker := 5
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var errs []error
|
||||
var activeHolders atomic.Int32
|
||||
var nextSlot atomic.Int64
|
||||
const recordFormat = "worker %02d write %02d\n"
|
||||
recordSize := len(fmt.Sprintf(recordFormat, 0, 0))
|
||||
|
||||
addError := func(err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
openWithRetry := func(flags int) (*os.File, error) {
|
||||
var openErr error
|
||||
for attempt := 0; attempt < 400; attempt++ {
|
||||
file, err := os.OpenFile(path, flags, 0)
|
||||
if err == nil {
|
||||
return file, nil
|
||||
}
|
||||
openErr = err
|
||||
if !errors.Is(err, os.ErrNotExist) && !errors.Is(err, syscall.ENOENT) {
|
||||
return nil, err
|
||||
}
|
||||
// Refresh parent directory entries in case the mount cache is stale.
|
||||
_, _ = os.ReadDir(filepath.Dir(path))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return nil, openErr
|
||||
}
|
||||
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
wg.Add(1)
|
||||
go func(id int) {
|
||||
defer wg.Done()
|
||||
f, err := openWithRetry(os.O_RDWR)
|
||||
if err != nil {
|
||||
addError(fmt.Errorf("worker %d open: %v", id, err))
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
for j := 0; j < writesPerWorker; j++ {
|
||||
// Non-blocking flock with retry to avoid FUSE server thread starvation.
|
||||
locked := false
|
||||
for attempt := 0; attempt < 3000; attempt++ {
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err == nil {
|
||||
locked = true
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
if !locked {
|
||||
addError(fmt.Errorf("worker %d: failed to acquire flock after retries", id))
|
||||
return
|
||||
}
|
||||
|
||||
if holders := activeHolders.Add(1); holders != 1 {
|
||||
activeHolders.Add(-1)
|
||||
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
addError(fmt.Errorf("worker %d: flock overlap detected with %d holders", id, holders))
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
|
||||
msg := fmt.Sprintf(recordFormat, id, j)
|
||||
if len(msg) != recordSize {
|
||||
activeHolders.Add(-1)
|
||||
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
addError(fmt.Errorf("worker %d: unexpected record size %d", id, len(msg)))
|
||||
return
|
||||
}
|
||||
|
||||
slot := nextSlot.Add(1) - 1
|
||||
offset := slot * int64(recordSize)
|
||||
n, err := f.WriteAt([]byte(msg), offset)
|
||||
if err != nil {
|
||||
activeHolders.Add(-1)
|
||||
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
addError(fmt.Errorf("worker %d write: %v", id, err))
|
||||
return
|
||||
}
|
||||
if n != len(msg) {
|
||||
activeHolders.Add(-1)
|
||||
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
addError(fmt.Errorf("worker %d short write: wrote %d of %d bytes", id, n, len(msg)))
|
||||
return
|
||||
}
|
||||
|
||||
activeHolders.Add(-1)
|
||||
syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
require.Empty(t, errs, "concurrent lock contention errors: %v", errs)
|
||||
|
||||
// Flush via the master fd. fsync on any fd to the file forces the
|
||||
// kernel to send FUSE_FSYNC, which drains workers' writes to the filer.
|
||||
require.NoError(t, master.Sync())
|
||||
|
||||
expectedLines := numWorkers * writesPerWorker
|
||||
expectedBytes := expectedLines * recordSize
|
||||
var data []byte
|
||||
var lastReadErr error
|
||||
ok := assert.Eventually(t, func() bool {
|
||||
if _, err := master.Seek(0, io.SeekStart); err != nil {
|
||||
lastReadErr = fmt.Errorf("seek: %w", err)
|
||||
return false
|
||||
}
|
||||
d, err := io.ReadAll(master)
|
||||
if err != nil {
|
||||
lastReadErr = fmt.Errorf("read: %w", err)
|
||||
return false
|
||||
}
|
||||
data = d
|
||||
lastReadErr = nil
|
||||
return len(data) == expectedBytes
|
||||
}, 15*time.Second, 100*time.Millisecond, "file should eventually contain exactly %d records from all workers", expectedLines)
|
||||
if !ok {
|
||||
// Diagnostics: dump what the mount currently shows for the parent dir
|
||||
// and the target path, so post-mortem analysis has something to go on.
|
||||
parent := filepath.Dir(path)
|
||||
if entries, readErr := os.ReadDir(parent); readErr != nil {
|
||||
t.Logf("diag: ReadDir(%q) failed: %v", parent, readErr)
|
||||
} else {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
t.Logf("diag: ReadDir(%q) returned %d entries: %v", parent, len(names), names)
|
||||
}
|
||||
if fi, statErr := os.Stat(path); statErr != nil {
|
||||
t.Logf("diag: Stat(%q) failed: %v", path, statErr)
|
||||
} else {
|
||||
t.Logf("diag: Stat(%q) ok: size=%d mode=%s", path, fi.Size(), fi.Mode())
|
||||
}
|
||||
if freshFd, openErr := os.OpenFile(path, os.O_RDONLY, 0); openErr != nil {
|
||||
t.Logf("diag: fresh open(%q, O_RDONLY) failed: %v", path, openErr)
|
||||
} else {
|
||||
fresh, readErr := io.ReadAll(freshFd)
|
||||
freshFd.Close()
|
||||
t.Logf("diag: fresh open(%q) read %d bytes (err=%v)", path, len(fresh), readErr)
|
||||
}
|
||||
t.Fatalf("read back incomplete: got %d bytes, want %d (lastReadErr=%v)", len(data), expectedBytes, lastReadErr)
|
||||
}
|
||||
actualLines := bytes.Count(data, []byte("\n"))
|
||||
assert.Equal(t, expectedLines, actualLines,
|
||||
"file should contain exactly %d lines from all workers", expectedLines)
|
||||
}
|
||||
15
test/fuse_integration/simple_test.go
Normal file
15
test/fuse_integration/simple_test.go
Normal file
@@ -0,0 +1,15 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Simple test to verify the package structure is correct
|
||||
func TestPackageStructure(t *testing.T) {
|
||||
t.Log("FUSE integration test package structure is correct")
|
||||
|
||||
// This test verifies that we can compile and run tests
|
||||
// in the fuse_test package without package name conflicts
|
||||
|
||||
t.Log("Package name verification passed")
|
||||
}
|
||||
94
test/fuse_integration/unlinked_open_test.go
Normal file
94
test/fuse_integration/unlinked_open_test.go
Normal file
@@ -0,0 +1,94 @@
|
||||
//go:build linux
|
||||
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// TestMetadataOnUnlinkedOpen covers the POSIX rule that an inode outlives its
|
||||
// last name while a descriptor holds it open: metadata operations through the
|
||||
// descriptor keep working between the removal and the final close, for a file
|
||||
// after unlink and for a directory after rmdir.
|
||||
func TestMetadataOnUnlinkedOpen(t *testing.T) {
|
||||
framework := NewFuseTestFramework(t, DefaultTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(DefaultTestConfig()))
|
||||
|
||||
t.Run("UnlinkedOpenFile", func(t *testing.T) {
|
||||
testUnlinkedOpenFile(t, framework)
|
||||
})
|
||||
|
||||
t.Run("RemovedOpenDirectory", func(t *testing.T) {
|
||||
testRemovedOpenDirectory(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
func testUnlinkedOpenFile(t *testing.T, framework *FuseTestFramework) {
|
||||
path := filepath.Join(framework.GetMountPoint(), "unlinked_open_file")
|
||||
fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR, 0644)
|
||||
require.NoError(t, err)
|
||||
defer unix.Close(fd)
|
||||
_, err = unix.Write(fd, []byte("hello"))
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.Remove(path))
|
||||
|
||||
require.NoError(t, unix.Ftruncate(fd, 2), "ftruncate on an unlinked open file")
|
||||
require.NoError(t, unix.Fchmod(fd, 0600), "fchmod on an unlinked open file")
|
||||
require.NoError(t, unix.Futimes(fd, []unix.Timeval{{Sec: 1234567890}, {Sec: 987654321}}),
|
||||
"futimes on an unlinked open file")
|
||||
|
||||
var st unix.Stat_t
|
||||
require.NoError(t, unix.Fstat(fd, &st), "fstat on an unlinked open file")
|
||||
assert.EqualValues(t, 0, st.Nlink, "nlink after unlink")
|
||||
assert.EqualValues(t, 0600, st.Mode&0777, "mode after fchmod")
|
||||
assert.EqualValues(t, 2, st.Size, "size after ftruncate")
|
||||
assert.EqualValues(t, 1234567890, st.Atim.Sec, "atime after futimes")
|
||||
assert.EqualValues(t, 987654321, st.Mtim.Sec, "mtime after futimes")
|
||||
|
||||
testFdXattrRoundTrip(t, fd)
|
||||
}
|
||||
|
||||
func testRemovedOpenDirectory(t *testing.T, framework *FuseTestFramework) {
|
||||
path := filepath.Join(framework.GetMountPoint(), "removed_open_dir")
|
||||
require.NoError(t, os.Mkdir(path, 0755))
|
||||
fd, err := unix.Open(path, unix.O_RDONLY|unix.O_DIRECTORY, 0)
|
||||
require.NoError(t, err)
|
||||
defer unix.Close(fd)
|
||||
|
||||
require.NoError(t, os.Remove(path))
|
||||
|
||||
require.NoError(t, unix.Fchmod(fd, 0770), "fchmod on a removed open directory")
|
||||
require.NoError(t, unix.Futimes(fd, []unix.Timeval{{Sec: 1234567890}, {Sec: 987654321}}),
|
||||
"futimes on a removed open directory")
|
||||
|
||||
var st unix.Stat_t
|
||||
require.NoError(t, unix.Fstat(fd, &st), "fstat on a removed open directory")
|
||||
assert.EqualValues(t, unix.S_IFDIR, st.Mode&unix.S_IFMT, "type after rmdir")
|
||||
assert.EqualValues(t, 0770, st.Mode&0777, "mode after fchmod")
|
||||
assert.EqualValues(t, 0, st.Nlink, "nlink after rmdir")
|
||||
assert.EqualValues(t, 987654321, st.Mtim.Sec, "mtime after futimes")
|
||||
|
||||
testFdXattrRoundTrip(t, fd)
|
||||
}
|
||||
|
||||
func testFdXattrRoundTrip(t *testing.T, fd int) {
|
||||
require.NoError(t, unix.Fsetxattr(fd, "user.k", []byte("v"), 0), "fsetxattr on a removed open inode")
|
||||
|
||||
buf := make([]byte, 8)
|
||||
n, err := unix.Fgetxattr(fd, "user.k", buf)
|
||||
require.NoError(t, err, "fgetxattr on a removed open inode")
|
||||
assert.Equal(t, "v", string(buf[:n]))
|
||||
|
||||
require.NoError(t, unix.Fremovexattr(fd, "user.k"), "fremovexattr on a removed open inode")
|
||||
_, err = unix.Fgetxattr(fd, "user.k", buf)
|
||||
assert.ErrorIs(t, err, unix.ENODATA, "fgetxattr after fremovexattr")
|
||||
}
|
||||
202
test/fuse_integration/working_demo_test.go
Normal file
202
test/fuse_integration/working_demo_test.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ============================================================================
|
||||
// IMPORTANT: This file contains a STANDALONE demonstration of the FUSE testing
|
||||
// framework that works around Go module conflicts between the main framework
|
||||
// and the SeaweedFS parent module.
|
||||
//
|
||||
// PURPOSE:
|
||||
// - Provides a working demonstration of framework capabilities for CI/CD
|
||||
// - Simulates FUSE operations using local filesystem (not actual FUSE mounts)
|
||||
// - Validates the testing approach and framework design
|
||||
// - Enables CI integration while module conflicts are resolved
|
||||
//
|
||||
// DUPLICATION RATIONALE:
|
||||
// - The full framework (framework.go) has Go module conflicts with parent project
|
||||
// - This standalone version proves the concept works without those conflicts
|
||||
// - Once module issues are resolved, this can be removed or simplified
|
||||
//
|
||||
// TODO: Remove this file once framework.go module conflicts are resolved
|
||||
// ============================================================================
|
||||
|
||||
// DemoTestConfig represents test configuration for the standalone demo
|
||||
// Note: This duplicates TestConfig from framework.go due to module conflicts
|
||||
type DemoTestConfig struct {
|
||||
ChunkSizeMB int
|
||||
Replication string
|
||||
TestTimeout time.Duration
|
||||
}
|
||||
|
||||
// DefaultDemoTestConfig returns default test configuration for demo
|
||||
func DefaultDemoTestConfig() DemoTestConfig {
|
||||
return DemoTestConfig{
|
||||
ChunkSizeMB: 8,
|
||||
Replication: "000",
|
||||
TestTimeout: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
// DemoFuseTestFramework represents the standalone testing framework
|
||||
// Note: This simulates FUSE operations using local filesystem for demonstration
|
||||
type DemoFuseTestFramework struct {
|
||||
t *testing.T
|
||||
config DemoTestConfig
|
||||
mountPath string
|
||||
cleanup []func()
|
||||
}
|
||||
|
||||
// NewDemoFuseTestFramework creates a new demo test framework instance
|
||||
func NewDemoFuseTestFramework(t *testing.T, config DemoTestConfig) *DemoFuseTestFramework {
|
||||
return &DemoFuseTestFramework{
|
||||
t: t,
|
||||
config: config,
|
||||
cleanup: make([]func(), 0),
|
||||
}
|
||||
}
|
||||
|
||||
// CreateTestFile creates a test file with given content
|
||||
func (f *DemoFuseTestFramework) CreateTestFile(filename string, content []byte) {
|
||||
if f.mountPath == "" {
|
||||
f.mountPath = "/tmp/fuse_test_mount"
|
||||
}
|
||||
|
||||
fullPath := filepath.Join(f.mountPath, filename)
|
||||
|
||||
// Ensure directory exists
|
||||
os.MkdirAll(filepath.Dir(fullPath), 0755)
|
||||
|
||||
// Write file (simulated - in real implementation would use FUSE mount)
|
||||
err := os.WriteFile(fullPath, content, 0644)
|
||||
if err != nil {
|
||||
f.t.Fatalf("Failed to create test file %s: %v", filename, err)
|
||||
}
|
||||
}
|
||||
|
||||
// AssertFileExists checks if file exists
|
||||
func (f *DemoFuseTestFramework) AssertFileExists(filename string) {
|
||||
fullPath := filepath.Join(f.mountPath, filename)
|
||||
if _, err := os.Stat(fullPath); os.IsNotExist(err) {
|
||||
f.t.Fatalf("Expected file %s to exist, but it doesn't", filename)
|
||||
}
|
||||
}
|
||||
|
||||
// AssertFileContent checks file content matches expected
|
||||
func (f *DemoFuseTestFramework) AssertFileContent(filename string, expected []byte) {
|
||||
fullPath := filepath.Join(f.mountPath, filename)
|
||||
actual, err := os.ReadFile(fullPath)
|
||||
if err != nil {
|
||||
f.t.Fatalf("Failed to read file %s: %v", filename, err)
|
||||
}
|
||||
|
||||
if string(actual) != string(expected) {
|
||||
f.t.Fatalf("File content mismatch for %s.\nExpected: %q\nActual: %q",
|
||||
filename, string(expected), string(actual))
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup performs test cleanup
|
||||
func (f *DemoFuseTestFramework) Cleanup() {
|
||||
for i := len(f.cleanup) - 1; i >= 0; i-- {
|
||||
f.cleanup[i]()
|
||||
}
|
||||
|
||||
// Clean up test mount directory
|
||||
if f.mountPath != "" {
|
||||
os.RemoveAll(f.mountPath)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFrameworkDemo demonstrates the FUSE testing framework capabilities
|
||||
// NOTE: This is a STANDALONE DEMONSTRATION that simulates FUSE operations
|
||||
// using local filesystem instead of actual FUSE mounts. It exists to prove
|
||||
// the framework concept works while Go module conflicts are resolved.
|
||||
func TestFrameworkDemo(t *testing.T) {
|
||||
t.Log("SeaweedFS FUSE Integration Testing Framework Demo")
|
||||
t.Log("This demo simulates FUSE operations using local filesystem")
|
||||
|
||||
// Initialize demo framework
|
||||
framework := NewDemoFuseTestFramework(t, DefaultDemoTestConfig())
|
||||
defer framework.Cleanup()
|
||||
|
||||
t.Run("ConfigurationValidation", func(t *testing.T) {
|
||||
config := DefaultDemoTestConfig()
|
||||
if config.ChunkSizeMB != 8 {
|
||||
t.Errorf("Expected chunk size 8MB, got %d", config.ChunkSizeMB)
|
||||
}
|
||||
if config.Replication != "000" {
|
||||
t.Errorf("Expected replication '000', got %s", config.Replication)
|
||||
}
|
||||
t.Log("Configuration validation passed")
|
||||
})
|
||||
|
||||
t.Run("BasicFileOperations", func(t *testing.T) {
|
||||
// Test file creation and reading
|
||||
content := []byte("Hello, SeaweedFS FUSE Testing!")
|
||||
filename := "demo_test.txt"
|
||||
|
||||
t.Log("Creating test file...")
|
||||
framework.CreateTestFile(filename, content)
|
||||
|
||||
t.Log("Verifying file exists...")
|
||||
framework.AssertFileExists(filename)
|
||||
|
||||
t.Log("Verifying file content...")
|
||||
framework.AssertFileContent(filename, content)
|
||||
|
||||
t.Log("Basic file operations test passed")
|
||||
})
|
||||
|
||||
t.Run("LargeFileSimulation", func(t *testing.T) {
|
||||
// Simulate large file testing
|
||||
largeContent := make([]byte, 1024*1024) // 1MB
|
||||
for i := range largeContent {
|
||||
largeContent[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
filename := "large_file_demo.dat"
|
||||
|
||||
t.Log("Creating large test file (1MB)...")
|
||||
framework.CreateTestFile(filename, largeContent)
|
||||
|
||||
t.Log("Verifying large file...")
|
||||
framework.AssertFileExists(filename)
|
||||
framework.AssertFileContent(filename, largeContent)
|
||||
|
||||
t.Log("Large file operations test passed")
|
||||
})
|
||||
|
||||
t.Run("ConcurrencySimulation", func(t *testing.T) {
|
||||
// Simulate concurrent operations
|
||||
numFiles := 5
|
||||
|
||||
t.Logf("Creating %d files concurrently...", numFiles)
|
||||
|
||||
for i := 0; i < numFiles; i++ {
|
||||
filename := filepath.Join("concurrent", "file_"+string(rune('A'+i))+".txt")
|
||||
content := []byte("Concurrent file content " + string(rune('A'+i)))
|
||||
|
||||
framework.CreateTestFile(filename, content)
|
||||
framework.AssertFileExists(filename)
|
||||
}
|
||||
|
||||
t.Log("Concurrent operations simulation passed")
|
||||
})
|
||||
|
||||
t.Log("Framework demonstration completed successfully!")
|
||||
t.Log("This DEMO shows the planned FUSE testing capabilities:")
|
||||
t.Log(" • Automated cluster setup/teardown (simulated)")
|
||||
t.Log(" • File operations testing (local filesystem simulation)")
|
||||
t.Log(" • Directory operations testing (planned)")
|
||||
t.Log(" • Large file handling (demonstrated)")
|
||||
t.Log(" • Concurrent operations testing (simulated)")
|
||||
t.Log(" • Error scenario validation (planned)")
|
||||
t.Log(" • Performance validation (planned)")
|
||||
t.Log("Full framework available in framework.go (pending module resolution)")
|
||||
}
|
||||
316
test/fuse_integration/write_buffer_cap_test.go
Normal file
316
test/fuse_integration/write_buffer_cap_test.go
Normal file
@@ -0,0 +1,316 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// mountDebugPort holds the debug/pprof port the test passed to the
|
||||
// mount process via -debug.port. It is set once at TestWriteBufferCap
|
||||
// entry and consulted from the write-timeout paths to fetch the mount
|
||||
// process's goroutine dump (the test's own dumpAllGoroutines only
|
||||
// covers the test process).
|
||||
var mountDebugPort int
|
||||
|
||||
// fetchMountGoroutines pulls a full goroutine dump from the mount
|
||||
// process's pprof endpoint. If the mount debug port isn't configured
|
||||
// or the HTTP call fails, a short explanation is returned instead of
|
||||
// an error — this is diagnostic best-effort, not a test assertion.
|
||||
func fetchMountGoroutines() string {
|
||||
if mountDebugPort == 0 {
|
||||
return "(mount debug port not configured)"
|
||||
}
|
||||
url := fmt.Sprintf("http://127.0.0.1:%d/debug/pprof/goroutine?debug=2", mountDebugPort)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(failed to reach mount pprof at %s: %v)", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("(failed to read mount pprof body: %v)", err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
// dumpAllGoroutines returns a full stack trace of every live goroutine.
|
||||
// Used on write-timeout to give CI actionable diagnosis if the write
|
||||
// buffer cap ever re-regresses into a hang.
|
||||
func dumpAllGoroutines() string {
|
||||
buf := make([]byte, 1<<20)
|
||||
for {
|
||||
n := runtime.Stack(buf, true)
|
||||
if n < len(buf) {
|
||||
return string(buf[:n])
|
||||
}
|
||||
buf = make([]byte, 2*len(buf))
|
||||
}
|
||||
}
|
||||
|
||||
// writeBufferCapConfig returns a TestConfig that exercises the new
|
||||
// -writeBufferSizeMB flag. The cap is set below the aggregate in-flight
|
||||
// write demand of the subtests below, so every new chunk has to pass
|
||||
// through the Reserve/Release backpressure path at least some of the
|
||||
// time. The cap is intentionally NOT minimal — over-tight settings
|
||||
// interact with the per-file writable-chunk limit and the FUSE MaxWrite
|
||||
// batching in ways that starve single-handle writers on slow CI.
|
||||
//
|
||||
// Also enables the mount's pprof debug endpoint so the test can fetch
|
||||
// mount-process goroutine dumps on write-timeout, which is the only
|
||||
// way to actually diagnose a backpressure deadlock (the test process
|
||||
// itself is just blocked in syscall.Write waiting on FUSE).
|
||||
func writeBufferCapConfig(debugPort int) *TestConfig {
|
||||
return &TestConfig{
|
||||
Collection: "",
|
||||
Replication: "000",
|
||||
ChunkSizeMB: 2, // 2 MiB chunks
|
||||
CacheSizeMB: 100, // read cache (unrelated)
|
||||
NumVolumes: 3,
|
||||
EnableDebug: false,
|
||||
// Route glog to stderr so the framework's process log capture
|
||||
// actually contains something — by default weed sends glog to
|
||||
// /tmp/weed.* files which the CI artifact upload step never
|
||||
// sees. Critical for diagnosing upload/saveToStorage errors
|
||||
// on Linux runs.
|
||||
MountGlobalOptions: []string{
|
||||
"-logtostderr=true",
|
||||
"-v=2",
|
||||
},
|
||||
MountOptions: []string{
|
||||
// 16 MiB total write buffer ⇒ up to 8 chunks in flight
|
||||
// across every open file handle on this mount. Large
|
||||
// enough to avoid starving a single handle on a slow
|
||||
// CI runner, small enough that the concurrent test
|
||||
// below still has to drain through it.
|
||||
"-writeBufferSizeMB=16",
|
||||
"-debug=true",
|
||||
fmt.Sprintf("-debug.port=%d", debugPort),
|
||||
},
|
||||
SkipCleanup: false,
|
||||
}
|
||||
}
|
||||
|
||||
// writeWithTimeout wraps os.WriteFile with a hard deadline so a stuck
|
||||
// write fails the test fast instead of consuming the full job budget.
|
||||
// This is belt-and-braces around the 45-minute workflow timeout and
|
||||
// makes write-buffer regressions surface as an actionable failure.
|
||||
func writeWithTimeout(t *testing.T, path string, data []byte, timeout time.Duration) {
|
||||
t.Helper()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- os.WriteFile(path, data, 0644) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
// Dump mount goroutines on any write error, not just
|
||||
// timeout — upload failures that surface via close()
|
||||
// as EIO leave the mount process running but in an
|
||||
// informative state (pending sealed chunks, error
|
||||
// counters, etc).
|
||||
t.Logf("write %s failed (%v) — dumping MOUNT goroutines:\n%s", path, err, fetchMountGoroutines())
|
||||
}
|
||||
require.NoError(t, err, "write %s", path)
|
||||
case <-time.After(timeout):
|
||||
t.Logf("write %s did not finish within %v — dumping TEST goroutines:\n%s", path, timeout, dumpAllGoroutines())
|
||||
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
|
||||
t.Fatalf("write %s timed out — write buffer cap is likely leaking or deadlocking", path)
|
||||
}
|
||||
}
|
||||
|
||||
// runSubtestWithWatchdog runs body on the current (subtest main)
|
||||
// goroutine and starts a watcher goroutine that logs diagnostics and
|
||||
// fails the test if body doesn't return within timeout.
|
||||
//
|
||||
// body must run on the main goroutine because test helpers inside it
|
||||
// (require.NoError, writeWithTimeout's own t.Fatalf on its internal
|
||||
// timeout) need t.Fatal / t.FailNow, which Go's testing docs restrict
|
||||
// to the goroutine running the test function. The watcher goroutine
|
||||
// only calls goroutine-safe t methods (t.Log, t.Logf, t.Errorf) so it
|
||||
// can mark the test failed and dump diagnostics without violating
|
||||
// that contract. If body is stuck past timeout the watcher still
|
||||
// surfaces the wedge (test + mount goroutine dumps + a FAIL mark);
|
||||
// body itself gets unblocked either by its own inner writeWithTimeout
|
||||
// firing t.Fatalf or by Go test's global -timeout.
|
||||
func runSubtestWithWatchdog(t *testing.T, timeout time.Duration, body func(t *testing.T)) {
|
||||
t.Helper()
|
||||
stop := make(chan struct{})
|
||||
defer close(stop)
|
||||
go func() {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-time.After(timeout):
|
||||
t.Logf("subtest exceeded %v watchdog — dumping TEST goroutines:\n%s", timeout, dumpAllGoroutines())
|
||||
t.Logf("dumping MOUNT goroutines:\n%s", fetchMountGoroutines())
|
||||
t.Errorf("subtest exceeded %v watchdog — see goroutine dumps above", timeout)
|
||||
}
|
||||
}()
|
||||
body(t)
|
||||
}
|
||||
|
||||
// TestWriteBufferCap exercises the end-to-end write-buffer cap on a
|
||||
// real FUSE mount. Without the cap, a volume-server stall would let
|
||||
// the swap file grow without bound (issue #8777). With the cap, writers
|
||||
// must serialize through a bounded budget while still producing correct
|
||||
// output — that correctness (and the absence of deadlocks) is what
|
||||
// this test verifies.
|
||||
//
|
||||
// Note: this test deliberately does not assert that Reserve *blocked*
|
||||
// at some observed used-byte peak. The mount runs as a subprocess so
|
||||
// its in-process WriteBufferAccountant state is not reachable from the
|
||||
// test without adding a metrics/RPC surface to the mount binary. The
|
||||
// deterministic peak-vs-cap assertion instead lives in the in-package
|
||||
// unit test TestWriteBufferCap_SharedAcrossPipelines, which drives a
|
||||
// controlled gated uploader and samples Used() throughout the run.
|
||||
func TestWriteBufferCap(t *testing.T) {
|
||||
mountDebugPort = freePort(t)
|
||||
config := writeBufferCapConfig(mountDebugPort)
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
const subtestTimeout = 3 * time.Minute
|
||||
|
||||
t.Run("ConcurrentWritesUnderCap", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testConcurrentWritesUnderCap(t, framework)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("LargeFileUnderCap", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testLargeFileUnderCap(t, framework)
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("DoesNotDeadlockAfterPressure", func(t *testing.T) {
|
||||
runSubtestWithWatchdog(t, subtestTimeout, func(t *testing.T) {
|
||||
testWriteBufferNoDeadlockAfterPressure(t, framework)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// testConcurrentWritesUnderCap opens several files in parallel with
|
||||
// aggregate demand that exceeds the 16 MiB write buffer cap, then
|
||||
// verifies every byte survived the round trip.
|
||||
func testConcurrentWritesUnderCap(t *testing.T, framework *FuseTestFramework) {
|
||||
const (
|
||||
numFiles = 4
|
||||
fileSize = 8 * 1024 * 1024 // 8 MiB per file ⇒ 32 MiB total vs 16 MiB cap
|
||||
)
|
||||
|
||||
dir := "write_buffer_cap_concurrent"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
payloads := make([][]byte, numFiles)
|
||||
for i := range payloads {
|
||||
buf := make([]byte, fileSize)
|
||||
_, err := rand.Read(buf)
|
||||
require.NoError(t, err)
|
||||
payloads[i] = buf
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, numFiles)
|
||||
timedOut := make(chan struct{}, numFiles)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
i := i
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
name := fmt.Sprintf("file_%02d.bin", i)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, name)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- os.WriteFile(path, payloads[i], 0644) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
errs <- fmt.Errorf("writer %d: %w", i, err)
|
||||
}
|
||||
case <-time.After(90 * time.Second):
|
||||
timedOut <- struct{}{}
|
||||
errs <- fmt.Errorf("writer %d: timed out after 90s", i)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
// If any writer timed out, dump every live goroutine so CI shows the
|
||||
// wedge instead of just a walltime.
|
||||
select {
|
||||
case <-timedOut:
|
||||
t.Logf("at least one concurrent writer timed out — dumping goroutines:\n%s", dumpAllGoroutines())
|
||||
default:
|
||||
}
|
||||
for err := range errs {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("wrote %d × %d MiB under 16 MiB cap in %v", numFiles, fileSize/(1024*1024), time.Since(start))
|
||||
|
||||
for i := 0; i < numFiles; i++ {
|
||||
name := fmt.Sprintf("file_%02d.bin", i)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, name)
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err, "read %s", name)
|
||||
require.Equal(t, len(payloads[i]), len(got), "size mismatch for %s", name)
|
||||
if !bytes.Equal(payloads[i], got) {
|
||||
t.Fatalf("content mismatch for %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testLargeFileUnderCap writes a single file whose size exceeds the
|
||||
// 16 MiB cap through a single handle, verifying that the pipeline
|
||||
// drains its own earlier chunks and makes forward progress rather than
|
||||
// self-deadlocking when the global budget is already full of its own
|
||||
// earlier sealed chunks.
|
||||
func testLargeFileUnderCap(t *testing.T, framework *FuseTestFramework) {
|
||||
const fileSize = 20 * 1024 * 1024 // 20 MiB ⇒ 10 chunks vs 8-slot budget
|
||||
|
||||
payload := make([]byte, fileSize)
|
||||
_, err := rand.Read(payload)
|
||||
require.NoError(t, err)
|
||||
|
||||
name := "write_buffer_cap_large.bin"
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
|
||||
start := time.Now()
|
||||
writeWithTimeout(t, path, payload, 90*time.Second)
|
||||
t.Logf("wrote %d MiB through one handle under 16 MiB cap in %v", fileSize/(1024*1024), time.Since(start))
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(payload), len(got))
|
||||
if !bytes.Equal(payload, got) {
|
||||
t.Fatal("content mismatch on large single-handle write")
|
||||
}
|
||||
}
|
||||
|
||||
// testWriteBufferNoDeadlockAfterPressure verifies the mount is still
|
||||
// healthy after being driven against the cap. A budget-slot leak would
|
||||
// eventually cause every new chunk allocation to hang; a quick canary
|
||||
// write catches that as a hard failure.
|
||||
func testWriteBufferNoDeadlockAfterPressure(t *testing.T, framework *FuseTestFramework) {
|
||||
name := "write_buffer_cap_canary.txt"
|
||||
path := filepath.Join(framework.GetMountPoint(), name)
|
||||
content := []byte("write buffer cap canary — mount still healthy")
|
||||
|
||||
writeWithTimeout(t, path, content, 30*time.Second)
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, got)
|
||||
}
|
||||
861
test/fuse_integration/writeback_cache_test.go
Normal file
861
test/fuse_integration/writeback_cache_test.go
Normal file
@@ -0,0 +1,861 @@
|
||||
package fuse_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// writebackConfig returns a TestConfig with writebackCache enabled
|
||||
func writebackConfig() *TestConfig {
|
||||
return &TestConfig{
|
||||
Collection: "",
|
||||
Replication: "000",
|
||||
ChunkSizeMB: 2,
|
||||
CacheSizeMB: 100,
|
||||
NumVolumes: 3,
|
||||
EnableDebug: false,
|
||||
MountOptions: []string{
|
||||
"-writebackCache",
|
||||
},
|
||||
SkipCleanup: false,
|
||||
}
|
||||
}
|
||||
|
||||
// waitForFileContent polls until a file has the expected content or timeout expires.
|
||||
// This is needed because writebackCache defers data upload to background goroutines,
|
||||
// so there is a brief window after close() where the file may not yet be readable.
|
||||
func waitForFileContent(t *testing.T, path string, expected []byte, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastErr error
|
||||
for time.Now().Before(deadline) {
|
||||
actual, err := os.ReadFile(path)
|
||||
if err == nil && bytes.Equal(expected, actual) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
lastErr = fmt.Errorf("content mismatch: got %d bytes, want %d bytes", len(actual), len(expected))
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("file %s did not have expected content within %v: %v", path, timeout, lastErr)
|
||||
}
|
||||
|
||||
// waitForFileSize polls until a file reports the expected size or timeout expires.
|
||||
func waitForFileSize(t *testing.T, path string, expectedSize int64, timeout time.Duration) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
info, err := os.Stat(path)
|
||||
if err == nil && info.Size() == expectedSize {
|
||||
return
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("file %s did not reach expected size %d within %v", path, expectedSize, timeout)
|
||||
}
|
||||
|
||||
// TestWritebackCacheBasicOperations tests fundamental file I/O with writebackCache enabled
|
||||
func TestWritebackCacheBasicOperations(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("WriteAndReadBack", func(t *testing.T) {
|
||||
testWritebackWriteAndReadBack(t, framework)
|
||||
})
|
||||
|
||||
t.Run("MultipleFilesSequential", func(t *testing.T) {
|
||||
testWritebackMultipleFilesSequential(t, framework)
|
||||
})
|
||||
|
||||
t.Run("LargeFile", func(t *testing.T) {
|
||||
testWritebackLargeFile(t, framework)
|
||||
})
|
||||
|
||||
t.Run("EmptyFile", func(t *testing.T) {
|
||||
testWritebackEmptyFile(t, framework)
|
||||
})
|
||||
|
||||
t.Run("OverwriteExistingFile", func(t *testing.T) {
|
||||
testWritebackOverwriteFile(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testWritebackWriteAndReadBack writes a file and verifies it can be read back
|
||||
// after the async flush completes.
|
||||
func testWritebackWriteAndReadBack(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_basic.txt"
|
||||
content := []byte("Hello from writebackCache test!")
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Write file — close() returns immediately with async flush
|
||||
require.NoError(t, os.WriteFile(mountPath, content, 0644))
|
||||
|
||||
// Wait for async flush to complete and verify content
|
||||
waitForFileContent(t, mountPath, content, 30*time.Second)
|
||||
}
|
||||
|
||||
// testWritebackMultipleFilesSequential writes multiple files sequentially
|
||||
// and verifies all are readable after async flushes complete.
|
||||
func testWritebackMultipleFilesSequential(t *testing.T, framework *FuseTestFramework) {
|
||||
dir := "writeback_sequential"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
numFiles := 50
|
||||
files := make(map[string][]byte, numFiles)
|
||||
|
||||
// Write files sequentially — each close() returns immediately
|
||||
for i := 0; i < numFiles; i++ {
|
||||
filename := fmt.Sprintf("file_%03d.txt", i)
|
||||
content := []byte(fmt.Sprintf("Sequential file %d content: %s", i, time.Now().Format(time.RFC3339Nano)))
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, filename)
|
||||
require.NoError(t, os.WriteFile(path, content, 0644))
|
||||
files[filename] = content
|
||||
}
|
||||
|
||||
// Verify all files after a brief wait for async flushes
|
||||
for filename, expectedContent := range files {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, filename)
|
||||
waitForFileContent(t, path, expectedContent, 30*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// testWritebackLargeFile writes a large file (multi-chunk) with writebackCache
|
||||
func testWritebackLargeFile(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_large.bin"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// 8MB file (spans multiple 2MB chunks)
|
||||
content := make([]byte, 8*1024*1024)
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, os.WriteFile(mountPath, content, 0644))
|
||||
|
||||
// Wait for file to be fully flushed
|
||||
waitForFileContent(t, mountPath, content, 60*time.Second)
|
||||
}
|
||||
|
||||
// testWritebackEmptyFile creates an empty file with writebackCache
|
||||
func testWritebackEmptyFile(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_empty.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Create empty file
|
||||
f, err := os.Create(mountPath)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Should exist and be empty
|
||||
info, err := os.Stat(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(0), info.Size())
|
||||
}
|
||||
|
||||
// testWritebackOverwriteFile tests overwriting an existing file
|
||||
func testWritebackOverwriteFile(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_overwrite.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// First write
|
||||
content1 := []byte("First version of the file")
|
||||
require.NoError(t, os.WriteFile(mountPath, content1, 0644))
|
||||
waitForFileContent(t, mountPath, content1, 30*time.Second)
|
||||
|
||||
// Overwrite with different content
|
||||
content2 := []byte("Second version — overwritten content that is longer than the first")
|
||||
require.NoError(t, os.WriteFile(mountPath, content2, 0644))
|
||||
waitForFileContent(t, mountPath, content2, 30*time.Second)
|
||||
}
|
||||
|
||||
// TestWritebackCacheFsync tests that fsync still forces synchronous flush
|
||||
// even when writebackCache is enabled
|
||||
func TestWritebackCacheFsync(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("FsyncForcesFlush", func(t *testing.T) {
|
||||
testFsyncForcesFlush(t, framework)
|
||||
})
|
||||
|
||||
t.Run("FsyncThenRead", func(t *testing.T) {
|
||||
testFsyncThenRead(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testFsyncForcesFlush verifies that calling fsync before close ensures
|
||||
// data is immediately available for reading, bypassing the async path.
|
||||
func testFsyncForcesFlush(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_fsync.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
content := []byte("Data that must be flushed synchronously via fsync")
|
||||
|
||||
// Open, write, fsync, close
|
||||
f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = f.Write(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
// fsync forces synchronous data+metadata flush
|
||||
require.NoError(t, f.Sync())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Data should be immediately available — no wait needed
|
||||
actual, err := os.ReadFile(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, actual)
|
||||
}
|
||||
|
||||
// testFsyncThenRead verifies that after fsync, a freshly opened read
|
||||
// returns the correct data without any delay.
|
||||
func testFsyncThenRead(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_fsync_read.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
content := make([]byte, 64*1024) // 64KB
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write with explicit fsync
|
||||
f, err := os.OpenFile(mountPath, os.O_CREATE|os.O_WRONLY, 0644)
|
||||
require.NoError(t, err)
|
||||
_, err = f.Write(content)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Sync())
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Immediate read should succeed
|
||||
actual, err := os.ReadFile(mountPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, actual)
|
||||
}
|
||||
|
||||
// TestWritebackCacheConcurrentSmallFiles is the primary test for issue #8718:
|
||||
// many small files written concurrently should all be eventually readable.
|
||||
func TestWritebackCacheConcurrentSmallFiles(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("ConcurrentSmallFiles", func(t *testing.T) {
|
||||
testWritebackConcurrentSmallFiles(t, framework)
|
||||
})
|
||||
|
||||
t.Run("ConcurrentSmallFilesMultiDir", func(t *testing.T) {
|
||||
testWritebackConcurrentSmallFilesMultiDir(t, framework)
|
||||
})
|
||||
|
||||
t.Run("RapidCreateCloseSequence", func(t *testing.T) {
|
||||
testWritebackRapidCreateClose(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testWritebackConcurrentSmallFiles simulates the rsync workload from #8718:
|
||||
// multiple workers creating many small files in parallel.
|
||||
func testWritebackConcurrentSmallFiles(t *testing.T, framework *FuseTestFramework) {
|
||||
dir := "writeback_concurrent_small"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
numWorkers := 8
|
||||
filesPerWorker := 20
|
||||
totalFiles := numWorkers * filesPerWorker
|
||||
|
||||
type fileRecord struct {
|
||||
path string
|
||||
content []byte
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var writeErrors []error
|
||||
records := make([]fileRecord, 0, totalFiles)
|
||||
|
||||
// Phase 1: Write files concurrently (simulating rsync workers)
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < numWorkers; w++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
|
||||
for f := 0; f < filesPerWorker; f++ {
|
||||
filename := fmt.Sprintf("w%02d_f%03d.dat", workerID, f)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, filename)
|
||||
|
||||
// Vary sizes: 100B to 100KB
|
||||
size := 100 + (workerID*filesPerWorker+f)*500
|
||||
if size > 100*1024 {
|
||||
size = 100*1024
|
||||
}
|
||||
content := make([]byte, size)
|
||||
if _, err := rand.Read(content); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
records = append(records, fileRecord{path: path, content: content})
|
||||
mu.Unlock()
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Empty(t, writeErrors, "write errors: %v", writeErrors)
|
||||
assert.Equal(t, totalFiles, len(records))
|
||||
|
||||
// Phase 2: Wait for async flushes and verify all files
|
||||
for _, rec := range records {
|
||||
waitForFileContent(t, rec.path, rec.content, 60*time.Second)
|
||||
}
|
||||
|
||||
// Phase 3: Verify directory listing has correct count
|
||||
entries, err := os.ReadDir(filepath.Join(framework.GetMountPoint(), dir))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, totalFiles, len(entries))
|
||||
}
|
||||
|
||||
// testWritebackConcurrentSmallFilesMultiDir tests concurrent writes across
|
||||
// multiple directories — a common pattern for parallel copy tools.
|
||||
func testWritebackConcurrentSmallFilesMultiDir(t *testing.T, framework *FuseTestFramework) {
|
||||
baseDir := "writeback_multidir"
|
||||
framework.CreateTestDir(baseDir)
|
||||
|
||||
numDirs := 4
|
||||
filesPerDir := 25
|
||||
|
||||
type fileRecord struct {
|
||||
path string
|
||||
content []byte
|
||||
}
|
||||
var mu sync.Mutex
|
||||
var records []fileRecord
|
||||
var writeErrors []error
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for d := 0; d < numDirs; d++ {
|
||||
subDir := filepath.Join(baseDir, fmt.Sprintf("dir_%02d", d))
|
||||
framework.CreateTestDir(subDir)
|
||||
|
||||
wg.Add(1)
|
||||
go func(dirID int, dirPath string) {
|
||||
defer wg.Done()
|
||||
|
||||
for f := 0; f < filesPerDir; f++ {
|
||||
filename := fmt.Sprintf("file_%03d.txt", f)
|
||||
path := filepath.Join(framework.GetMountPoint(), dirPath, filename)
|
||||
content := []byte(fmt.Sprintf("dir=%d file=%d data=%s", dirID, f, time.Now().Format(time.RFC3339Nano)))
|
||||
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("dir %d file %d: %v", dirID, f, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
records = append(records, fileRecord{path: path, content: content})
|
||||
mu.Unlock()
|
||||
}
|
||||
}(d, subDir)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Empty(t, writeErrors, "write errors: %v", writeErrors)
|
||||
|
||||
// Verify all files
|
||||
for _, rec := range records {
|
||||
waitForFileContent(t, rec.path, rec.content, 60*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// testWritebackRapidCreateClose rapidly creates and closes files to stress
|
||||
// the async flush goroutine pool.
|
||||
func testWritebackRapidCreateClose(t *testing.T, framework *FuseTestFramework) {
|
||||
dir := "writeback_rapid"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
numFiles := 200
|
||||
type fileRecord struct {
|
||||
path string
|
||||
content []byte
|
||||
}
|
||||
records := make([]fileRecord, numFiles)
|
||||
|
||||
// Rapidly create files without pausing
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("rapid_%04d.bin", i))
|
||||
content := []byte(fmt.Sprintf("rapid-file-%d", i))
|
||||
require.NoError(t, os.WriteFile(path, content, 0644))
|
||||
records[i] = fileRecord{path: path, content: content}
|
||||
}
|
||||
|
||||
// Verify all files eventually appear with correct content
|
||||
for _, rec := range records {
|
||||
waitForFileContent(t, rec.path, rec.content, 60*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWritebackCacheDataIntegrity tests that data integrity is preserved
|
||||
// across various write patterns with writebackCache enabled.
|
||||
func TestWritebackCacheDataIntegrity(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
t.Run("AppendAfterClose", func(t *testing.T) {
|
||||
testWritebackAppendAfterClose(t, framework)
|
||||
})
|
||||
|
||||
t.Run("PartialWrites", func(t *testing.T) {
|
||||
testWritebackPartialWrites(t, framework)
|
||||
})
|
||||
|
||||
t.Run("FileSizeCorrectness", func(t *testing.T) {
|
||||
testWritebackFileSizeCorrectness(t, framework)
|
||||
})
|
||||
|
||||
t.Run("BinaryData", func(t *testing.T) {
|
||||
testWritebackBinaryData(t, framework)
|
||||
})
|
||||
}
|
||||
|
||||
// testWritebackAppendAfterClose writes a file, closes it (triggering async flush),
|
||||
// waits for flush, then reopens and appends more data.
|
||||
func testWritebackAppendAfterClose(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_append.txt"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// First write
|
||||
part1 := []byte("First part of the data.\n")
|
||||
require.NoError(t, os.WriteFile(mountPath, part1, 0644))
|
||||
|
||||
// Wait for first async flush
|
||||
waitForFileContent(t, mountPath, part1, 30*time.Second)
|
||||
|
||||
// Append more data
|
||||
part2 := []byte("Second part appended.\n")
|
||||
f, err := os.OpenFile(mountPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
require.NoError(t, err)
|
||||
_, err = f.Write(part2)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Verify combined content
|
||||
expected := append(part1, part2...)
|
||||
waitForFileContent(t, mountPath, expected, 30*time.Second)
|
||||
}
|
||||
|
||||
// testWritebackPartialWrites tests writing to specific offsets within a file
|
||||
func testWritebackPartialWrites(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_partial.bin"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Create file with initial content
|
||||
initial := bytes.Repeat([]byte("A"), 4096)
|
||||
require.NoError(t, os.WriteFile(mountPath, initial, 0644))
|
||||
waitForFileContent(t, mountPath, initial, 30*time.Second)
|
||||
|
||||
// Open and write at specific offset
|
||||
f, err := os.OpenFile(mountPath, os.O_WRONLY, 0644)
|
||||
require.NoError(t, err)
|
||||
patch := []byte("PATCHED")
|
||||
_, err = f.WriteAt(patch, 100)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, f.Close())
|
||||
|
||||
// Build expected content
|
||||
expected := make([]byte, 4096)
|
||||
copy(expected, initial)
|
||||
copy(expected[100:], patch)
|
||||
|
||||
waitForFileContent(t, mountPath, expected, 30*time.Second)
|
||||
}
|
||||
|
||||
// testWritebackFileSizeCorrectness verifies that file sizes are correct
|
||||
// after async flush completes.
|
||||
func testWritebackFileSizeCorrectness(t *testing.T, framework *FuseTestFramework) {
|
||||
sizes := []int{0, 1, 100, 4096, 65536, 1024 * 1024}
|
||||
|
||||
for _, size := range sizes {
|
||||
filename := fmt.Sprintf("writeback_size_%d.bin", size)
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
content := make([]byte, size)
|
||||
if size > 0 {
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(t, err, "rand.Read failed for size %d", size)
|
||||
}
|
||||
|
||||
require.NoError(t, os.WriteFile(mountPath, content, 0644), "failed to write file of size %d", size)
|
||||
|
||||
if size > 0 {
|
||||
waitForFileSize(t, mountPath, int64(size), 30*time.Second)
|
||||
waitForFileContent(t, mountPath, content, 30*time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// testWritebackBinaryData verifies that arbitrary binary data (including null bytes)
|
||||
// is preserved correctly through the async flush path.
|
||||
func testWritebackBinaryData(t *testing.T, framework *FuseTestFramework) {
|
||||
filename := "writeback_binary.bin"
|
||||
mountPath := filepath.Join(framework.GetMountPoint(), filename)
|
||||
|
||||
// Generate data with all byte values including nulls
|
||||
content := make([]byte, 256*100)
|
||||
for i := range content {
|
||||
content[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
require.NoError(t, os.WriteFile(mountPath, content, 0644))
|
||||
waitForFileContent(t, mountPath, content, 30*time.Second)
|
||||
}
|
||||
|
||||
// TestWritebackCachePerformance measures whether writebackCache actually
|
||||
// improves throughput for small file workloads compared to synchronous flush.
|
||||
func TestWritebackCachePerformance(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping performance test in short mode")
|
||||
}
|
||||
|
||||
numFiles := 200
|
||||
fileSize := 4096 // 4KB files
|
||||
|
||||
// Generate test data upfront
|
||||
testData := make([][]byte, numFiles)
|
||||
for i := range testData {
|
||||
testData[i] = make([]byte, fileSize)
|
||||
rand.Read(testData[i])
|
||||
}
|
||||
|
||||
// Benchmark with writebackCache enabled
|
||||
t.Run("WithWritebackCache", func(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
dir := "perf_writeback"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
|
||||
require.NoError(t, os.WriteFile(path, testData[i], 0644))
|
||||
}
|
||||
writebackDuration := time.Since(start)
|
||||
|
||||
// Wait for all files to be flushed
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
|
||||
waitForFileContent(t, path, testData[i], 60*time.Second)
|
||||
}
|
||||
|
||||
t.Logf("writebackCache: wrote %d files in %v (%.0f files/sec)",
|
||||
numFiles, writebackDuration, float64(numFiles)/writebackDuration.Seconds())
|
||||
})
|
||||
|
||||
// Benchmark without writebackCache (synchronous flush)
|
||||
t.Run("WithoutWritebackCache", func(t *testing.T) {
|
||||
config := DefaultTestConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
dir := "perf_sync"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%04d.bin", i))
|
||||
require.NoError(t, os.WriteFile(path, testData[i], 0644))
|
||||
}
|
||||
syncDuration := time.Since(start)
|
||||
|
||||
t.Logf("synchronous: wrote %d files in %v (%.0f files/sec)",
|
||||
numFiles, syncDuration, float64(numFiles)/syncDuration.Seconds())
|
||||
})
|
||||
}
|
||||
|
||||
// TestWritebackCacheConcurrentMixedOps tests a mix of operations happening
|
||||
// concurrently with writebackCache: creates, reads, overwrites, and deletes.
|
||||
func TestWritebackCacheConcurrentMixedOps(t *testing.T) {
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
dir := "writeback_mixed"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
numFiles := 50
|
||||
var mu sync.Mutex
|
||||
var errors []error
|
||||
var completedWrites int64
|
||||
|
||||
addError := func(err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
// Phase 1: Create initial files and wait for async flushes
|
||||
initialContents := make(map[int][]byte, numFiles)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
|
||||
content := []byte(fmt.Sprintf("initial content %d", i))
|
||||
require.NoError(t, os.WriteFile(path, content, 0644))
|
||||
initialContents[i] = content
|
||||
}
|
||||
|
||||
// Poll until initial files are flushed (instead of fixed sleep)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", i))
|
||||
waitForFileContent(t, path, initialContents[i], 30*time.Second)
|
||||
}
|
||||
|
||||
// Phase 2: Concurrent mixed operations
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Writers: overwrite existing files
|
||||
for i := 0; i < 4; i++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numFiles; j++ {
|
||||
if j%4 != workerID {
|
||||
continue // each worker handles a subset
|
||||
}
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
|
||||
content := []byte(fmt.Sprintf("overwritten by worker %d at %s", workerID, time.Now().Format(time.RFC3339Nano)))
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("writer %d file %d: %v", workerID, j, err))
|
||||
return
|
||||
}
|
||||
atomic.AddInt64(&completedWrites, 1)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// Readers: read files (may see old or new content, but should not error)
|
||||
for i := 0; i < 4; i++ {
|
||||
wg.Add(1)
|
||||
go func(readerID int) {
|
||||
defer wg.Done()
|
||||
for j := 0; j < numFiles; j++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("file_%03d.txt", j))
|
||||
_, err := os.ReadFile(path)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
addError(fmt.Errorf("reader %d file %d: %v", readerID, j, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
// New file creators
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for i := 0; i < 20; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
|
||||
content := []byte(fmt.Sprintf("new file %d", i))
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
addError(fmt.Errorf("creator file %d: %v", i, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
require.Empty(t, errors, "mixed operation errors: %v", errors)
|
||||
assert.True(t, atomic.LoadInt64(&completedWrites) > 0, "should have completed some writes")
|
||||
|
||||
// Verify new files exist after async flushes complete (poll instead of fixed sleep)
|
||||
for i := 0; i < 20; i++ {
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("new_file_%03d.txt", i))
|
||||
expected := []byte(fmt.Sprintf("new file %d", i))
|
||||
waitForFileContent(t, path, expected, 30*time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWritebackCacheStressSmallFiles is a focused stress test for the
|
||||
// async flush path with many small files — the core scenario from #8718.
|
||||
func TestWritebackCacheStressSmallFiles(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping stress test in short mode")
|
||||
}
|
||||
|
||||
config := writebackConfig()
|
||||
framework := NewFuseTestFramework(t, config)
|
||||
defer framework.Cleanup()
|
||||
|
||||
require.NoError(t, framework.Setup(config))
|
||||
|
||||
dir := "writeback_stress"
|
||||
framework.CreateTestDir(dir)
|
||||
|
||||
numWorkers := 16
|
||||
filesPerWorker := 100
|
||||
totalFiles := numWorkers * filesPerWorker
|
||||
|
||||
type fileRecord struct {
|
||||
path string
|
||||
content []byte
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
var writeErrors []error
|
||||
records := make([]fileRecord, 0, totalFiles)
|
||||
|
||||
start := time.Now()
|
||||
|
||||
// Simulate rsync-like workload: many workers each writing small files
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < numWorkers; w++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for f := 0; f < filesPerWorker; f++ {
|
||||
filename := fmt.Sprintf("w%02d/f%04d.dat", workerID, f)
|
||||
path := filepath.Join(framework.GetMountPoint(), dir, filename)
|
||||
|
||||
// Ensure subdirectory exists
|
||||
if f == 0 {
|
||||
subDir := filepath.Join(framework.GetMountPoint(), dir, fmt.Sprintf("w%02d", workerID))
|
||||
if err := os.MkdirAll(subDir, 0755); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("worker %d mkdir: %v", workerID, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Small file: 1KB-10KB (typical for rsync of config/source files)
|
||||
size := 1024 + (f%10)*1024
|
||||
content := make([]byte, size)
|
||||
if _, err := rand.Read(content); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d rand: %v", workerID, f, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, content, 0644); err != nil {
|
||||
mu.Lock()
|
||||
writeErrors = append(writeErrors, fmt.Errorf("worker %d file %d: %v", workerID, f, err))
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
records = append(records, fileRecord{path: path, content: content})
|
||||
mu.Unlock()
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
writeDuration := time.Since(start)
|
||||
// Stream to stderr so the line is visible even if the test hangs and the
|
||||
// 45m suite alarm panics before t.Logf flushes its buffer.
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: wrote %d files in %v (%.0f files/sec)\n",
|
||||
totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
|
||||
t.Logf("wrote %d files in %v (%.0f files/sec)",
|
||||
totalFiles, writeDuration, float64(totalFiles)/writeDuration.Seconds())
|
||||
|
||||
require.Empty(t, writeErrors, "write errors: %v", writeErrors)
|
||||
assert.Equal(t, totalFiles, len(records))
|
||||
|
||||
// Verify all files are eventually readable with correct content.
|
||||
// A whole-phase budget bounds total verify time so a stuck mount fails
|
||||
// with diagnostics instead of being swallowed by the 45m suite alarm.
|
||||
const verifyPhaseBudget = 10 * time.Minute
|
||||
const perFileDeadline = 120 * time.Second
|
||||
verifyStart := time.Now()
|
||||
verifyPhaseDeadline := verifyStart.Add(verifyPhaseBudget)
|
||||
var verifyErrors []error
|
||||
for i, rec := range records {
|
||||
if time.Now().After(verifyPhaseDeadline) {
|
||||
buf := make([]byte, 1<<20)
|
||||
n := runtime.Stack(buf, true)
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verify-phase budget %v exceeded at file %d/%d (%s); goroutine dump:\n%s\n",
|
||||
verifyPhaseBudget, i, len(records), rec.path, buf[:n])
|
||||
t.Fatalf("verify phase exceeded %v at file %d/%d (%s); accumulated errors: %v",
|
||||
verifyPhaseBudget, i, len(records), rec.path, verifyErrors)
|
||||
}
|
||||
fileDeadline := time.Now().Add(perFileDeadline)
|
||||
var lastErr error
|
||||
retries := 0
|
||||
for time.Now().Before(fileDeadline) {
|
||||
actual, err := os.ReadFile(rec.path)
|
||||
if err == nil && bytes.Equal(rec.content, actual) {
|
||||
lastErr = nil
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
} else {
|
||||
lastErr = fmt.Errorf("content mismatch for %s: got %d bytes, want %d", rec.path, len(actual), len(rec.content))
|
||||
}
|
||||
retries++
|
||||
// Surface the in-flight error every ~5s of polling so a stuck file
|
||||
// is visible in real time, not only after the test exits.
|
||||
if retries%10 == 0 {
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s still failing after %d retries: %v\n",
|
||||
i, len(records), rec.path, retries, lastErr)
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if lastErr != nil {
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: file %d/%d %s gave up after %v: %v\n",
|
||||
i, len(records), rec.path, perFileDeadline, lastErr)
|
||||
verifyErrors = append(verifyErrors, lastErr)
|
||||
}
|
||||
// Periodic progress so the test isn't silent during the verify phase.
|
||||
if (i+1)%100 == 0 {
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: verified %d/%d (elapsed %v, %d errors)\n",
|
||||
i+1, len(records), time.Since(verifyStart), len(verifyErrors))
|
||||
}
|
||||
}
|
||||
require.Empty(t, verifyErrors, "verification errors after stress test: %v", verifyErrors)
|
||||
|
||||
fmt.Fprintf(os.Stderr, "TestWritebackCacheStressSmallFiles: all %d files verified in %v\n",
|
||||
totalFiles, time.Since(verifyStart))
|
||||
t.Logf("all %d files verified successfully", totalFiles)
|
||||
}
|
||||
Reference in New Issue
Block a user