snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/parquet/.gitignore

@@ -0,0 +1,40 @@
+# Python virtual environment
+venv/
+.venv/
+env/
+ENV/
+
+# Python cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+
+# Test artifacts
+*.log
+test_run.log
+weed-test.log
+
+# SeaweedFS data directories
+filerldb2/
+idx/
+dat/
+*.idx
+*.dat
+
+# Temporary test files
+.pytest_cache/
+.coverage
+htmlcov/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db

test/s3/parquet/Makefile

@@ -0,0 +1,390 @@
+# Makefile for S3 Parquet Integration Tests
+# This Makefile provides targets for running comprehensive S3 Parquet tests with PyArrow
+
+# Default values
+SEAWEEDFS_BINARY ?= weed
+S3_PORT ?= 8333
+TEST_TIMEOUT ?= 15m
+ACCESS_KEY ?= some_access_key1
+SECRET_KEY ?= some_secret_key1
+BUCKET_NAME ?= test-parquet-bucket
+ENABLE_SSE_S3 ?= false
+
+# Python configuration
+PYTHON ?= python3
+VENV_DIR ?= .venv
+PYTHON_TEST_SCRIPT ?= s3_parquet_test.py
+
+# Test directory
+TEST_DIR := $(shell pwd)
+SEAWEEDFS_ROOT := $(shell cd ../../../ && pwd)
+
+# Colors for output
+RED := \033[0;31m
+GREEN := \033[0;32m
+YELLOW := \033[1;33m
+NC := \033[0m # No Color
+
+.PHONY: all build-weed check-binary check-python ci-test clean debug-logs debug-status help manual-start manual-stop setup-python start-seaweedfs start-seaweedfs-ci stop-seaweedfs stop-seaweedfs-safe test test-cross-fs test-cross-fs-with-server test-implicit-dir test-implicit-dir-with-server test-native-s3 test-native-s3-with-server test-native-s3-with-sse test-quick test-sse-s3-compat test-with-server
+
+all: test
+
+# Build SeaweedFS binary (GitHub Actions compatible)
+build-weed:
+	@echo "Building SeaweedFS binary..."
+	@cd $(SEAWEEDFS_ROOT)/weed && go install -buildvcs=false
+	@echo "✅ SeaweedFS binary built successfully"
+
+help:
+	@echo "SeaweedFS S3 Parquet Integration Tests"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  test              - Run full S3 Parquet integration tests (small and large files)"
+	@echo "  test-with-server  - Run full tests with automatic server management (CI compatible)"
+	@echo "  test-quick        - Run quick tests with small files only (sets TEST_QUICK=1)"
+	@echo "  test-implicit-dir - Test implicit directory fix for s3fs compatibility"
+	@echo "  test-implicit-dir-with-server - Test implicit directory fix with server management"
+	@echo "  test-native-s3    - Test PyArrow's native S3 filesystem (assumes server running)"
+	@echo "  test-native-s3-with-server - Test PyArrow's native S3 filesystem with server management"
+	@echo "  test-native-s3-with-sse - Test PyArrow's native S3 with SSE-S3 encryption enabled"
+	@echo "  test-cross-fs     - Test cross-filesystem compatibility (s3fs ↔ PyArrow native)"
+	@echo "  test-cross-fs-with-server - Test cross-filesystem compatibility with server management"
+	@echo "  test-sse-s3-compat - Comprehensive SSE-S3 compatibility test (multipart uploads)"
+	@echo "  setup-python      - Setup Python virtual environment and install dependencies"
+	@echo "  check-python      - Check if Python and required packages are available"
+	@echo "  start-seaweedfs   - Start SeaweedFS server for testing"
+	@echo "  start-seaweedfs-ci - Start SeaweedFS server (CI-safe version)"
+	@echo "  stop-seaweedfs    - Stop SeaweedFS server"
+	@echo "  stop-seaweedfs-safe - Stop SeaweedFS server (CI-safe version)"
+	@echo "  clean             - Clean up test artifacts"
+	@echo "  check-binary      - Check if SeaweedFS binary exists"
+	@echo "  build-weed        - Build SeaweedFS binary"
+	@echo ""
+	@echo "Configuration:"
+	@echo "  SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
+	@echo "  S3_PORT=$(S3_PORT)"
+	@echo "  BUCKET_NAME=$(BUCKET_NAME)"
+	@echo "  ENABLE_SSE_S3=$(ENABLE_SSE_S3)"
+	@echo "  PYTHON=$(PYTHON)"
+
+check-binary:
+	@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1; then \
+		echo "$(RED)Error: SeaweedFS binary '$(SEAWEEDFS_BINARY)' not found in PATH$(NC)"; \
+		echo "Please build SeaweedFS first by running 'make' in the root directory"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)SeaweedFS binary found: $$(which $(SEAWEEDFS_BINARY))$(NC)"
+
+check-python:
+	@if ! command -v $(PYTHON) > /dev/null 2>&1; then \
+		echo "$(RED)Error: Python '$(PYTHON)' not found$(NC)"; \
+		echo "Please install Python 3.8 or later"; \
+		exit 1; \
+	fi
+	@echo "$(GREEN)Python found: $$(which $(PYTHON)) ($$($(PYTHON) --version))$(NC)"
+
+setup-python: check-python
+	@echo "$(YELLOW)Setting up Python virtual environment...$(NC)"
+	@if [ ! -d "$(VENV_DIR)" ]; then \
+		$(PYTHON) -m venv $(VENV_DIR); \
+		echo "$(GREEN)Virtual environment created$(NC)"; \
+	fi
+	@echo "$(YELLOW)Installing Python dependencies...$(NC)"
+	@$(VENV_DIR)/bin/pip install --upgrade pip > /dev/null
+	@$(VENV_DIR)/bin/pip install -r requirements.txt
+	@echo "$(GREEN)Python dependencies installed successfully$(NC)"
+
+start-seaweedfs-ci: check-binary
+	@echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)"
+	
+	# Clean up any existing processes first (CI-safe) - aggressive cleanup
+	@echo "Aggressively cleaning up any existing processes on S3 port $(S3_PORT) and master port 9333..."
+	@if command -v lsof >/dev/null 2>&1; then \
+		for attempt in 1 2 3; do \
+			lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \
+			lsof -ti :9333 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \
+			sleep 1; \
+		done; \
+	fi
+	@sleep 3
+	
+	# Create necessary directories
+	@mkdir -p /tmp/seaweedfs-test-parquet
+	
+	# Clean up any old server logs
+	@rm -f /tmp/seaweedfs-parquet-*.log || true
+	
+	# Start weed mini with embedded S3
+	@echo "Starting weed mini with embedded S3..."
+	@if [ "$(ENABLE_SSE_S3)" = "true" ]; then \
+		echo "  SSE-S3 encryption: ENABLED"; \
+		printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"buckets":[{"name":"$(BUCKET_NAME)","encryption":{"sseS3":{"enabled":true}}}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \
+	else \
+		echo "  SSE-S3 encryption: DISABLED"; \
+		printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \
+	fi
+	@$(SEAWEEDFS_BINARY) mini \
+		-dir=/tmp/seaweedfs-test-parquet \
+		-ip.bind=0.0.0.0 \
+		-s3.port=$(S3_PORT) \
+		-s3.config=/tmp/seaweedfs-parquet-s3.json \
+		> /tmp/seaweedfs-parquet-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
+	@echo "Waiting for S3 service to be fully ready (max 90 seconds)..."
+	@bash -c 'for i in $$(seq 1 90); do \
+		if curl -s -H "Authorization: AWS4-HMAC-SHA256 Credential=$(ACCESS_KEY)" http://localhost:$(S3_PORT)/ > /dev/null 2>&1; then \
+			echo "✅ S3 service is ready"; \
+			sleep 2; \
+			exit 0; \
+		fi; \
+		sleep 1; \
+	done; \
+	echo "ERROR S3 service failed to start within 90 seconds"; \
+	echo "=== Server log output ==="; \
+	cat /tmp/seaweedfs-parquet-mini.log 2>/dev/null || echo "No startup log available"; \
+	exit 1'
+
+start-seaweedfs: check-binary
+	@echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)"
+	@# Use port-based cleanup for consistency and safety
+	@echo "Cleaning up any existing processes..."
+	@lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true
+	@sleep 2
+	@$(MAKE) start-seaweedfs-ci
+
+stop-seaweedfs:
+	@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
+	@# Use port-based cleanup for consistency and safety
+	@if [ -f /tmp/weed-mini.pid ]; then \
+		echo "Stopping weed mini..."; \
+		kill $$(cat /tmp/weed-mini.pid) || true; \
+		rm -f /tmp/weed-mini.pid; \
+	fi
+	@lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true
+	@sleep 2
+	@echo "$(GREEN)SeaweedFS server stopped$(NC)"
+
+# CI-safe server stop that's more conservative
+stop-seaweedfs-safe:
+	@echo "$(YELLOW)Safely stopping SeaweedFS server...$(NC)"
+	@# Use port-based cleanup which is safer in CI
+	@if command -v lsof >/dev/null 2>&1; then \
+		echo "Using lsof for port-based cleanup..."; \
+		lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
+	else \
+		echo "lsof not available, using netstat approach..."; \
+		netstat -tlnp 2>/dev/null | grep :$(S3_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
+	fi
+	@sleep 2
+	@echo "$(GREEN)SeaweedFS server safely stopped$(NC)"
+
+clean:
+	@echo "$(YELLOW)Cleaning up Parquet test artifacts...$(NC)"
+	@rm -rf /tmp/seaweedfs-test-parquet-*
+	@rm -f /tmp/seaweedfs-parquet-*.log
+	@rm -f /tmp/seaweedfs-parquet-s3.json
+	@rm -f s3_parquet_test_errors_*.log
+	@rm -rf $(VENV_DIR)
+	@echo "$(GREEN)Parquet test cleanup completed$(NC)"
+
+# Test with automatic server management (GitHub Actions compatible)
+test-with-server: build-weed setup-python
+	@echo "🚀 Starting Parquet integration tests with automated server management..."
+	@echo "Starting SeaweedFS cluster..."
+	@if $(MAKE) start-seaweedfs-ci > weed-test.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully"; \
+		echo "Running Parquet integration tests..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=$(BUCKET_NAME) \
+		$(VENV_DIR)/bin/$(PYTHON) $(PYTHON_TEST_SCRIPT) || exit 1; \
+		echo "✅ All tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test.log 2>/dev/null || echo "No startup log available"; \
+		echo "=== System information ==="; \
+		ps aux | grep -E "weed|make" | grep -v grep || echo "No relevant processes found"; \
+		exit 1; \
+	fi
+
+# Run tests assuming SeaweedFS is already running
+test: setup-python
+	@echo "$(YELLOW)Running Parquet integration tests...$(NC)"
+	@echo "$(YELLOW)Assuming SeaweedFS is already running on localhost:$(S3_PORT)$(NC)"
+	@S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+	 S3_ACCESS_KEY=$(ACCESS_KEY) \
+	 S3_SECRET_KEY=$(SECRET_KEY) \
+	 BUCKET_NAME=$(BUCKET_NAME) \
+	 $(VENV_DIR)/bin/$(PYTHON) $(PYTHON_TEST_SCRIPT)
+
+# Run quick tests with small files only
+test-quick: setup-python
+	@echo "$(YELLOW)Running quick Parquet tests (small files only)...$(NC)"
+	@echo "$(YELLOW)Assuming SeaweedFS is already running on localhost:$(S3_PORT)$(NC)"
+	@S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+	 S3_ACCESS_KEY=$(ACCESS_KEY) \
+	 S3_SECRET_KEY=$(SECRET_KEY) \
+	 BUCKET_NAME=$(BUCKET_NAME) \
+	 TEST_QUICK=1 \
+	 $(VENV_DIR)/bin/$(PYTHON) $(PYTHON_TEST_SCRIPT)
+
+# Test implicit directory fix for s3fs compatibility
+test-implicit-dir: setup-python
+	@echo "$(YELLOW)Running implicit directory fix tests...$(NC)"
+	@echo "$(YELLOW)Assuming SeaweedFS is already running on localhost:$(S3_PORT)$(NC)"
+	@S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+	 S3_ACCESS_KEY=$(ACCESS_KEY) \
+	 S3_SECRET_KEY=$(SECRET_KEY) \
+	 BUCKET_NAME=test-implicit-dir \
+	 $(VENV_DIR)/bin/$(PYTHON) test_implicit_directory_fix.py
+
+# Test implicit directory fix with automatic server management
+test-implicit-dir-with-server: build-weed setup-python
+	@echo "🚀 Starting implicit directory fix tests with automated server management..."
+	@echo "Starting SeaweedFS cluster..."
+	@if $(MAKE) start-seaweedfs-ci > weed-test.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully"; \
+		echo "Running implicit directory fix tests..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=test-implicit-dir \
+		$(VENV_DIR)/bin/$(PYTHON) test_implicit_directory_fix.py || exit 1; \
+		echo "✅ All tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test.log 2>/dev/null || echo "No startup log available"; \
+		exit 1; \
+	fi
+
+# Debug targets
+debug-logs:
+	@echo "$(YELLOW)=== Mini Log ===$(NC)"
+	@tail -n 50 /tmp/seaweedfs-parquet-mini.log || echo "No mini log found"
+
+debug-status:
+	@echo "$(YELLOW)=== Process Status ===$(NC)"
+	@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
+	@echo "$(YELLOW)=== Port Status ===$(NC)"
+	@netstat -an | grep -E "($(S3_PORT))" || echo "No ports in use"
+
+# Manual test targets for development
+manual-start: start-seaweedfs
+	@echo "$(GREEN)SeaweedFS with S3 is now running for manual testing$(NC)"
+	@echo "You can now run Parquet tests manually"
+	@echo "Run 'make manual-stop' when finished"
+
+manual-stop: stop-seaweedfs clean
+
+# Test PyArrow's native S3 filesystem
+test-native-s3: setup-python
+	@echo "$(YELLOW)Running PyArrow native S3 filesystem tests...$(NC)"
+	@echo "$(YELLOW)Assuming SeaweedFS is already running on localhost:$(S3_PORT)$(NC)"
+	@S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+	 S3_ACCESS_KEY=$(ACCESS_KEY) \
+	 S3_SECRET_KEY=$(SECRET_KEY) \
+	 BUCKET_NAME=$(BUCKET_NAME) \
+	 $(VENV_DIR)/bin/$(PYTHON) test_pyarrow_native_s3.py
+
+# Test PyArrow's native S3 filesystem with automatic server management
+test-native-s3-with-server: build-weed setup-python
+	@echo "🚀 Starting PyArrow native S3 filesystem tests with automated server management..."
+	@echo "Starting SeaweedFS cluster..."
+	@if $(MAKE) start-seaweedfs-ci > weed-test.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully"; \
+		echo "Running PyArrow native S3 filesystem tests..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=$(BUCKET_NAME) \
+		$(VENV_DIR)/bin/$(PYTHON) test_pyarrow_native_s3.py || exit 1; \
+		echo "✅ All tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test.log 2>/dev/null || echo "No startup log available"; \
+		exit 1; \
+	fi
+
+# Test cross-filesystem compatibility (s3fs ↔ PyArrow native S3)
+test-cross-fs: setup-python
+	@echo "$(YELLOW)Running cross-filesystem compatibility tests...$(NC)"
+	@echo "$(YELLOW)Assuming SeaweedFS is already running on localhost:$(S3_PORT)$(NC)"
+	@S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+	 S3_ACCESS_KEY=$(ACCESS_KEY) \
+	 S3_SECRET_KEY=$(SECRET_KEY) \
+	 BUCKET_NAME=$(BUCKET_NAME) \
+	 $(VENV_DIR)/bin/$(PYTHON) test_cross_filesystem_compatibility.py
+
+# Test cross-filesystem compatibility with automatic server management
+test-cross-fs-with-server: build-weed setup-python
+	@echo "🚀 Starting cross-filesystem compatibility tests with automated server management..."
+	@echo "Starting SeaweedFS cluster..."
+	@if $(MAKE) start-seaweedfs-ci > weed-test.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully"; \
+		echo "Running cross-filesystem compatibility tests..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=$(BUCKET_NAME) \
+		$(VENV_DIR)/bin/$(PYTHON) test_cross_filesystem_compatibility.py || exit 1; \
+		echo "✅ All tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test.log 2>/dev/null || echo "No startup log available"; \
+		exit 1; \
+	fi
+
+# Test PyArrow's native S3 filesystem compatibility with SSE-S3 enabled backend
+# (For encryption-specific validation, use test-sse-s3-compat)
+test-native-s3-with-sse: build-weed setup-python
+	@echo "🚀 Testing PyArrow native S3 compatibility with SSE-S3 enabled backend..."
+	@echo "Starting SeaweedFS cluster with SSE-S3 enabled..."
+	@if $(MAKE) start-seaweedfs-ci ENABLE_SSE_S3=true > weed-test-sse.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully with SSE-S3"; \
+		echo "Running PyArrow native S3 filesystem tests with SSE-S3..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=$(BUCKET_NAME) \
+		$(VENV_DIR)/bin/$(PYTHON) test_pyarrow_native_s3.py || exit 1; \
+		echo "✅ All SSE-S3 tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster with SSE-S3"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test-sse.log 2>/dev/null || echo "No startup log available"; \
+		exit 1; \
+	fi
+
+# Comprehensive SSE-S3 compatibility test
+test-sse-s3-compat: build-weed setup-python
+	@echo "🚀 Starting comprehensive SSE-S3 compatibility tests..."
+	@echo "Starting SeaweedFS cluster with SSE-S3 enabled..."
+	@if $(MAKE) start-seaweedfs-ci ENABLE_SSE_S3=true > weed-test-sse-compat.log 2>&1; then \
+		echo "✅ SeaweedFS cluster started successfully with SSE-S3"; \
+		echo "Running comprehensive SSE-S3 compatibility tests..."; \
+		trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
+		S3_ENDPOINT_URL=http://localhost:$(S3_PORT) \
+		S3_ACCESS_KEY=$(ACCESS_KEY) \
+		S3_SECRET_KEY=$(SECRET_KEY) \
+		BUCKET_NAME=$(BUCKET_NAME) \
+		$(VENV_DIR)/bin/$(PYTHON) test_sse_s3_compatibility.py || exit 1; \
+		echo "✅ All SSE-S3 compatibility tests completed successfully"; \
+	else \
+		echo "❌ Failed to start SeaweedFS cluster with SSE-S3"; \
+		echo "=== Server startup logs ==="; \
+		tail -100 weed-test-sse-compat.log 2>/dev/null || echo "No startup log available"; \
+		exit 1; \
+	fi
+
+# CI/CD targets
+ci-test: test-with-server
+

test/s3/parquet/README.md

@@ -0,0 +1,312 @@
+# PyArrow Parquet S3 Compatibility Tests
+
+This directory contains tests for PyArrow Parquet compatibility with SeaweedFS S3 API, including the implicit directory detection fix.
+
+## Overview
+
+**Status**: ✅ **All PyArrow methods work correctly with SeaweedFS**
+
+SeaweedFS implements implicit directory detection to improve compatibility with s3fs and PyArrow. When PyArrow writes datasets using `write_dataset()`, it may create directory markers that can confuse s3fs. SeaweedFS now handles these correctly by returning 404 for HEAD requests on implicit directories (directories with children), forcing s3fs to use LIST-based discovery.
+
+## Quick Start
+
+### Running the Example Script
+
+```bash
+# Start SeaweedFS server
+make start-seaweedfs-ci
+
+# Run the example script
+python3 example_pyarrow_native.py
+
+# Or with uv (if available)
+uv run example_pyarrow_native.py
+
+# Stop the server when done
+make stop-seaweedfs-safe
+```
+
+### Running Tests
+
+```bash
+# Setup Python environment
+make setup-python
+
+# Run all tests with server (small and large files)
+make test-with-server
+
+# Run quick tests with small files only (faster for development)
+make test-quick
+
+# Run implicit directory fix tests
+make test-implicit-dir-with-server
+
+# Run PyArrow native S3 filesystem tests
+make test-native-s3-with-server
+
+# Run cross-filesystem compatibility tests (s3fs ↔ PyArrow native)
+make test-cross-fs-with-server
+
+# Run SSE-S3 encryption tests
+make test-sse-s3-compat
+
+# Clean up
+make clean
+```
+
+### Using PyArrow with SeaweedFS
+
+#### Option 1: Using s3fs (recommended for compatibility)
+
+```python
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pyarrow.dataset as pads
+import s3fs
+
+# Configure s3fs
+fs = s3fs.S3FileSystem(
+    key='your_access_key',
+    secret='your_secret_key',
+    endpoint_url='http://localhost:8333',
+    use_ssl=False
+)
+
+# Write dataset (creates directory structure)
+table = pa.table({'id': [1, 2, 3], 'value': ['a', 'b', 'c']})
+pads.write_dataset(table, 'bucket/dataset', filesystem=fs)
+
+# Read dataset (all methods work!)
+dataset = pads.dataset('bucket/dataset', filesystem=fs)  # ✅
+table = pq.read_table('bucket/dataset', filesystem=fs)   # ✅
+dataset = pq.ParquetDataset('bucket/dataset', filesystem=fs)  # ✅
+```
+
+#### Option 2: Using PyArrow's native S3 filesystem (pure PyArrow)
+
+```python
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pyarrow.dataset as pads
+import pyarrow.fs as pafs
+
+# Configure PyArrow's native S3 filesystem
+s3 = pafs.S3FileSystem(
+    access_key='your_access_key',
+    secret_key='your_secret_key',
+    endpoint_override='localhost:8333',
+    scheme='http',
+    allow_bucket_creation=True,
+    allow_bucket_deletion=True
+)
+
+# Write dataset
+table = pa.table({'id': [1, 2, 3], 'value': ['a', 'b', 'c']})
+pads.write_dataset(table, 'bucket/dataset', filesystem=s3)
+
+# Read dataset (all methods work!)
+table = pq.read_table('bucket/dataset', filesystem=s3)  # ✅
+dataset = pq.ParquetDataset('bucket/dataset', filesystem=s3)  # ✅
+dataset = pads.dataset('bucket/dataset', filesystem=s3)  # ✅
+```
+
+## Test Files
+
+### Main Test Suite
+- **`s3_parquet_test.py`** - Comprehensive PyArrow test suite
+  - Tests 2 write methods × 5 read methods × 2 dataset sizes = 20 combinations
+  - Uses s3fs library for S3 operations
+  - All tests pass with the implicit directory fix ✅
+
+### PyArrow Native S3 Tests
+- **`test_pyarrow_native_s3.py`** - PyArrow's native S3 filesystem tests
+  - Tests PyArrow's built-in S3FileSystem (pyarrow.fs.S3FileSystem)
+  - Pure PyArrow solution without s3fs dependency
+  - Tests 3 read methods × 2 dataset sizes = 6 scenarios
+  - All tests pass ✅
+
+- **`test_sse_s3_compatibility.py`** - SSE-S3 encryption compatibility tests
+  - Tests PyArrow native S3 with SSE-S3 server-side encryption
+  - Tests 5 different file sizes (10 to 500,000 rows)
+  - Verifies multipart upload encryption works correctly
+  - All tests pass ✅
+
+### Cross-Filesystem Compatibility Tests
+- **`test_cross_filesystem_compatibility.py`** - Verifies cross-compatibility between s3fs and PyArrow native S3
+  - Tests write with s3fs → read with PyArrow native S3
+  - Tests write with PyArrow native S3 → read with s3fs
+  - Tests 2 directions × 3 read methods × 2 dataset sizes = 12 scenarios
+  - Validates that files written by either filesystem can be read by the other
+  - **All tests pass** ✅
+  - See **`CROSS_FILESYSTEM_COMPATIBILITY.md`** for detailed test results and analysis
+
+### Implicit Directory Tests
+- **`test_implicit_directory_fix.py`** - Specific tests for the implicit directory fix
+  - Tests HEAD request behavior
+  - Tests s3fs directory detection
+  - Tests PyArrow dataset reading
+  - All 6 tests pass ✅
+
+### Examples
+- **`example_pyarrow_native.py`** - Simple standalone example
+  - Demonstrates PyArrow's native S3 filesystem usage
+  - Can be run with `uv run` or regular Python
+  - Minimal dependencies (pyarrow, boto3)
+
+### Configuration
+- **`Makefile`** - Build and test automation
+- **`requirements.txt`** - Python dependencies (pyarrow, s3fs, boto3)
+- **`.gitignore`** - Ignore patterns for test artifacts
+
+## Documentation
+
+### Technical Documentation
+- **`TEST_COVERAGE.md`** - Comprehensive test coverage documentation
+  - Unit tests (Go): 17 test cases
+  - Integration tests (Python): 6 test cases
+  - End-to-end tests (Python): 20 test cases
+
+- **`FINAL_ROOT_CAUSE_ANALYSIS.md`** - Deep technical analysis
+  - Root cause of the s3fs compatibility issue
+  - How the implicit directory fix works
+  - Performance considerations
+
+- **`CROSS_FILESYSTEM_COMPATIBILITY.md`** - Cross-filesystem compatibility test results ✅ **NEW**
+  - Validates s3fs ↔ PyArrow native S3 interoperability
+  - Confirms files written by either can be read by the other
+  - Test methodology and detailed results
+
+- **`MINIO_DIRECTORY_HANDLING.md`** - Comparison with MinIO
+  - How MinIO handles directory markers
+  - Differences in implementation approaches
+
+## The Implicit Directory Fix
+
+### Problem
+When PyArrow writes datasets with `write_dataset()`, it may create 0-byte directory markers. s3fs's `info()` method calls HEAD on these paths, and if HEAD returns 200 with size=0, s3fs incorrectly reports them as files instead of directories. This causes PyArrow to fail with "Parquet file size is 0 bytes".
+
+### Solution
+SeaweedFS now returns 404 for HEAD requests on implicit directories (0-byte objects or directories with children, when requested without a trailing slash). This forces s3fs to fall back to LIST-based discovery, which correctly identifies directories by checking for children.
+
+### Implementation
+The fix is implemented in `weed/s3api/s3api_object_handlers.go`:
+- `HeadObjectHandler` - Returns 404 for implicit directories
+- `hasChildren` - Helper function to check if a path has children
+
+See the source code for detailed inline documentation.
+
+### Test Coverage
+- **Unit tests** (Go): `weed/s3api/s3api_implicit_directory_test.go`
+  - Run: `cd weed/s3api && go test -v -run TestImplicitDirectory`
+  
+- **Integration tests** (Python): `test_implicit_directory_fix.py`
+  - Run: `cd test/s3/parquet && make test-implicit-dir-with-server`
+
+- **End-to-end tests** (Python): `s3_parquet_test.py`
+  - Run: `cd test/s3/parquet && make test-with-server`
+
+## Makefile Targets
+
+```bash
+# Setup
+make setup-python          # Create Python virtual environment and install dependencies
+make build-weed           # Build SeaweedFS binary
+
+# Testing
+make test                 # Run full tests (assumes server is already running)
+make test-with-server     # Run full PyArrow test suite with server (small + large files)
+make test-quick           # Run quick tests with small files only (assumes server is running)
+make test-implicit-dir-with-server  # Run implicit directory tests with server
+make test-native-s3       # Run PyArrow native S3 tests (assumes server is running)
+make test-native-s3-with-server  # Run PyArrow native S3 tests with server management
+make test-cross-fs        # Run cross-filesystem compatibility tests (assumes server is running)
+make test-cross-fs-with-server  # Run cross-filesystem compatibility tests with server management
+make test-sse-s3-compat   # Run comprehensive SSE-S3 encryption compatibility tests
+
+# Server Management
+make start-seaweedfs-ci   # Start SeaweedFS in background (CI mode)
+make stop-seaweedfs-safe  # Stop SeaweedFS gracefully
+make clean                # Clean up all test artifacts
+
+# Development
+make help                 # Show all available targets
+```
+
+## Continuous Integration
+
+The tests are automatically run in GitHub Actions on every push/PR that affects S3 or filer code:
+
+**Workflow**: `.github/workflows/s3-parquet-tests.yml`
+
+**Test Matrix**:
+- Python versions: 3.9, 3.11, 3.12
+- PyArrow integration tests (s3fs): 20 test combinations
+- PyArrow native S3 tests: 6 test scenarios ✅
+- Cross-filesystem compatibility tests: 12 test scenarios ✅ **NEW**
+- SSE-S3 encryption tests: 5 file sizes ✅
+- Implicit directory fix tests: 6 test scenarios
+- Go unit tests: 17 test cases
+
+**Test Steps** (run for each Python version):
+1. Build SeaweedFS
+2. Run PyArrow Parquet integration tests (`make test-with-server`)
+3. Run implicit directory fix tests (`make test-implicit-dir-with-server`)
+4. Run PyArrow native S3 filesystem tests (`make test-native-s3-with-server`)
+5. Run cross-filesystem compatibility tests (`make test-cross-fs-with-server`) ✅ **NEW**
+6. Run SSE-S3 encryption compatibility tests (`make test-sse-s3-compat`)
+7. Run Go unit tests for implicit directory handling
+
+**Triggers**:
+- Push/PR to master (when `weed/s3api/**` or `weed/filer/**` changes)
+- Manual trigger via GitHub UI (workflow_dispatch)
+
+## Requirements
+
+- Python 3.8+
+- PyArrow 22.0.0+
+- s3fs 2024.12.0+
+- boto3 1.40.0+
+- SeaweedFS (latest)
+
+## AWS S3 Compatibility
+
+The implicit directory fix makes SeaweedFS behavior more compatible with AWS S3:
+- AWS S3 typically doesn't create directory markers for implicit directories
+- HEAD on "dataset" (when only "dataset/file.txt" exists) returns 404 on AWS
+- SeaweedFS now matches this behavior for implicit directories with children
+
+## Edge Cases Handled
+
+✅ **Implicit directories with children** → 404 (forces LIST-based discovery)  
+✅ **Empty files (0-byte, no children)** → 200 (legitimate empty file)  
+✅ **Empty directories (no children)** → 200 (legitimate empty directory)  
+✅ **Explicit directory requests (trailing slash)** → 200 (normal directory behavior)  
+✅ **Versioned buckets** → Skip implicit directory check (versioned semantics)  
+✅ **Regular files** → 200 (normal file behavior)
+
+## Performance
+
+The implicit directory check adds minimal overhead:
+- Only triggered for 0-byte objects or directories without trailing slash
+- Cost: One LIST operation with Limit=1 (~1-5ms)
+- No impact on regular file operations
+
+## Contributing
+
+When adding new tests:
+1. Add test cases to the appropriate test file
+2. Update TEST_COVERAGE.md
+3. Run the full test suite to ensure no regressions
+4. Update this README if adding new functionality
+
+## References
+
+- [PyArrow Documentation](https://arrow.apache.org/docs/python/parquet.html)
+- [s3fs Documentation](https://s3fs.readthedocs.io/)
+- [SeaweedFS S3 API](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API)
+- [AWS S3 API Reference](https://docs.aws.amazon.com/AmazonS3/latest/API/)
+
+---
+
+**Last Updated**: November 19, 2025  
+**Status**: All tests passing ✅

test/s3/parquet/debug_write_dataset.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Debug script to understand what pads.write_dataset creates."""
+
+import sys
+import pyarrow as pa
+import pyarrow.dataset as pads
+import s3fs
+
+# Create a simple test table
+table = pa.table({'id': [1, 2, 3], 'value': [1.0, 2.0, 3.0]})
+
+# Initialize S3 filesystem
+fs = s3fs.S3FileSystem(
+    client_kwargs={'endpoint_url': 'http://localhost:8333'},
+    key='some_access_key1',
+    secret='some_secret_key1',
+    use_listings_cache=False,
+)
+
+# Create bucket
+if not fs.exists('test-bucket'):
+    fs.mkdir('test-bucket')
+
+# Write with pads.write_dataset
+test_path = 's3://test-bucket/test-write-simple/'
+print(f"Writing to: {test_path}")
+print(f"Table schema: {table.schema}")
+print(f"Table rows: {table.num_rows}")
+
+try:
+    pads.write_dataset(table, test_path, format='parquet', filesystem=fs)
+    print("\n✓ Write succeeded")
+    
+    # List all files recursively
+    print(f"\nListing all files recursively under {test_path}:")
+    import os
+    base_path = 'test-bucket/test-write-simple'
+    def list_recursive(path, indent=0):
+        try:
+            items = fs.ls(path, detail=False)
+            for item in items:
+                is_dir = fs.isdir(item)
+                item_name = item.split('/')[-1] if '/' in item else item
+                if is_dir:
+                    print(f"{'  ' * indent}📁 {item_name}/")
+                    list_recursive(item, indent + 1)
+                else:
+                    # Get file size
+                    try:
+                        info = fs.info(item)
+                        size = info.get('size', 0)
+                        print(f"{'  ' * indent}📄 {item_name} ({size} bytes)")
+                    except:
+                        print(f"{'  ' * indent}📄 {item_name}")
+        except Exception as e:
+            print(f"{'  ' * indent}Error listing {path}: {e}")
+
+    list_recursive(base_path)
+    
+    # Try to read back with different methods
+    print(f"\n\nTrying to read back using different methods:")
+    
+    # Method 1: pads.dataset with the same path
+    print(f"\n1. pads.dataset('{test_path}'):")
+    try:
+        dataset = pads.dataset(test_path, format='parquet', filesystem=fs)
+        result = dataset.to_table()
+        print(f"   ✓ Success: {result.num_rows} rows")
+    except Exception as e:
+        print(f"   ✗ Failed: {e}")
+    
+    # Method 2: pads.dataset with the dir containing parquet files
+    print(f"\n2. pads.dataset without trailing slash:")
+    test_path_no_slash = 's3://test-bucket/test-write-simple'
+    try:
+        dataset = pads.dataset(test_path_no_slash, format='parquet', filesystem=fs)
+        result = dataset.to_table()
+        print(f"   ✓ Success: {result.num_rows} rows")
+    except Exception as e:
+        print(f"   ✗ Failed: {e}")
+
+except Exception as e:
+    import traceback
+    print(f"✗ Error: {e}")
+    traceback.print_exc()
+    sys.exit(1)

test/s3/parquet/example_pyarrow_native.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# /// script
+# dependencies = [
+#     "pyarrow>=22",
+#     "boto3>=1.28.0",
+# ]
+# ///
+
+"""
+Simple example of using PyArrow's native S3 filesystem with SeaweedFS.
+
+This is a minimal example demonstrating how to write and read Parquet files
+using PyArrow's built-in S3FileSystem without any additional dependencies
+like s3fs.
+
+Usage:
+    # Set environment variables
+    export S3_ENDPOINT_URL=localhost:8333
+    export S3_ACCESS_KEY=some_access_key1
+    export S3_SECRET_KEY=some_secret_key1
+    export BUCKET_NAME=test-parquet-bucket
+
+    # Run the script
+    python3 example_pyarrow_native.py
+    
+    # Or run with uv (if available)
+    uv run example_pyarrow_native.py
+"""
+
+import os
+import secrets
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+
+from parquet_test_utils import create_sample_table
+
+# Configuration
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-parquet-bucket")
+S3_ENDPOINT_URL = os.getenv("S3_ENDPOINT_URL", "localhost:8333")
+S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "some_secret_key1")
+
+# Determine scheme from endpoint
+if S3_ENDPOINT_URL.startswith("http://"):
+    scheme = "http"
+    endpoint = S3_ENDPOINT_URL[7:]
+elif S3_ENDPOINT_URL.startswith("https://"):
+    scheme = "https"
+    endpoint = S3_ENDPOINT_URL[8:]
+else:
+    scheme = "http"  # Default to http for localhost
+    endpoint = S3_ENDPOINT_URL
+
+print(f"Connecting to S3 endpoint: {scheme}://{endpoint}")
+
+# Initialize PyArrow's NATIVE S3 filesystem
+s3 = pafs.S3FileSystem(
+    access_key=S3_ACCESS_KEY,
+    secret_key=S3_SECRET_KEY,
+    endpoint_override=endpoint,
+    scheme=scheme,
+    allow_bucket_creation=True,
+    allow_bucket_deletion=True,
+)
+
+print("✓ Connected to S3 endpoint")
+
+
+# Create bucket if needed (using boto3)
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+    
+    s3_client = boto3.client(
+        's3',
+        endpoint_url=f"{scheme}://{endpoint}",
+        aws_access_key_id=S3_ACCESS_KEY,
+        aws_secret_access_key=S3_SECRET_KEY,
+        region_name='us-east-1',
+    )
+    
+    try:
+        s3_client.head_bucket(Bucket=BUCKET_NAME)
+        print(f"✓ Bucket exists: {BUCKET_NAME}")
+    except ClientError as e:
+        if e.response['Error']['Code'] == '404':
+            print(f"Creating bucket: {BUCKET_NAME}")
+            s3_client.create_bucket(Bucket=BUCKET_NAME)
+            print(f"✓ Bucket created: {BUCKET_NAME}")
+        else:
+            raise
+except ImportError:
+    print("Warning: boto3 not available, assuming bucket exists")
+
+# Generate a unique filename
+filename = f"{BUCKET_NAME}/dataset-{secrets.token_hex(8)}/test.parquet"
+
+print(f"\nWriting Parquet dataset to: {filename}")
+
+# Write dataset
+table = create_sample_table(200_000)
+pads.write_dataset(
+    table,
+    filename,
+    filesystem=s3,
+    format="parquet",
+)
+
+print(f"✓ Wrote {table.num_rows:,} rows")
+
+# Read with pq.read_table
+print("\nReading with pq.read_table...")
+table_read = pq.read_table(filename, filesystem=s3)
+print(f"✓ Read {table_read.num_rows:,} rows")
+
+# Read with pq.ParquetDataset
+print("\nReading with pq.ParquetDataset...")
+dataset = pq.ParquetDataset(filename, filesystem=s3)
+table_dataset = dataset.read()
+print(f"✓ Read {table_dataset.num_rows:,} rows")
+
+# Read with pads.dataset
+print("\nReading with pads.dataset...")
+dataset_pads = pads.dataset(filename, filesystem=s3)
+table_pads = dataset_pads.to_table()
+print(f"✓ Read {table_pads.num_rows:,} rows")
+
+print("\n✅ All operations completed successfully!")
+print(f"\nFile written to: {filename}")
+print("You can verify the file using the SeaweedFS S3 API or weed shell")
+

test/s3/parquet/parquet_test_utils.py

@@ -0,0 +1,41 @@
+"""
+Shared utility functions for PyArrow Parquet tests.
+
+This module provides common test utilities used across multiple test scripts
+to avoid code duplication and ensure consistency.
+"""
+
+import pyarrow as pa
+
+
+def create_sample_table(num_rows: int = 5) -> pa.Table:
+    """Create a sample PyArrow table for testing.
+    
+    Args:
+        num_rows: Number of rows to generate (default: 5)
+    
+    Returns:
+        PyArrow Table with test data containing:
+        - id: int64 sequential IDs (0 to num_rows-1)
+        - name: string user names (user_0, user_1, ...)
+        - value: float64 values (id * 1.5)
+        - flag: bool alternating True/False based on even/odd id
+    
+    Example:
+        >>> table = create_sample_table(3)
+        >>> print(table)
+        pyarrow.Table
+        id: int64
+        name: string
+        value: double
+        flag: bool
+    """
+    return pa.table(
+        {
+            "id": pa.array(range(num_rows), type=pa.int64()),
+            "name": pa.array([f"user_{i}" for i in range(num_rows)], type=pa.string()),
+            "value": pa.array([float(i) * 1.5 for i in range(num_rows)], type=pa.float64()),
+            "flag": pa.array([i % 2 == 0 for i in range(num_rows)], type=pa.bool_()),
+        }
+    )
+

test/s3/parquet/requirements.txt

@@ -0,0 +1,7 @@
+# Python dependencies for S3 Parquet tests
+# Install with: pip install -r requirements.txt
+
+pyarrow>=10.0.0
+s3fs>=2023.12.0
+boto3>=1.28.0
+

test/s3/parquet/s3_parquet_test.py

@@ -0,0 +1,421 @@
+#!/usr/bin/env python3
+"""
+Test script for S3-compatible storage with PyArrow Parquet files.
+
+This script tests different write methods (PyArrow write_dataset vs. pq.write_table to buffer)
+combined with different read methods (PyArrow dataset, direct s3fs read, buffered read) to
+identify which combinations work with large files that span multiple row groups.
+
+This test specifically addresses issues with large tables using PyArrow where files span
+multiple row-groups (default row_group size is around 130,000 rows).
+
+Requirements:
+    - pyarrow>=22
+    - s3fs>=2024.12.0
+
+Environment Variables:
+    S3_ENDPOINT_URL: S3 endpoint (default: http://localhost:8333)
+    S3_ACCESS_KEY: S3 access key (default: some_access_key1)
+    S3_SECRET_KEY: S3 secret key (default: some_secret_key1)
+    BUCKET_NAME: S3 bucket name (default: test-parquet-bucket)
+    TEST_QUICK: Run only small/quick tests (default: 0, set to 1 for quick mode)
+
+Usage:
+    # Run with default environment variables
+    python3 s3_parquet_test.py
+
+    # Run with custom environment variables
+    S3_ENDPOINT_URL=http://localhost:8333 \
+    S3_ACCESS_KEY=mykey \
+    S3_SECRET_KEY=mysecret \
+    BUCKET_NAME=mybucket \
+    python3 s3_parquet_test.py
+"""
+
+import io
+import logging
+import os
+import secrets
+import sys
+import traceback
+from datetime import datetime
+from typing import Tuple
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.parquet as pq
+
+try:
+    import s3fs
+except ImportError:
+    logging.error("s3fs not installed. Install with: pip install s3fs")
+    sys.exit(1)
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+# Error log file
+ERROR_LOG_FILE = f"s3_parquet_test_errors_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log"
+
+# Configuration from environment variables with defaults
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "http://localhost:8333")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "some_secret_key1")
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-parquet-bucket")
+TEST_QUICK = os.getenv("TEST_QUICK", "0") == "1"
+
+# Create randomized test directory
+TEST_RUN_ID = secrets.token_hex(8)
+TEST_DIR = f"{BUCKET_NAME}/parquet-tests/{TEST_RUN_ID}"
+
+# Test file sizes
+TEST_SIZES = {
+    "small": 5,
+    "large": 200_000,  # This will create multiple row groups
+}
+
+# Filter to only small tests if quick mode is enabled
+if TEST_QUICK:
+    TEST_SIZES = {"small": TEST_SIZES["small"]}
+    logging.info("Quick test mode enabled - running only small tests")
+
+
+def create_sample_table(num_rows: int = 5) -> pa.Table:
+    """Create a sample PyArrow table for testing."""
+    return pa.table({
+        "id": pa.array(range(num_rows), type=pa.int64()),
+        "name": pa.array([f"user_{i}" for i in range(num_rows)], type=pa.string()),
+        "value": pa.array([float(i) * 1.5 for i in range(num_rows)], type=pa.float64()),
+        "flag": pa.array([i % 2 == 0 for i in range(num_rows)], type=pa.bool_()),
+    })
+
+
+def log_error(operation: str, short_msg: str) -> None:
+    """Log error details to file with full traceback."""
+    with open(ERROR_LOG_FILE, "a") as f:
+        f.write(f"\n{'='*80}\n")
+        f.write(f"Operation: {operation}\n")
+        f.write(f"Time: {datetime.now().isoformat()}\n")
+        f.write(f"Message: {short_msg}\n")
+        f.write("Full Traceback:\n")
+        f.write(traceback.format_exc())
+        f.write(f"{'='*80}\n")
+
+
+def init_s3fs() -> s3fs.S3FileSystem:
+    """Initialize and return S3FileSystem."""
+    logging.info("Initializing S3FileSystem...")
+    logging.info(f"  Endpoint: {S3_ENDPOINT_URL}")
+    logging.info(f"  Bucket: {BUCKET_NAME}")
+    try:
+        fs = s3fs.S3FileSystem(
+            client_kwargs={"endpoint_url": S3_ENDPOINT_URL},
+            key=S3_ACCESS_KEY,
+            secret=S3_SECRET_KEY,
+            use_listings_cache=False,
+        )
+        logging.info("✓ S3FileSystem initialized successfully\n")
+        return fs
+    except Exception:
+        logging.exception("✗ Failed to initialize S3FileSystem")
+        raise
+
+
+def ensure_bucket_exists(fs: s3fs.S3FileSystem) -> None:
+    """Ensure the test bucket exists."""
+    try:
+        if not fs.exists(BUCKET_NAME):
+            logging.info(f"Creating bucket: {BUCKET_NAME}")
+            fs.mkdir(BUCKET_NAME)
+            logging.info(f"✓ Bucket created: {BUCKET_NAME}")
+        else:
+            logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+    except Exception:
+        logging.exception("✗ Failed to create/check bucket")
+        raise
+
+
+# Write Methods
+
+def write_with_pads(table: pa.Table, path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str]:
+    """Write using pads.write_dataset with filesystem parameter."""
+    try:
+        pads.write_dataset(table, path, format="parquet", filesystem=fs)
+        return True, "pads.write_dataset"
+    except Exception as e:
+        error_msg = f"pads.write_dataset: {type(e).__name__}"
+        log_error("write_with_pads", error_msg)
+        return False, error_msg
+
+
+def write_with_buffer_and_s3fs(table: pa.Table, path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str]:
+    """Write using pq.write_table to buffer, then upload via s3fs."""
+    try:
+        buffer = io.BytesIO()
+        pq.write_table(table, buffer)
+        buffer.seek(0)
+        with fs.open(path, "wb") as f:
+            f.write(buffer.read())
+        return True, "pq.write_table+s3fs.open"
+    except Exception as e:
+        error_msg = f"pq.write_table+s3fs.open: {type(e).__name__}"
+        log_error("write_with_buffer_and_s3fs", error_msg)
+        return False, error_msg
+
+
+# Read Methods
+
+def get_parquet_files(path: str, fs: s3fs.S3FileSystem) -> list:
+    """
+    Helper to discover all parquet files for a given path.
+    
+    Args:
+        path: S3 path (file or directory)
+        fs: S3FileSystem instance
+        
+    Returns:
+        List of parquet file paths
+        
+    Raises:
+        ValueError: If no parquet files are found in a directory
+    """
+    if fs.isdir(path):
+        # Find all parquet files in the directory
+        files = [f for f in fs.ls(path) if f.endswith('.parquet')]
+        if not files:
+            raise ValueError(f"No parquet files found in directory: {path}")
+        return files
+    else:
+        # Single file path
+        return [path]
+
+
+def read_with_pads_dataset(path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str, int]:
+    """Read using pads.dataset - handles both single files and directories."""
+    try:
+        # pads.dataset() should auto-discover parquet files in the directory
+        dataset = pads.dataset(path, format="parquet", filesystem=fs)
+        result = dataset.to_table()
+        return True, "pads.dataset", result.num_rows
+    except Exception as e:
+        error_msg = f"pads.dataset: {type(e).__name__}"
+        log_error("read_with_pads_dataset", error_msg)
+        return False, error_msg, 0
+
+
+def read_direct_s3fs(path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str, int]:
+    """Read directly via s3fs.open() streaming."""
+    try:
+        # Get all parquet files (handles both single file and directory)
+        parquet_files = get_parquet_files(path, fs)
+        
+        # Read all parquet files and concatenate them
+        tables = []
+        for file_path in parquet_files:
+            with fs.open(file_path, "rb") as f:
+                table = pq.read_table(f)
+                tables.append(table)
+        
+        # Concatenate all tables into one
+        if len(tables) == 1:
+            result = tables[0]
+        else:
+            result = pa.concat_tables(tables)
+        
+        return True, "s3fs.open+pq.read_table", result.num_rows
+    except Exception as e:
+        error_msg = f"s3fs.open+pq.read_table: {type(e).__name__}"
+        log_error("read_direct_s3fs", error_msg)
+        return False, error_msg, 0
+
+
+def read_buffered_s3fs(path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str, int]:
+    """Read via s3fs.open() into buffer, then pq.read_table."""
+    try:
+        # Get all parquet files (handles both single file and directory)
+        parquet_files = get_parquet_files(path, fs)
+        
+        # Read all parquet files and concatenate them
+        tables = []
+        for file_path in parquet_files:
+            with fs.open(file_path, "rb") as f:
+                buffer = io.BytesIO(f.read())
+            buffer.seek(0)
+            table = pq.read_table(buffer)
+            tables.append(table)
+        
+        # Concatenate all tables into one
+        if len(tables) == 1:
+            result = tables[0]
+        else:
+            result = pa.concat_tables(tables)
+        
+        return True, "s3fs.open+BytesIO+pq.read_table", result.num_rows
+    except Exception as e:
+        error_msg = f"s3fs.open+BytesIO+pq.read_table: {type(e).__name__}"
+        log_error("read_buffered_s3fs", error_msg)
+        return False, error_msg, 0
+
+
+def read_with_parquet_dataset(path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str, int]:
+    """Read using pq.ParquetDataset - designed for directories."""
+    try:
+        # ParquetDataset is specifically designed to handle directories
+        dataset = pq.ParquetDataset(path, filesystem=fs)
+        result = dataset.read()
+        return True, "pq.ParquetDataset", result.num_rows
+    except Exception as e:
+        error_msg = f"pq.ParquetDataset: {type(e).__name__}"
+        log_error("read_with_parquet_dataset", error_msg)
+        return False, error_msg, 0
+
+
+def read_with_pq_read_table(path: str, fs: s3fs.S3FileSystem) -> Tuple[bool, str, int]:
+    """Read using pq.read_table with filesystem parameter."""
+    try:
+        # pq.read_table() with filesystem should handle directories
+        result = pq.read_table(path, filesystem=fs)
+        return True, "pq.read_table+filesystem", result.num_rows
+    except Exception as e:
+        error_msg = f"pq.read_table+filesystem: {type(e).__name__}"
+        log_error("read_with_pq_read_table", error_msg)
+        return False, error_msg, 0
+
+
+def test_combination(
+    fs: s3fs.S3FileSystem,
+    test_name: str,
+    write_func,
+    read_func,
+    num_rows: int,
+) -> Tuple[bool, str]:
+    """Test a specific write/read combination."""
+    table = create_sample_table(num_rows=num_rows)
+    path = f"{TEST_DIR}/{test_name}/data.parquet"
+
+    # Write
+    write_ok, write_msg = write_func(table, path, fs)
+    if not write_ok:
+        return False, f"WRITE_FAIL: {write_msg}"
+
+    # Read
+    read_ok, read_msg, rows_read = read_func(path, fs)
+    if not read_ok:
+        return False, f"READ_FAIL: {read_msg}"
+
+    # Verify
+    if rows_read != num_rows:
+        return False, f"DATA_MISMATCH: expected {num_rows}, got {rows_read}"
+
+    return True, f"{write_msg} + {read_msg}"
+
+
+def cleanup_test_files(fs: s3fs.S3FileSystem) -> None:
+    """Clean up test files from S3."""
+    try:
+        if fs.exists(TEST_DIR):
+            logging.info(f"Cleaning up test directory: {TEST_DIR}")
+            fs.rm(TEST_DIR, recursive=True)
+            logging.info("✓ Test directory cleaned up")
+    except Exception as e:
+        logging.warning(f"Failed to cleanup test directory: {e}")
+
+
+def main():
+    """Run all write/read method combinations."""
+    print("=" * 80)
+    print("Write/Read Method Combination Tests for S3-Compatible Storage")
+    print("Testing PyArrow Parquet Files with Multiple Row Groups")
+    if TEST_QUICK:
+        print("*** QUICK TEST MODE - Small files only ***")
+    print("=" * 80 + "\n")
+
+    print("Configuration:")
+    print(f"  S3 Endpoint: {S3_ENDPOINT_URL}")
+    print(f"  Bucket: {BUCKET_NAME}")
+    print(f"  Test Directory: {TEST_DIR}")
+    print(f"  Quick Mode: {'Yes (small files only)' if TEST_QUICK else 'No (all file sizes)'}")
+    print()
+
+    try:
+        fs = init_s3fs()
+        ensure_bucket_exists(fs)
+    except Exception as e:
+        print(f"Cannot proceed without S3 connection: {e}")
+        return 1
+
+    # Define all write methods
+    write_methods = [
+        ("pads", write_with_pads),
+        ("buffer+s3fs", write_with_buffer_and_s3fs),
+    ]
+
+    # Define all read methods
+    read_methods = [
+        ("pads.dataset", read_with_pads_dataset),
+        ("pq.ParquetDataset", read_with_parquet_dataset),
+        ("pq.read_table", read_with_pq_read_table),
+        ("s3fs+direct", read_direct_s3fs),
+        ("s3fs+buffered", read_buffered_s3fs),
+    ]
+
+    results = []
+
+    # Test all combinations for each file size
+    for size_name, num_rows in TEST_SIZES.items():
+        print(f"\n{'='*80}")
+        print(f"Testing with {size_name} files ({num_rows:,} rows)")
+        print(f"{'='*80}\n")
+        print(f"{'Write Method':<20} | {'Read Method':<20} | {'Result':<40}")
+        print("-" * 85)
+
+        for write_name, write_func in write_methods:
+            for read_name, read_func in read_methods:
+                test_name = f"{size_name}_{write_name}_{read_name}"
+                success, message = test_combination(
+                    fs, test_name, write_func, read_func, num_rows
+                )
+                results.append((test_name, success, message))
+                status = "✓ PASS" if success else "✗ FAIL"
+                print(f"{write_name:<20} | {read_name:<20} | {status}: {message[:35]}")
+
+    # Summary
+    print("\n" + "=" * 80)
+    print("SUMMARY")
+    print("=" * 80)
+    passed = sum(1 for _, success, _ in results if success)
+    total = len(results)
+    print(f"\nTotal: {passed}/{total} passed\n")
+
+    # Group results by file size
+    for size_name in TEST_SIZES.keys():
+        size_results = [r for r in results if size_name in r[0]]
+        size_passed = sum(1 for _, success, _ in size_results if success)
+        print(f"{size_name.upper()}: {size_passed}/{len(size_results)} passed")
+
+    print("\n" + "=" * 80)
+    if passed == total:
+        print("✓ ALL TESTS PASSED!")
+    else:
+        print(f"✗ {total - passed} test(s) failed")
+        print("\nFailing combinations:")
+        for name, success, message in results:
+            if not success:
+                parts = name.split("_")
+                size = parts[0]
+                write = parts[1]
+                read = "_".join(parts[2:])
+                print(f"  - {size:6} | {write:15} | {read:20} -> {message[:50]}")
+
+    print("=" * 80 + "\n")
+    print(f"Error details logged to: {ERROR_LOG_FILE}")
+    print("=" * 80 + "\n")
+
+    # Cleanup
+    cleanup_test_files(fs)
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

test/s3/parquet/test_cross_filesystem_compatibility.py

@@ -0,0 +1,453 @@
+#!/usr/bin/env python3
+"""
+Cross-filesystem compatibility tests for PyArrow Parquet files.
+
+This test verifies that Parquet files written using one filesystem implementation
+(s3fs or PyArrow native S3) can be correctly read using the other implementation.
+
+Test Matrix:
+- Write with s3fs → Read with PyArrow native S3
+- Write with PyArrow native S3 → Read with s3fs
+
+Requirements:
+    - pyarrow>=22.0.0
+    - s3fs>=2024.12.0
+    - boto3>=1.40.0
+
+Environment Variables:
+    S3_ENDPOINT_URL: S3 endpoint (default: http://localhost:8333)
+    S3_ACCESS_KEY: S3 access key (default: some_access_key1)
+    S3_SECRET_KEY: S3 secret key (default: some_secret_key1)
+    BUCKET_NAME: S3 bucket name (default: test-parquet-bucket)
+    TEST_QUICK: Run only small/quick tests (default: 0, set to 1 for quick mode)
+
+Usage:
+    # Run with default environment variables
+    python3 test_cross_filesystem_compatibility.py
+
+    # Run with custom environment variables
+    S3_ENDPOINT_URL=http://localhost:8333 \
+    S3_ACCESS_KEY=mykey \
+    S3_SECRET_KEY=mysecret \
+    BUCKET_NAME=mybucket \
+    python3 test_cross_filesystem_compatibility.py
+"""
+
+import os
+import secrets
+import sys
+import logging
+from typing import Optional, Tuple
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+import s3fs
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+    HAS_BOTO3 = True
+except ImportError:
+    HAS_BOTO3 = False
+
+from parquet_test_utils import create_sample_table
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+# Configuration from environment variables with defaults
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "http://localhost:8333")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "some_secret_key1")
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-parquet-bucket")
+TEST_QUICK = os.getenv("TEST_QUICK", "0") == "1"
+
+# Create randomized test directory
+TEST_RUN_ID = secrets.token_hex(8)
+TEST_DIR = f"parquet-cross-fs-tests/{TEST_RUN_ID}"
+
+# Test file sizes
+TEST_SIZES = {
+    "small": 5,
+    "large": 200_000,  # This will create multiple row groups
+}
+
+# Filter to only small tests if quick mode is enabled
+if TEST_QUICK:
+    TEST_SIZES = {"small": TEST_SIZES["small"]}
+    logging.info("Quick test mode enabled - running only small tests")
+
+
+def init_s3fs() -> Optional[s3fs.S3FileSystem]:
+    """Initialize s3fs filesystem."""
+    try:
+        logging.info("Initializing s3fs...")
+        fs = s3fs.S3FileSystem(
+            client_kwargs={"endpoint_url": S3_ENDPOINT_URL},
+            key=S3_ACCESS_KEY,
+            secret=S3_SECRET_KEY,
+            use_listings_cache=False,
+        )
+        logging.info("✓ s3fs initialized successfully")
+        return fs
+    except Exception:
+        logging.exception("✗ Failed to initialize s3fs")
+        return None
+
+
+def init_pyarrow_s3() -> Tuple[Optional[pafs.S3FileSystem], str, str]:
+    """Initialize PyArrow's native S3 filesystem.
+    
+    Returns:
+        tuple: (S3FileSystem instance, scheme, endpoint)
+    """
+    try:
+        logging.info("Initializing PyArrow S3FileSystem...")
+        
+        # Determine scheme from endpoint
+        if S3_ENDPOINT_URL.startswith("http://"):
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL[7:]  # Remove http://
+        elif S3_ENDPOINT_URL.startswith("https://"):
+            scheme = "https"
+            endpoint = S3_ENDPOINT_URL[8:]  # Remove https://
+        else:
+            # Default to http for localhost
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL
+        
+        # Enable bucket creation and deletion for testing
+        s3 = pafs.S3FileSystem(
+            access_key=S3_ACCESS_KEY,
+            secret_key=S3_SECRET_KEY,
+            endpoint_override=endpoint,
+            scheme=scheme,
+            allow_bucket_creation=True,
+            allow_bucket_deletion=True,
+        )
+        
+        logging.info("✓ PyArrow S3FileSystem initialized successfully")
+        return s3, scheme, endpoint
+    except Exception:
+        logging.exception("✗ Failed to initialize PyArrow S3FileSystem")
+        return None, "", ""
+
+
+def ensure_bucket_exists(s3fs_fs: s3fs.S3FileSystem, pyarrow_s3: pafs.S3FileSystem) -> bool:
+    """Ensure the test bucket exists using s3fs."""
+    try:
+        if not s3fs_fs.exists(BUCKET_NAME):
+            logging.info(f"Creating bucket: {BUCKET_NAME}")
+            try:
+                s3fs_fs.mkdir(BUCKET_NAME)
+                logging.info(f"✓ Bucket created: {BUCKET_NAME}")
+            except FileExistsError:
+                # Bucket was created between the check and mkdir call
+                logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+        else:
+            logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+        return True
+    except Exception:
+        logging.exception("✗ Failed to create/check bucket")
+        return False
+
+
+def write_with_s3fs(table: pa.Table, path: str, s3fs_fs: s3fs.S3FileSystem) -> bool:
+    """Write Parquet file using s3fs filesystem."""
+    try:
+        pads.write_dataset(table, path, format="parquet", filesystem=s3fs_fs)
+        return True
+    except Exception:
+        logging.exception("✗ Failed to write with s3fs")
+        return False
+
+
+def write_with_pyarrow_s3(table: pa.Table, path: str, pyarrow_s3: pafs.S3FileSystem) -> bool:
+    """Write Parquet file using PyArrow native S3 filesystem."""
+    try:
+        pads.write_dataset(table, path, format="parquet", filesystem=pyarrow_s3)
+        return True
+    except Exception:
+        logging.exception("✗ Failed to write with PyArrow S3")
+        return False
+
+
+def read_with_s3fs(path: str, s3fs_fs: s3fs.S3FileSystem) -> Tuple[bool, Optional[pa.Table], str]:
+    """Read Parquet file using s3fs filesystem with multiple methods."""
+    errors = []
+    
+    # Try pq.read_table
+    try:
+        table = pq.read_table(path, filesystem=s3fs_fs)
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pq.read_table: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pq.read_table"
+    
+    # Try pq.ParquetDataset
+    try:
+        dataset = pq.ParquetDataset(path, filesystem=s3fs_fs)
+        table = dataset.read()
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pq.ParquetDataset: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pq.ParquetDataset"
+    
+    # Try pads.dataset
+    try:
+        dataset = pads.dataset(path, format="parquet", filesystem=s3fs_fs)
+        table = dataset.to_table()
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pads.dataset: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pads.dataset"
+    
+    return False, None, " | ".join(errors)
+
+
+def read_with_pyarrow_s3(path: str, pyarrow_s3: pafs.S3FileSystem) -> Tuple[bool, Optional[pa.Table], str]:
+    """Read Parquet file using PyArrow native S3 filesystem with multiple methods."""
+    errors = []
+    
+    # Try pq.read_table
+    try:
+        table = pq.read_table(path, filesystem=pyarrow_s3)
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pq.read_table: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pq.read_table"
+    
+    # Try pq.ParquetDataset
+    try:
+        dataset = pq.ParquetDataset(path, filesystem=pyarrow_s3)
+        table = dataset.read()
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pq.ParquetDataset: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pq.ParquetDataset"
+    
+    # Try pads.dataset
+    try:
+        dataset = pads.dataset(path, filesystem=pyarrow_s3)
+        table = dataset.to_table()
+    except Exception as e:  # noqa: BLE001 - Intentionally broad for compatibility testing
+        errors.append(f"pads.dataset: {type(e).__name__}: {e}")
+    else:
+        return True, table, "pads.dataset"
+    
+    return False, None, " | ".join(errors)
+
+
+def verify_table_integrity(original: pa.Table, read: pa.Table) -> Tuple[bool, str]:
+    """Verify that read table matches the original table."""
+    # Check row count
+    if read.num_rows != original.num_rows:
+        return False, f"Row count mismatch: expected {original.num_rows}, got {read.num_rows}"
+    
+    # Check schema
+    if not read.schema.equals(original.schema):
+        return False, f"Schema mismatch: expected {original.schema}, got {read.schema}"
+    
+    # Sort both tables by 'id' column before comparison to handle potential row order differences
+    original_sorted = original.sort_by([('id', 'ascending')])
+    read_sorted = read.sort_by([('id', 'ascending')])
+    
+    # Check data equality
+    if not read_sorted.equals(original_sorted):
+        # Provide detailed error information
+        error_details = []
+        for col_name in original.column_names:
+            col_original = original_sorted.column(col_name)
+            col_read = read_sorted.column(col_name)
+            if not col_original.equals(col_read):
+                error_details.append(f"column '{col_name}' differs")
+        return False, f"Data mismatch: {', '.join(error_details)}"
+    
+    return True, "Data verified successfully"
+
+
+def test_write_s3fs_read_pyarrow(
+    test_name: str,
+    num_rows: int,
+    s3fs_fs: s3fs.S3FileSystem,
+    pyarrow_s3: pafs.S3FileSystem
+) -> Tuple[bool, str]:
+    """Test: Write with s3fs, read with PyArrow native S3."""
+    try:
+        table = create_sample_table(num_rows)
+        path = f"{BUCKET_NAME}/{TEST_DIR}/{test_name}/data.parquet"
+        
+        # Write with s3fs
+        logging.info(f"  Writing {num_rows:,} rows with s3fs to {path}...")
+        if not write_with_s3fs(table, path, s3fs_fs):
+            return False, "Write with s3fs failed"
+        logging.info("  ✓ Write completed")
+        
+        # Read with PyArrow native S3
+        logging.info("  Reading with PyArrow native S3...")
+        success, read_table, method = read_with_pyarrow_s3(path, pyarrow_s3)
+        if not success:
+            return False, f"Read with PyArrow S3 failed: {method}"
+        logging.info(f"  ✓ Read {read_table.num_rows:,} rows using {method}")
+        
+        # Verify data integrity
+        verify_success, verify_msg = verify_table_integrity(table, read_table)
+        if not verify_success:
+            return False, f"Verification failed: {verify_msg}"
+        logging.info(f"  ✓ {verify_msg}")
+        
+        return True, f"s3fs→PyArrow: {method}"
+        
+    except Exception as e:  # noqa: BLE001 - Top-level exception handler for test orchestration
+        logging.exception("  ✗ Test failed")
+        return False, f"{type(e).__name__}: {e}"
+
+
+def test_write_pyarrow_read_s3fs(
+    test_name: str,
+    num_rows: int,
+    s3fs_fs: s3fs.S3FileSystem,
+    pyarrow_s3: pafs.S3FileSystem
+) -> Tuple[bool, str]:
+    """Test: Write with PyArrow native S3, read with s3fs."""
+    try:
+        table = create_sample_table(num_rows)
+        path = f"{BUCKET_NAME}/{TEST_DIR}/{test_name}/data.parquet"
+        
+        # Write with PyArrow native S3
+        logging.info(f"  Writing {num_rows:,} rows with PyArrow native S3 to {path}...")
+        if not write_with_pyarrow_s3(table, path, pyarrow_s3):
+            return False, "Write with PyArrow S3 failed"
+        logging.info("  ✓ Write completed")
+        
+        # Read with s3fs
+        logging.info("  Reading with s3fs...")
+        success, read_table, method = read_with_s3fs(path, s3fs_fs)
+        if not success:
+            return False, f"Read with s3fs failed: {method}"
+        logging.info(f"  ✓ Read {read_table.num_rows:,} rows using {method}")
+        
+        # Verify data integrity
+        verify_success, verify_msg = verify_table_integrity(table, read_table)
+        if not verify_success:
+            return False, f"Verification failed: {verify_msg}"
+        logging.info(f"  ✓ {verify_msg}")
+        
+        return True, f"PyArrow→s3fs: {method}"
+        
+    except Exception as e:  # noqa: BLE001 - Top-level exception handler for test orchestration
+        logging.exception("  ✗ Test failed")
+        return False, f"{type(e).__name__}: {e}"
+
+
+def cleanup_test_files(s3fs_fs: s3fs.S3FileSystem) -> None:
+    """Clean up test files from S3."""
+    try:
+        test_path = f"{BUCKET_NAME}/{TEST_DIR}"
+        if s3fs_fs.exists(test_path):
+            logging.info(f"Cleaning up test directory: {test_path}")
+            s3fs_fs.rm(test_path, recursive=True)
+            logging.info("✓ Test directory cleaned up")
+    except Exception:
+        logging.exception("Failed to cleanup test directory")
+
+
+def main():
+    """Run cross-filesystem compatibility tests."""
+    print("=" * 80)
+    print("Cross-Filesystem Compatibility Tests for PyArrow Parquet")
+    print("Testing: s3fs ↔ PyArrow Native S3 Filesystem")
+    if TEST_QUICK:
+        print("*** QUICK TEST MODE - Small files only ***")
+    print("=" * 80 + "\n")
+
+    print("Configuration:")
+    print(f"  S3 Endpoint: {S3_ENDPOINT_URL}")
+    print(f"  Access Key: {S3_ACCESS_KEY}")
+    print(f"  Bucket: {BUCKET_NAME}")
+    print(f"  Test Directory: {TEST_DIR}")
+    print(f"  Quick Mode: {'Yes (small files only)' if TEST_QUICK else 'No (all file sizes)'}")
+    print(f"  PyArrow Version: {pa.__version__}")
+    print()
+
+    # Initialize both filesystems
+    s3fs_fs = init_s3fs()
+    if s3fs_fs is None:
+        print("Cannot proceed without s3fs connection")
+        return 1
+
+    pyarrow_s3, _scheme, _endpoint = init_pyarrow_s3()
+    if pyarrow_s3 is None:
+        print("Cannot proceed without PyArrow S3 connection")
+        return 1
+
+    print()
+
+    # Ensure bucket exists
+    if not ensure_bucket_exists(s3fs_fs, pyarrow_s3):
+        print("Cannot proceed without bucket")
+        return 1
+
+    print()
+
+    results = []
+
+    # Test all file sizes
+    for size_name, num_rows in TEST_SIZES.items():
+        print(f"\n{'='*80}")
+        print(f"Testing with {size_name} files ({num_rows:,} rows)")
+        print(f"{'='*80}\n")
+
+        # Test 1: Write with s3fs, read with PyArrow native S3
+        test_name = f"{size_name}_s3fs_to_pyarrow"
+        print(f"Test: Write with s3fs → Read with PyArrow native S3")
+        success, message = test_write_s3fs_read_pyarrow(
+            test_name, num_rows, s3fs_fs, pyarrow_s3
+        )
+        results.append((test_name, success, message))
+        status = "✓ PASS" if success else "✗ FAIL"
+        print(f"{status}: {message}\n")
+
+        # Test 2: Write with PyArrow native S3, read with s3fs
+        test_name = f"{size_name}_pyarrow_to_s3fs"
+        print(f"Test: Write with PyArrow native S3 → Read with s3fs")
+        success, message = test_write_pyarrow_read_s3fs(
+            test_name, num_rows, s3fs_fs, pyarrow_s3
+        )
+        results.append((test_name, success, message))
+        status = "✓ PASS" if success else "✗ FAIL"
+        print(f"{status}: {message}\n")
+
+    # Summary
+    print("\n" + "=" * 80)
+    print("SUMMARY")
+    print("=" * 80)
+    passed = sum(1 for _, success, _ in results if success)
+    total = len(results)
+    print(f"\nTotal: {passed}/{total} passed\n")
+
+    for test_name, success, message in results:
+        status = "✓" if success else "✗"
+        print(f"  {status} {test_name}: {message}")
+
+    print("\n" + "=" * 80)
+    if passed == total:
+        print("✓ ALL CROSS-FILESYSTEM TESTS PASSED!")
+        print()
+        print("Conclusion: Files written with s3fs and PyArrow native S3 are")
+        print("fully compatible and can be read by either filesystem implementation.")
+    else:
+        print(f"✗ {total - passed} test(s) failed")
+
+    print("=" * 80 + "\n")
+
+    # Cleanup
+    cleanup_test_files(s3fs_fs)
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

test/s3/parquet/test_implicit_directory_fix.py

@@ -0,0 +1,308 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the implicit directory fix for s3fs compatibility.
+
+This test verifies that:
+1. Implicit directory markers (0-byte objects with children) return 404 on HEAD
+2. s3fs correctly identifies them as directories via LIST fallback
+3. PyArrow can read datasets created with write_dataset()
+
+The fix makes SeaweedFS behave like AWS S3 and improves s3fs compatibility.
+"""
+
+import io
+import logging
+import os
+import sys
+import traceback
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.parquet as pq
+import s3fs
+import boto3
+from botocore.exceptions import ClientError
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Configuration
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "http://localhost:8333")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "some_secret_key1")
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-implicit-dir")
+
+def create_sample_table(num_rows: int = 1000) -> pa.Table:
+    """Create a sample PyArrow table."""
+    return pa.table({
+        'id': pa.array(range(num_rows), type=pa.int64()),
+        'value': pa.array([f'value_{i}' for i in range(num_rows)], type=pa.string()),
+        'score': pa.array([float(i) * 1.5 for i in range(num_rows)], type=pa.float64()),
+    })
+
+def setup_s3():
+    """Set up S3 clients."""
+    # s3fs client
+    fs = s3fs.S3FileSystem(
+        key=S3_ACCESS_KEY,
+        secret=S3_SECRET_KEY,
+        client_kwargs={'endpoint_url': S3_ENDPOINT_URL},
+        use_ssl=False
+    )
+    
+    # boto3 client for raw S3 operations
+    s3_client = boto3.client(
+        's3',
+        endpoint_url=S3_ENDPOINT_URL,
+        aws_access_key_id=S3_ACCESS_KEY,
+        aws_secret_access_key=S3_SECRET_KEY,
+        region_name='us-east-1',
+        use_ssl=False
+    )
+    
+    return fs, s3_client
+
+def test_implicit_directory_head_behavior(fs, s3_client):
+    """Test that HEAD on implicit directory markers returns 404."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 1: Implicit Directory HEAD Behavior")
+    logger.info("="*80)
+    
+    test_path = f"{BUCKET_NAME}/test_implicit_dir"
+    
+    # Clean up any existing data
+    try:
+        fs.rm(test_path, recursive=True)
+    except:
+        pass
+    
+    # Create a dataset using PyArrow (creates implicit directory)
+    logger.info(f"Creating dataset at: {test_path}")
+    table = create_sample_table(1000)
+    pads.write_dataset(table, test_path, filesystem=fs, format='parquet')
+    
+    # List what was created
+    logger.info("\nFiles created:")
+    files = fs.ls(test_path, detail=True)
+    for f in files:
+        logger.info(f"  {f['name']} - size: {f['size']} bytes, type: {f['type']}")
+    
+    # Test HEAD request on the directory marker (without trailing slash)
+    logger.info(f"\nTesting HEAD on: {test_path}")
+    try:
+        response = s3_client.head_object(Bucket=BUCKET_NAME, Key='test_implicit_dir')
+        logger.info(f"  HEAD response: {response['ResponseMetadata']['HTTPStatusCode']}")
+        logger.info(f"  Content-Length: {response.get('ContentLength', 'N/A')}")
+        logger.info(f"  Content-Type: {response.get('ContentType', 'N/A')}")
+        logger.warning("  ⚠️  Expected 404, but got 200 - fix may not be working")
+        return False
+    except ClientError as e:
+        if e.response['Error']['Code'] == '404':
+            logger.info("  ✓ HEAD returned 404 (expected - implicit directory)")
+            return True
+        else:
+            logger.error(f"  ✗ Unexpected error: {e}")
+            return False
+
+def test_s3fs_directory_detection(fs):
+    """Test that s3fs correctly detects the directory."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 2: s3fs Directory Detection")
+    logger.info("="*80)
+    
+    test_path = f"{BUCKET_NAME}/test_implicit_dir"
+    
+    # Test s3fs.info()
+    logger.info(f"\nTesting s3fs.info('{test_path}'):")
+    try:
+        info = fs.info(test_path)
+        logger.info(f"  Type: {info.get('type', 'N/A')}")
+        logger.info(f"  Size: {info.get('size', 'N/A')}")
+        
+        if info.get('type') == 'directory':
+            logger.info("  ✓ s3fs correctly identified as directory")
+            return True
+        else:
+            logger.warning(f"  ⚠️  s3fs identified as: {info.get('type')}")
+            return False
+    except Exception as e:
+        logger.error(f"  ✗ Error: {e}")
+        return False
+
+def test_s3fs_isdir(fs):
+    """Test that s3fs.isdir() works correctly."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 3: s3fs.isdir() Method")
+    logger.info("="*80)
+    
+    test_path = f"{BUCKET_NAME}/test_implicit_dir"
+    
+    logger.info(f"\nTesting s3fs.isdir('{test_path}'):")
+    try:
+        is_dir = fs.isdir(test_path)
+        logger.info(f"  Result: {is_dir}")
+        
+        if is_dir:
+            logger.info("  ✓ s3fs.isdir() correctly returned True")
+            return True
+        else:
+            logger.warning("  ⚠️  s3fs.isdir() returned False")
+            return False
+    except Exception as e:
+        logger.error(f"  ✗ Error: {e}")
+        return False
+
+def test_pyarrow_dataset_read(fs):
+    """Test that PyArrow can read the dataset."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 4: PyArrow Dataset Read")
+    logger.info("="*80)
+    
+    test_path = f"{BUCKET_NAME}/test_implicit_dir"
+    
+    logger.info(f"\nReading dataset from: {test_path}")
+    try:
+        ds = pads.dataset(test_path, filesystem=fs, format='parquet')
+        table = ds.to_table()
+        logger.info(f"  ✓ Successfully read {len(table)} rows")
+        logger.info(f"  Columns: {table.column_names}")
+        return True
+    except Exception as e:
+        logger.error(f"  ✗ Failed to read dataset: {e}")
+        traceback.print_exc()
+        return False
+
+def test_explicit_directory_marker(fs, s3_client):
+    """Test that explicit directory markers (with trailing slash) still work."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 5: Explicit Directory Marker (with trailing slash)")
+    logger.info("="*80)
+    
+    # Create an explicit directory marker
+    logger.info(f"Creating explicit directory: {BUCKET_NAME}/explicit_dir/")
+    try:
+        s3_client.put_object(
+            Bucket=BUCKET_NAME,
+            Key='explicit_dir/',
+            Body=b'',
+            ContentType='httpd/unix-directory'
+        )
+        logger.info("  ✓ Created explicit directory marker")
+    except Exception as e:
+        logger.error(f"  ✗ Failed to create: {e}")
+        return False
+    
+    # Test HEAD with trailing slash
+    logger.info(f"\nTesting HEAD on: {BUCKET_NAME}/explicit_dir/")
+    try:
+        response = s3_client.head_object(Bucket=BUCKET_NAME, Key='explicit_dir/')
+        logger.info(f"  ✓ HEAD returned 200 (expected for explicit directory)")
+        logger.info(f"  Content-Type: {response.get('ContentType', 'N/A')}")
+        return True
+    except ClientError as e:
+        logger.error(f"  ✗ HEAD failed: {e}")
+        return False
+
+def test_empty_file_not_directory(fs, s3_client):
+    """Test that legitimate empty files are not treated as directories."""
+    logger.info("\n" + "="*80)
+    logger.info("TEST 6: Empty File (not a directory)")
+    logger.info("="*80)
+    
+    # Create an empty file with text/plain mime type
+    logger.info(f"\nCreating empty file: {BUCKET_NAME}/empty.txt")
+    try:
+        s3_client.put_object(
+            Bucket=BUCKET_NAME,
+            Key='empty.txt',
+            Body=b'',
+            ContentType='text/plain'
+        )
+        logger.info("  ✓ Created empty file")
+    except Exception as e:
+        logger.error(f"  ✗ Failed to create: {e}")
+        return False
+    
+    # Test HEAD
+    logger.info(f"\nTesting HEAD on: {BUCKET_NAME}/empty.txt")
+    try:
+        response = s3_client.head_object(Bucket=BUCKET_NAME, Key='empty.txt')
+        logger.info(f"  ✓ HEAD returned 200 (expected for empty file)")
+        logger.info(f"  Content-Type: {response.get('ContentType', 'N/A')}")
+        
+        # Verify s3fs doesn't think it's a directory
+        info = fs.info(f"{BUCKET_NAME}/empty.txt")
+        if info.get('type') == 'file':
+            logger.info("  ✓ s3fs correctly identified as file")
+            return True
+        else:
+            logger.warning(f"  ⚠️  s3fs identified as: {info.get('type')}")
+            return False
+    except Exception as e:
+        logger.error(f"  ✗ Error: {e}")
+        return False
+
+def main():
+    """Run all tests."""
+    logger.info("="*80)
+    logger.info("Implicit Directory Fix Test Suite")
+    logger.info("="*80)
+    logger.info(f"Endpoint: {S3_ENDPOINT_URL}")
+    logger.info(f"Bucket: {BUCKET_NAME}")
+    logger.info("="*80)
+    
+    # Set up S3 clients
+    fs, s3_client = setup_s3()
+    
+    # Create bucket if it doesn't exist
+    try:
+        s3_client.create_bucket(Bucket=BUCKET_NAME)
+        logger.info(f"\n✓ Created bucket: {BUCKET_NAME}")
+    except ClientError as e:
+        error_code = e.response['Error']['Code']
+        if error_code in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
+            logger.info(f"\n✓ Bucket already exists: {BUCKET_NAME}")
+        else:
+            logger.error(f"\n✗ Failed to create bucket: {e}")
+            return 1
+    
+    # Run tests
+    results = []
+    
+    results.append(("Implicit Directory HEAD", test_implicit_directory_head_behavior(fs, s3_client)))
+    results.append(("s3fs Directory Detection", test_s3fs_directory_detection(fs)))
+    results.append(("s3fs.isdir() Method", test_s3fs_isdir(fs)))
+    results.append(("PyArrow Dataset Read", test_pyarrow_dataset_read(fs)))
+    results.append(("Explicit Directory Marker", test_explicit_directory_marker(fs, s3_client)))
+    results.append(("Empty File Not Directory", test_empty_file_not_directory(fs, s3_client)))
+    
+    # Print summary
+    logger.info("\n" + "="*80)
+    logger.info("TEST SUMMARY")
+    logger.info("="*80)
+    
+    passed = sum(1 for _, result in results if result)
+    total = len(results)
+    
+    for name, result in results:
+        status = "✓ PASS" if result else "✗ FAIL"
+        logger.info(f"{status}: {name}")
+    
+    logger.info("="*80)
+    logger.info(f"Results: {passed}/{total} tests passed")
+    logger.info("="*80)
+    
+    if passed == total:
+        logger.info("\n🎉 All tests passed! The implicit directory fix is working correctly.")
+        return 0
+    else:
+        logger.warning(f"\n⚠️  {total - passed} test(s) failed. The fix may not be fully working.")
+        return 1
+
+if __name__ == "__main__":
+    sys.exit(main())
+

test/s3/parquet/test_pyarrow_native_s3.py

@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+"""
+Test script for PyArrow's NATIVE S3 filesystem with SeaweedFS.
+
+This test uses PyArrow's built-in S3FileSystem (pyarrow.fs.S3FileSystem)
+instead of s3fs, providing a pure PyArrow solution for reading and writing
+Parquet files to S3-compatible storage.
+
+Requirements:
+    - pyarrow>=10.0.0
+
+Environment Variables:
+    S3_ENDPOINT_URL: S3 endpoint (default: localhost:8333)
+    S3_ACCESS_KEY: S3 access key (default: some_access_key1)
+    S3_SECRET_KEY: S3 secret key (default: some_secret_key1)
+    BUCKET_NAME: S3 bucket name (default: test-parquet-bucket)
+    TEST_QUICK: Run only small/quick tests (default: 0, set to 1 for quick mode)
+
+Usage:
+    # Run with default environment variables
+    python3 test_pyarrow_native_s3.py
+
+    # Run with custom environment variables
+    S3_ENDPOINT_URL=localhost:8333 \
+    S3_ACCESS_KEY=mykey \
+    S3_SECRET_KEY=mysecret \
+    BUCKET_NAME=mybucket \
+    python3 test_pyarrow_native_s3.py
+"""
+
+import os
+import secrets
+import sys
+import logging
+from typing import Optional
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+    HAS_BOTO3 = True
+except ImportError:
+    HAS_BOTO3 = False
+
+from parquet_test_utils import create_sample_table
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+# Configuration from environment variables with defaults
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "localhost:8333")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "some_secret_key1")
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-parquet-bucket")
+TEST_QUICK = os.getenv("TEST_QUICK", "0") == "1"
+
+# Create randomized test directory
+TEST_RUN_ID = secrets.token_hex(8)
+TEST_DIR = f"parquet-native-tests/{TEST_RUN_ID}"
+
+# Test file sizes
+TEST_SIZES = {
+    "small": 5,
+    "large": 200_000,  # This will create multiple row groups
+}
+
+# Filter to only small tests if quick mode is enabled
+if TEST_QUICK:
+    TEST_SIZES = {"small": TEST_SIZES["small"]}
+    logging.info("Quick test mode enabled - running only small tests")
+
+
+def init_s3_filesystem() -> tuple[Optional[pafs.S3FileSystem], str, str]:
+    """Initialize PyArrow's native S3 filesystem.
+    
+    Returns:
+        tuple: (S3FileSystem instance, scheme, endpoint)
+    """
+    try:
+        logging.info("Initializing PyArrow S3FileSystem...")
+        logging.info(f"  Endpoint: {S3_ENDPOINT_URL}")
+        logging.info(f"  Bucket: {BUCKET_NAME}")
+        
+        # Determine scheme from endpoint
+        if S3_ENDPOINT_URL.startswith("http://"):
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL[7:]  # Remove http://
+        elif S3_ENDPOINT_URL.startswith("https://"):
+            scheme = "https"
+            endpoint = S3_ENDPOINT_URL[8:]  # Remove https://
+        else:
+            # Default to http for localhost
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL
+        
+        # Enable bucket creation and deletion for testing
+        s3 = pafs.S3FileSystem(
+            access_key=S3_ACCESS_KEY,
+            secret_key=S3_SECRET_KEY,
+            endpoint_override=endpoint,
+            scheme=scheme,
+            allow_bucket_creation=True,
+            allow_bucket_deletion=True,
+        )
+        
+        logging.info("✓ PyArrow S3FileSystem initialized successfully\n")
+        return s3, scheme, endpoint
+    except Exception:
+        logging.exception("✗ Failed to initialize PyArrow S3FileSystem")
+        return None, "", ""
+
+
+def ensure_bucket_exists_boto3(scheme: str, endpoint: str) -> bool:
+    """Ensure the test bucket exists using boto3."""
+    if not HAS_BOTO3:
+        logging.error("boto3 is required for bucket creation")
+        return False
+    
+    try:
+        # Create boto3 client
+        endpoint_url = f"{scheme}://{endpoint}"
+        s3_client = boto3.client(
+            's3',
+            endpoint_url=endpoint_url,
+            aws_access_key_id=S3_ACCESS_KEY,
+            aws_secret_access_key=S3_SECRET_KEY,
+            region_name='us-east-1',
+        )
+        
+        # Check if bucket exists
+        try:
+            s3_client.head_bucket(Bucket=BUCKET_NAME)
+            logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+            return True
+        except ClientError as e:
+            error_code = e.response['Error']['Code']
+            if error_code == '404':
+                # Bucket doesn't exist, create it
+                logging.info(f"Creating bucket: {BUCKET_NAME}")
+                s3_client.create_bucket(Bucket=BUCKET_NAME)
+                logging.info(f"✓ Bucket created: {BUCKET_NAME}")
+                return True
+            else:
+                raise
+    except Exception:
+        logging.exception("✗ Failed to create/check bucket")
+        return False
+
+
+def ensure_bucket_exists(s3: pafs.S3FileSystem) -> bool:
+    """Ensure the test bucket exists using PyArrow's native S3FileSystem."""
+    try:
+        # Check if bucket exists by trying to list it
+        try:
+            file_info = s3.get_file_info(BUCKET_NAME)
+            if file_info.type == pafs.FileType.Directory:
+                logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+                return True
+        except OSError as e:
+            # OSError typically means bucket not found or network/permission issues
+            error_msg = str(e).lower()
+            if "not found" in error_msg or "does not exist" in error_msg or "nosuchbucket" in error_msg:
+                logging.debug(f"Bucket '{BUCKET_NAME}' not found, will attempt creation: {e}")
+            else:
+                # Log other OSErrors (network, auth, etc.) for debugging
+                logging.debug(f"Error checking bucket '{BUCKET_NAME}', will attempt creation anyway: {type(e).__name__}: {e}")
+        except Exception as e:
+            # Catch any other unexpected exceptions and log them
+            logging.debug(f"Unexpected error checking bucket '{BUCKET_NAME}', will attempt creation: {type(e).__name__}: {e}")
+        
+        # Try to create the bucket
+        logging.info(f"Creating bucket: {BUCKET_NAME}")
+        s3.create_dir(BUCKET_NAME)
+        logging.info(f"✓ Bucket created: {BUCKET_NAME}")
+        return True
+    except Exception:
+        logging.exception(f"✗ Failed to create/check bucket '{BUCKET_NAME}' with PyArrow")
+        return False
+
+
+def test_write_and_read(s3: pafs.S3FileSystem, test_name: str, num_rows: int) -> tuple[bool, str]:
+    """Test writing and reading a Parquet dataset using PyArrow's native S3 filesystem."""
+    try:
+        table = create_sample_table(num_rows)
+        
+        # Write using pads.write_dataset
+        filename = f"{BUCKET_NAME}/{TEST_DIR}/{test_name}/data.parquet"
+        logging.info(f"  Writing {num_rows:,} rows to {filename}...")
+        
+        pads.write_dataset(
+            table,
+            filename,
+            filesystem=s3,
+            format="parquet",
+        )
+        logging.info("  ✓ Write completed")
+        
+        # Test Method 1: Read with pq.read_table
+        logging.info("  Reading with pq.read_table...")
+        table_read = pq.read_table(filename, filesystem=s3)
+        if table_read.num_rows != num_rows:
+            return False, f"pq.read_table: Row count mismatch (expected {num_rows}, got {table_read.num_rows})"
+        
+        # Check schema first
+        if not table_read.schema.equals(table.schema):
+            return False, f"pq.read_table: Schema mismatch (expected {table.schema}, got {table_read.schema})"
+        
+        # Sort both tables by 'id' column before comparison to handle potential row order differences
+        table_sorted = table.sort_by([('id', 'ascending')])
+        table_read_sorted = table_read.sort_by([('id', 'ascending')])
+        
+        if not table_read_sorted.equals(table_sorted):
+            # Provide more detailed error information
+            error_details = []
+            for col_name in table.column_names:
+                col_original = table_sorted.column(col_name)
+                col_read = table_read_sorted.column(col_name)
+                if not col_original.equals(col_read):
+                    error_details.append(f"column '{col_name}' differs")
+            return False, f"pq.read_table: Table contents mismatch ({', '.join(error_details)})"
+        logging.info(f"  ✓ pq.read_table: {table_read.num_rows:,} rows")
+        
+        # Test Method 2: Read with pq.ParquetDataset
+        logging.info("  Reading with pq.ParquetDataset...")
+        dataset = pq.ParquetDataset(filename, filesystem=s3)
+        table_dataset = dataset.read()
+        if table_dataset.num_rows != num_rows:
+            return False, f"pq.ParquetDataset: Row count mismatch (expected {num_rows}, got {table_dataset.num_rows})"
+        
+        # Sort before comparison
+        table_dataset_sorted = table_dataset.sort_by([('id', 'ascending')])
+        if not table_dataset_sorted.equals(table_sorted):
+            error_details = []
+            for col_name in table.column_names:
+                col_original = table_sorted.column(col_name)
+                col_read = table_dataset_sorted.column(col_name)
+                if not col_original.equals(col_read):
+                    error_details.append(f"column '{col_name}' differs")
+            return False, f"pq.ParquetDataset: Table contents mismatch ({', '.join(error_details)})"
+        logging.info(f"  ✓ pq.ParquetDataset: {table_dataset.num_rows:,} rows")
+        
+        # Test Method 3: Read with pads.dataset
+        logging.info("  Reading with pads.dataset...")
+        dataset_pads = pads.dataset(filename, filesystem=s3)
+        table_pads = dataset_pads.to_table()
+        if table_pads.num_rows != num_rows:
+            return False, f"pads.dataset: Row count mismatch (expected {num_rows}, got {table_pads.num_rows})"
+        
+        # Sort before comparison
+        table_pads_sorted = table_pads.sort_by([('id', 'ascending')])
+        if not table_pads_sorted.equals(table_sorted):
+            error_details = []
+            for col_name in table.column_names:
+                col_original = table_sorted.column(col_name)
+                col_read = table_pads_sorted.column(col_name)
+                if not col_original.equals(col_read):
+                    error_details.append(f"column '{col_name}' differs")
+            return False, f"pads.dataset: Table contents mismatch ({', '.join(error_details)})"
+        logging.info(f"  ✓ pads.dataset: {table_pads.num_rows:,} rows")
+        
+        return True, "All read methods passed"
+        
+    except Exception as exc:
+        logging.exception("  ✗ Test failed")
+        return False, f"{type(exc).__name__}: {exc}"
+
+
+def cleanup_test_files(s3: pafs.S3FileSystem) -> None:
+    """Clean up test files from S3.
+    
+    Note: We cannot use s3.delete_dir() directly because SeaweedFS uses implicit
+    directories (path prefixes without physical directory objects). PyArrow's
+    delete_dir() attempts to delete the directory marker itself, which fails with
+    "INTERNAL_FAILURE" on SeaweedFS. Instead, we list and delete files individually,
+    letting implicit directories disappear automatically.
+    """
+    try:
+        test_path = f"{BUCKET_NAME}/{TEST_DIR}"
+        logging.info(f"Cleaning up test directory: {test_path}")
+        
+        # List and delete files individually to handle implicit directories
+        try:
+            file_selector = pafs.FileSelector(test_path, recursive=True)
+            files = s3.get_file_info(file_selector)
+            
+            # Delete files first (not directories)
+            for file_info in files:
+                if file_info.type == pafs.FileType.File:
+                    s3.delete_file(file_info.path)
+                    logging.debug(f"  Deleted file: {file_info.path}")
+            
+            logging.info("✓ Test directory cleaned up")
+        except OSError as e:
+            # Handle the case where the path doesn't exist or is inaccessible
+            if "does not exist" in str(e).lower() or "not found" in str(e).lower():
+                logging.info("✓ Test directory already clean or doesn't exist")
+            else:
+                raise
+    except Exception:
+        logging.exception("Failed to cleanup test directory")
+
+
+def main():
+    """Run all tests with PyArrow's native S3 filesystem."""
+    print("=" * 80)
+    print("PyArrow Native S3 Filesystem Tests for SeaweedFS")
+    print("Testing Parquet Files with Multiple Row Groups")
+    if TEST_QUICK:
+        print("*** QUICK TEST MODE - Small files only ***")
+    print("=" * 80 + "\n")
+
+    print("Configuration:")
+    print(f"  S3 Endpoint: {S3_ENDPOINT_URL}")
+    print(f"  Access Key: {S3_ACCESS_KEY}")
+    print(f"  Bucket: {BUCKET_NAME}")
+    print(f"  Test Directory: {TEST_DIR}")
+    print(f"  Quick Mode: {'Yes (small files only)' if TEST_QUICK else 'No (all file sizes)'}")
+    print(f"  PyArrow Version: {pa.__version__}")
+    print()
+
+    # Initialize S3 filesystem
+    s3, scheme, endpoint = init_s3_filesystem()
+    if s3 is None:
+        print("Cannot proceed without S3 connection")
+        return 1
+
+    # Ensure bucket exists - try PyArrow first, fall back to boto3
+    bucket_created = ensure_bucket_exists(s3)
+    if not bucket_created:
+        logging.info("Trying to create bucket with boto3...")
+        bucket_created = ensure_bucket_exists_boto3(scheme, endpoint)
+    
+    if not bucket_created:
+        print("Cannot proceed without bucket")
+        return 1
+
+    results = []
+
+    # Test all file sizes
+    for size_name, num_rows in TEST_SIZES.items():
+        print(f"\n{'='*80}")
+        print(f"Testing with {size_name} files ({num_rows:,} rows)")
+        print(f"{'='*80}\n")
+
+        test_name = f"{size_name}_test"
+        success, message = test_write_and_read(s3, test_name, num_rows)
+        results.append((test_name, success, message))
+        
+        status = "✓ PASS" if success else "✗ FAIL"
+        print(f"\n{status}: {message}\n")
+
+    # Summary
+    print("\n" + "=" * 80)
+    print("SUMMARY")
+    print("=" * 80)
+    passed = sum(1 for _, success, _ in results if success)
+    total = len(results)
+    print(f"\nTotal: {passed}/{total} passed\n")
+
+    for test_name, success, message in results:
+        status = "✓" if success else "✗"
+        print(f"  {status} {test_name}: {message}")
+
+    print("\n" + "=" * 80)
+    if passed == total:
+        print("✓ ALL TESTS PASSED!")
+    else:
+        print(f"✗ {total - passed} test(s) failed")
+
+    print("=" * 80 + "\n")
+
+    # Cleanup
+    cleanup_test_files(s3)
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+

test/s3/parquet/test_sse_s3_compatibility.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""
+Test script for SSE-S3 compatibility with PyArrow native S3 filesystem.
+
+This test specifically targets the SSE-S3 multipart upload bug where
+SeaweedFS panics with "bad IV length" when reading multipart uploads
+that were encrypted with bucket-default SSE-S3.
+
+Requirements:
+    - pyarrow>=10.0.0
+    - boto3>=1.28.0
+
+Environment Variables:
+    S3_ENDPOINT_URL: S3 endpoint (default: localhost:8333)
+    S3_ACCESS_KEY: S3 access key (default: some_access_key1)
+    S3_SECRET_KEY: S3 secret key (default: some_secret_key1)
+    BUCKET_NAME: S3 bucket name (default: test-parquet-bucket)
+
+Usage:
+    # Start SeaweedFS with SSE-S3 enabled
+    make start-seaweedfs-ci ENABLE_SSE_S3=true
+    
+    # Run the test
+    python3 test_sse_s3_compatibility.py
+"""
+
+import os
+import secrets
+import sys
+import logging
+from typing import Optional
+
+import pyarrow as pa
+import pyarrow.dataset as pads
+import pyarrow.fs as pafs
+import pyarrow.parquet as pq
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+    HAS_BOTO3 = True
+except ImportError:
+    HAS_BOTO3 = False
+    logging.exception("boto3 is required for this test")
+    sys.exit(1)
+
+from parquet_test_utils import create_sample_table
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+# Configuration
+S3_ENDPOINT_URL = os.environ.get("S3_ENDPOINT_URL", "localhost:8333")
+S3_ACCESS_KEY = os.environ.get("S3_ACCESS_KEY", "some_access_key1")
+S3_SECRET_KEY = os.environ.get("S3_SECRET_KEY", "some_secret_key1")
+BUCKET_NAME = os.getenv("BUCKET_NAME", "test-parquet-bucket")
+
+TEST_RUN_ID = secrets.token_hex(8)
+TEST_DIR = f"sse-s3-tests/{TEST_RUN_ID}"
+
+# Test sizes designed to trigger multipart uploads
+# PyArrow typically uses 5MB chunks, so these sizes should trigger multipart
+TEST_SIZES = {
+    "tiny": 10,                    # Single part
+    "small": 1_000,               # Single part
+    "medium": 50_000,             # Single part (~1.5MB)
+    "large": 200_000,             # Multiple parts (~6MB)
+    "very_large": 500_000,        # Multiple parts (~15MB)
+}
+
+
+def init_s3_filesystem() -> tuple[Optional[pafs.S3FileSystem], str, str]:
+    """Initialize PyArrow's native S3 filesystem."""
+    try:
+        logging.info("Initializing PyArrow S3FileSystem...")
+        
+        # Determine scheme from endpoint
+        if S3_ENDPOINT_URL.startswith("http://"):
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL[7:]
+        elif S3_ENDPOINT_URL.startswith("https://"):
+            scheme = "https"
+            endpoint = S3_ENDPOINT_URL[8:]
+        else:
+            scheme = "http"
+            endpoint = S3_ENDPOINT_URL
+        
+        s3 = pafs.S3FileSystem(
+            access_key=S3_ACCESS_KEY,
+            secret_key=S3_SECRET_KEY,
+            endpoint_override=endpoint,
+            scheme=scheme,
+            allow_bucket_creation=True,
+            allow_bucket_deletion=True,
+        )
+        
+        logging.info("✓ PyArrow S3FileSystem initialized\n")
+        return s3, scheme, endpoint
+    except Exception:
+        logging.exception("✗ Failed to initialize PyArrow S3FileSystem")
+        return None, "", ""
+
+
+def ensure_bucket_exists(scheme: str, endpoint: str) -> bool:
+    """Ensure the test bucket exists using boto3."""
+    try:
+        endpoint_url = f"{scheme}://{endpoint}"
+        s3_client = boto3.client(
+            's3',
+            endpoint_url=endpoint_url,
+            aws_access_key_id=S3_ACCESS_KEY,
+            aws_secret_access_key=S3_SECRET_KEY,
+            region_name='us-east-1',
+        )
+        
+        try:
+            s3_client.head_bucket(Bucket=BUCKET_NAME)
+            logging.info(f"✓ Bucket exists: {BUCKET_NAME}")
+        except ClientError as e:
+            error_code = e.response['Error']['Code']
+            if error_code == '404':
+                logging.info(f"Creating bucket: {BUCKET_NAME}")
+                s3_client.create_bucket(Bucket=BUCKET_NAME)
+                logging.info(f"✓ Bucket created: {BUCKET_NAME}")
+            else:
+                logging.exception("✗ Failed to access bucket")
+                return False
+        
+        # Note: SeaweedFS doesn't support GetBucketEncryption API
+        # so we can't verify if SSE-S3 is enabled via API
+        # We assume it's configured correctly in the s3.json config file
+        logging.info("✓ Assuming SSE-S3 is configured in s3.json")
+        return True
+            
+    except Exception:
+        logging.exception("✗ Failed to check bucket")
+        return False
+
+
+def test_write_read_with_sse(
+    s3: pafs.S3FileSystem,
+    test_name: str,
+    num_rows: int
+) -> tuple[bool, str, int]:
+    """Test writing and reading with SSE-S3 encryption."""
+    try:
+        table = create_sample_table(num_rows)
+        filename = f"{BUCKET_NAME}/{TEST_DIR}/{test_name}/data.parquet"
+        
+        logging.info(f"  Writing {num_rows:,} rows...")
+        pads.write_dataset(
+            table,
+            filename,
+            filesystem=s3,
+            format="parquet",
+        )
+        
+        logging.info("  Reading back...")
+        table_read = pq.read_table(filename, filesystem=s3)
+        
+        if table_read.num_rows != num_rows:
+            return False, f"Row count mismatch: {table_read.num_rows} != {num_rows}", 0
+        
+        return True, "Success", table_read.num_rows
+        
+    except Exception as e:
+        error_msg = f"{type(e).__name__}: {e!s}"
+        logging.exception("  ✗ Failed")
+        return False, error_msg, 0
+
+
+def main():
+    """Run SSE-S3 compatibility tests."""
+    print("=" * 80)
+    print("SSE-S3 Compatibility Tests for PyArrow Native S3")
+    print("Testing Multipart Upload Encryption")
+    print("=" * 80 + "\n")
+
+    print("Configuration:")
+    print(f"  S3 Endpoint: {S3_ENDPOINT_URL}")
+    print(f"  Bucket: {BUCKET_NAME}")
+    print(f"  Test Directory: {TEST_DIR}")
+    print(f"  PyArrow Version: {pa.__version__}")
+    print()
+
+    # Initialize
+    s3, scheme, endpoint = init_s3_filesystem()
+    if s3 is None:
+        print("Cannot proceed without S3 connection")
+        return 1
+
+    # Check bucket and SSE-S3
+    if not ensure_bucket_exists(scheme, endpoint):
+        print("\n⚠ WARNING: Failed to access or create the test bucket!")
+        print("This test requires a reachable bucket with SSE-S3 enabled.")
+        print("Please ensure SeaweedFS is running with: make start-seaweedfs-ci ENABLE_SSE_S3=true")
+        return 1
+
+    print()
+    results = []
+
+    # Test all sizes
+    for size_name, num_rows in TEST_SIZES.items():
+        print(f"\n{'='*80}")
+        print(f"Testing {size_name} dataset ({num_rows:,} rows)")
+        print(f"{'='*80}")
+        
+        success, message, rows_read = test_write_read_with_sse(
+            s3, size_name, num_rows
+        )
+        results.append((size_name, num_rows, success, message, rows_read))
+        
+        if success:
+            print(f"  ✓ SUCCESS: Read {rows_read:,} rows")
+        else:
+            print(f"  ✗ FAILED: {message}")
+
+    # Summary
+    print("\n" + "=" * 80)
+    print("SUMMARY")
+    print("=" * 80)
+    
+    passed = sum(1 for _, _, success, _, _ in results if success)
+    total = len(results)
+    print(f"\nTotal: {passed}/{total} tests passed\n")
+    
+    print(f"{'Size':<15} {'Rows':>10} {'Status':<10} {'Rows Read':>10} {'Message':<40}")
+    print("-" * 90)
+    for size_name, num_rows, success, message, rows_read in results:
+        status = "✓ PASS" if success else "✗ FAIL"
+        rows_str = f"{rows_read:,}" if success else "N/A"
+        print(f"{size_name:<15} {num_rows:>10,} {status:<10} {rows_str:>10} {message[:40]}")
+
+    print("\n" + "=" * 80)
+    if passed == total:
+        print("✓ ALL TESTS PASSED WITH SSE-S3!")
+        print("\nThis means:")
+        print("  - SSE-S3 encryption is working correctly")
+        print("  - PyArrow native S3 filesystem is compatible")
+        print("  - Multipart uploads are handled properly")
+    else:
+        print(f"✗ {total - passed} test(s) failed")
+        print("\nPossible issues:")
+        print("  - SSE-S3 multipart upload bug with empty IV")
+        print("  - Encryption/decryption mismatch")
+        print("  - File corruption during upload")
+
+    print("=" * 80 + "\n")
+
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+