snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

test/s3/remote_cache/command_remote_copy_local_test.go

@@ -0,0 +1,272 @@
+package remote_cache
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestRemoteCopyLocalBasic tests copying local-only files to remote
+func TestRemoteCopyLocalBasic(t *testing.T) {
+	checkServersRunning(t)
+
+	testKey := fmt.Sprintf("copylocal-basic-%d.txt", time.Now().UnixNano())
+	testData := createTestFile(t, testKey, 2048)
+
+	// File is now local-only (not on remote yet)
+	// Use remote.copy.local to copy it to remote
+	t.Log("Copying local file to remote...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local failed")
+	t.Logf("Copy output: %s", output)
+
+	// Verify file is still readable
+	verifyFileContent(t, testKey, testData)
+
+	// Now uncache and read again - should work if copied to remote
+	uncacheLocal(t, testKey)
+	time.Sleep(500 * time.Millisecond)
+	verifyFileContent(t, testKey, testData)
+}
+
+// TestRemoteCopyLocalDryRun tests preview mode without actual copying
+func TestRemoteCopyLocalDryRun(t *testing.T) {
+	checkServersRunning(t)
+
+	testKey := fmt.Sprintf("copylocal-dryrun-%d.txt", time.Now().UnixNano())
+	createTestFile(t, testKey, 1024)
+
+	// Run in dry-run mode
+	t.Log("Running remote.copy.local in dry-run mode...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -dryRun=true", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err)
+	t.Logf("Dry-run output: %s", output)
+
+	// Output should indicate what would be copied
+	assert.Contains(t, strings.ToLower(output), "dry", "dry-run output should mention dry run")
+}
+
+// TestRemoteCopyLocalWithInclude tests copying only matching files
+func TestRemoteCopyLocalWithInclude(t *testing.T) {
+	checkServersRunning(t)
+
+	// Create multiple files
+	pdfFile := fmt.Sprintf("copylocal-doc-%d.pdf", time.Now().UnixNano())
+	txtFile := fmt.Sprintf("copylocal-doc-%d.txt", time.Now().UnixNano())
+
+	pdfData := createTestFile(t, pdfFile, 1024)
+	txtData := createTestFile(t, txtFile, 1024)
+
+	// Copy only PDF files
+	t.Log("Copying only PDF files to remote...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -include=*.pdf", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local with include failed")
+	t.Logf("Copy output: %s", output)
+
+	// Verify both files are still readable
+	verifyFileContent(t, pdfFile, pdfData)
+	verifyFileContent(t, txtFile, txtData)
+}
+
+// TestRemoteCopyLocalWithExclude tests excluding pattern from copy
+func TestRemoteCopyLocalWithExclude(t *testing.T) {
+	checkServersRunning(t)
+
+	// Create test files
+	keepFile := fmt.Sprintf("copylocal-keep-%d.dat", time.Now().UnixNano())
+	tmpFile := fmt.Sprintf("copylocal-temp-%d.tmp", time.Now().UnixNano())
+
+	keepData := createTestFile(t, keepFile, 1024)
+	tmpData := createTestFile(t, tmpFile, 1024)
+
+	// Copy excluding .tmp files
+	t.Log("Copying excluding .tmp files...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -exclude=*.tmp", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local with exclude failed")
+	t.Logf("Copy output: %s", output)
+
+	// Both should still be readable
+	verifyFileContent(t, keepFile, keepData)
+	verifyFileContent(t, tmpFile, tmpData)
+}
+
+// TestRemoteCopyLocalForceUpdate tests overwriting existing remote files
+func TestRemoteCopyLocalForceUpdate(t *testing.T) {
+	checkServersRunning(t)
+
+	testKey := fmt.Sprintf("copylocal-force-%d.txt", time.Now().UnixNano())
+
+	// Create and copy file to remote
+	originalData := createTestFile(t, testKey, 1024)
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -include=%s", testBucket, testKey)
+	_, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "initial copy failed")
+
+	// Modify the file locally (simulate local change)
+	newData := []byte("Updated content for force update test")
+	uploadToPrimary(t, testKey, newData)
+	time.Sleep(500 * time.Millisecond)
+
+	// Copy again with force update
+	t.Log("Copying with force update...")
+	cmd = fmt.Sprintf("remote.copy.local -dir=/buckets/%s -include=%s -forceUpdate=true", testBucket, testKey)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local with forceUpdate failed")
+	t.Logf("Force update output: %s", output)
+
+	// Verify new content
+	verifyFileContent(t, testKey, newData)
+
+	// Uncache and verify it was updated on remote
+	uncacheLocal(t, testKey)
+	time.Sleep(500 * time.Millisecond)
+	verifyFileContent(t, testKey, newData)
+
+	// Clean up - restore original for other tests
+	uploadToPrimary(t, testKey, originalData)
+}
+
+// TestRemoteCopyLocalConcurrency tests parallel copy operations
+func TestRemoteCopyLocalConcurrency(t *testing.T) {
+	checkServersRunning(t)
+
+	// Create multiple files
+	var files []string
+	var dataMap = make(map[string][]byte)
+
+	for i := 0; i < 5; i++ {
+		key := fmt.Sprintf("copylocal-concurrent-%d-%d.bin", time.Now().UnixNano(), i)
+		data := createTestFile(t, key, 2048)
+		files = append(files, key)
+		dataMap[key] = data
+		time.Sleep(10 * time.Millisecond) // Small delay to ensure unique timestamps
+	}
+
+	// Copy with high concurrency
+	t.Log("Copying with concurrency=8...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -concurrent=8", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local with concurrency failed")
+	t.Logf("Copy output: %s", output)
+
+	// Verify all files are still readable
+	for _, file := range files {
+		verifyFileContent(t, file, dataMap[file])
+	}
+}
+
+// TestRemoteCopyLocalEmptyDirectory tests handling empty directories
+func TestRemoteCopyLocalEmptyDirectory(t *testing.T) {
+	checkServersRunning(t)
+
+	// Try to copy from a directory with no local-only files
+	// (all files are already synced to remote)
+	t.Log("Testing copy from directory with no local-only files...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+
+	// Should not error, just report nothing to copy
+	require.NoError(t, err, "remote.copy.local should handle empty directory gracefully")
+	t.Logf("Empty directory output: %s", output)
+}
+
+// TestRemoteCopyLocalLargeFile tests copying large files
+func TestRemoteCopyLocalLargeFile(t *testing.T) {
+	checkServersRunning(t)
+
+	testKey := fmt.Sprintf("copylocal-large-%d.bin", time.Now().UnixNano())
+
+	// Create a 10MB file
+	t.Log("Creating 10MB test file...")
+	testData := createTestFile(t, testKey, 10*1024*1024)
+
+	// Copy to remote
+	t.Log("Copying large file to remote...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -include=%s", testBucket, testKey)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local large file failed")
+	t.Logf("Large file copy output: %s", output)
+
+	// Verify file integrity
+	verifyFileContent(t, testKey, testData)
+
+	// Uncache and verify it was copied to remote
+	uncacheLocal(t, testKey)
+	time.Sleep(1 * time.Second)
+	verifyFileContent(t, testKey, testData)
+}
+
+// TestRemoteCopyLocalAlreadyExists tests skipping files already on remote
+func TestRemoteCopyLocalAlreadyExists(t *testing.T) {
+	checkServersRunning(t)
+
+	testKey := fmt.Sprintf("copylocal-exists-%d.txt", time.Now().UnixNano())
+	testData := createTestFile(t, testKey, 1024)
+
+	// First copy
+	t.Log("First copy to remote...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -include=%s", testBucket, testKey)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "first copy failed")
+	t.Logf("First copy output: %s", output)
+
+	// Second copy without forceUpdate - should skip
+	t.Log("Second copy (should skip)...")
+	output, err = runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "second copy failed")
+	t.Logf("Second copy output: %s", output)
+
+	// File should still be readable
+	verifyFileContent(t, testKey, testData)
+}
+
+// TestRemoteCopyLocalNotMounted tests error when directory not mounted
+func TestRemoteCopyLocalNotMounted(t *testing.T) {
+	checkServersRunning(t)
+
+	// Try to copy from a non-mounted directory
+	notMountedDir := fmt.Sprintf("/notmounted-%d", time.Now().UnixNano())
+
+	t.Log("Testing copy from non-mounted directory...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=%s", notMountedDir)
+	output, err := runWeedShellWithOutput(t, cmd)
+
+	// Should fail or show error
+	hasError := err != nil || strings.Contains(strings.ToLower(output), "not mounted") || strings.Contains(strings.ToLower(output), "error")
+	assert.True(t, hasError, "Expected error or error message for non-mounted directory, got: %s", output)
+	t.Logf("Non-mounted directory result: err=%v, output: %s", err, output)
+}
+
+// TestRemoteCopyLocalMinMaxSize tests size-based filtering
+func TestRemoteCopyLocalMinMaxSize(t *testing.T) {
+	checkServersRunning(t)
+
+	// Create files of different sizes
+	smallFile := fmt.Sprintf("copylocal-small-%d.bin", time.Now().UnixNano())
+	mediumFile := fmt.Sprintf("copylocal-medium-%d.bin", time.Now().UnixNano())
+	largeFile := fmt.Sprintf("copylocal-large-%d.bin", time.Now().UnixNano())
+
+	smallData := createTestFile(t, smallFile, 500)    // 500 bytes
+	mediumData := createTestFile(t, mediumFile, 5000) // 5KB
+	largeData := createTestFile(t, largeFile, 50000)  // 50KB
+
+	// Copy only files between 1KB and 10KB
+	t.Log("Copying files between 1KB and 10KB...")
+	cmd := fmt.Sprintf("remote.copy.local -dir=/buckets/%s -minSize=1024 -maxSize=10240", testBucket)
+	output, err := runWeedShellWithOutput(t, cmd)
+	require.NoError(t, err, "remote.copy.local with size filters failed")
+	t.Logf("Size filter output: %s", output)
+
+	// All files should still be readable
+	verifyFileContent(t, smallFile, smallData)
+	verifyFileContent(t, mediumFile, mediumData)
+	verifyFileContent(t, largeFile, largeData)
+}