upstream-seaweedfs/weed/storage/erasure_coding/ec_roundtrip_test.go

package erasure_coding

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"os"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

// TestEcReadRoundTrip tests the EC encode→read cycle via LocateData for various
// .dat file sizes, paying special attention to the large/small block boundary.
//
// The nLargeBlockRows calculation must correctly distinguish between large and small
// blocks. A previous bug (issue #8947) caused an off-by-one error when
// shardDatSize was an exact multiple of largeBlockSize, leading to data corruption.
func TestEcReadRoundTrip(t *testing.T) {
	const (
		large = largeBlockSize // 10000
		small = smallBlockSize // 100
	)

	largeRowSize := large * DataShardsCount // 100000
	smallRowSize := small * DataShardsCount // 1000

	testCases := []struct {
		name    string
		datSize int64
	}{
		// Exact multiples of largeRowSize — triggers the nLargeBlockRows off-by-one bug
		{"1_large_row_exact", int64(largeRowSize)},
		{"2_large_rows_exact", int64(2 * largeRowSize)},
		{"3_large_rows_exact", int64(3 * largeRowSize)},

		// Just over a large row boundary — has small blocks
		{"1_large_row_plus_1", int64(largeRowSize + 1)},
		{"2_large_rows_plus_small", int64(2*largeRowSize + smallRowSize)},
		{"1_large_row_plus_half_small", int64(largeRowSize + smallRowSize/2)},

		// Just under a large row boundary — all small blocks
		{"just_under_1_large_row", int64(largeRowSize - 1)},
		{"just_under_2_large_rows", int64(2*largeRowSize - 1)},

		// Small data — no large blocks at all
		{"small_only", int64(smallRowSize * 3)},
		{"small_single_row", int64(smallRowSize)},

		// Boundary with mixed large and small
		{"boundary_spanning", int64(largeRowSize + smallRowSize*5 + 50)},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			testEcRead(t, large, small, tc.datSize)
		})
	}
}

// testEcRead creates a .dat file, EC-encodes it, then verifies LocateData-based reads
// return correct data at positions throughout the file (especially near the large/small
// block boundary).
func testEcRead(t *testing.T, large, small, datSize int64) {
	t.Helper()

	dir := t.TempDir()
	baseFileName := fmt.Sprintf("%s/rt_%d", dir, datSize)

	// 1. Create a .dat file with deterministic random data
	originalData := make([]byte, datSize)
	_, err := rand.Read(originalData)
	require.NoError(t, err, "generating random data")

	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
	require.NoError(t, err, "writing .dat file")

	ctx := NewDefaultECContext("", 0)

	// 2. EC encode with test block sizes
	_, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
	require.NoError(t, err, "EC encoding")

	// 3. Open EC shard files for reading
	ecFiles, err := openEcFiles(baseFileName, true, ctx)
	require.NoError(t, err, "opening EC files")
	defer closeEcFiles(ecFiles)

	ecStat, err := ecFiles[0].Stat()
	require.NoError(t, err)
	shardFileSize := ecStat.Size()

	// Compute shardDatSize as the production code does when datFileSize is known
	shardDatSizeFromDat := datSize / int64(ctx.DataShards)

	// 4. Verify EC reads at various positions
	largeRowSize := large * DataShardsCount
	encoderLargeRows := datSize / int64(largeRowSize)
	boundaryOffset := encoderLargeRows * int64(largeRowSize)

	readSize := types.Size(small / 2) // read half a small block
	testOffsets := collectTestOffsets(datSize, int64(readSize), boundaryOffset, large, small)

	for _, offset := range testOffsets {
		// Test with shardDatSize from datFileSize (the production path with fix)
		intervals := LocateData(large, small, shardDatSizeFromDat, offset, readSize)
		ecData, err := assembleFromIntervals(ecFiles, intervals, large, small)
		require.NoError(t, err, "reading EC data at offset %d (datFileSize path)", offset)

		expected := originalData[offset : offset+int64(readSize)]
		if !bytes.Equal(expected, ecData) {
			t.Errorf("EC read mismatch at offset %d (datFileSize path, shardDatSize=%d, nLargeBlockRows=%d)",
				offset, shardDatSizeFromDat, shardDatSizeFromDat/large)
		}

		// Test with shardDatSize from ecdFileSize-1 (the fallback path for old volumes)
		intervalsFallback := LocateData(large, small, shardFileSize-1, offset, readSize)
		ecDataFallback, err := assembleFromIntervals(ecFiles, intervalsFallback, large, small)
		if err == nil && !bytes.Equal(expected, ecDataFallback) {
			// The fallback path may fail for exact multiples — log as warning
			t.Logf("WARN: EC read mismatch at offset %d (fallback path, shardFileSize=%d)",
				offset, shardFileSize)
		}
	}
}

// locateOffsetBuggy reimplements locateOffset with the old buggy formula:
//
//	nLargeBlockRows = (shardDatSize - 1) / largeBlockLength
//
// This caused an off-by-one error when shardDatSize was an exact multiple of
// largeBlockLength, miscounting the number of large block rows.
func locateOffsetBuggy(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
	largeRowSize := largeBlockLength * DataShardsCount
	nLargeBlockRows = (shardDatSize - 1) / largeBlockLength // THE BUG

	if offset < nLargeBlockRows*largeRowSize {
		isLargeBlock = true
		blockIndex = int(offset / largeBlockLength)
		innerBlockOffset = offset % largeBlockLength
		return
	}

	isLargeBlock = false
	offset -= nLargeBlockRows * largeRowSize
	blockIndex = int(offset / smallBlockLength)
	innerBlockOffset = offset % smallBlockLength
	return
}

// locateDataBuggy is LocateData using the old buggy locateOffset.
func locateDataBuggy(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) []Interval {
	blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffsetBuggy(largeBlockLength, smallBlockLength, shardDatSize, offset)

	var intervals []Interval
	for size > 0 {
		blockRemaining := largeBlockLength - innerBlockOffset
		if !isLargeBlock {
			blockRemaining = smallBlockLength - innerBlockOffset
		}
		if blockRemaining <= 0 {
			blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
			innerBlockOffset = 0
			continue
		}
		interval := Interval{
			BlockIndex:          blockIndex,
			InnerBlockOffset:    innerBlockOffset,
			IsLargeBlock:        isLargeBlock,
			LargeBlockRowsCount: int(nLargeBlockRows),
		}
		if int64(size) <= blockRemaining {
			interval.Size = size
			intervals = append(intervals, interval)
			return intervals
		}
		interval.Size = types.Size(blockRemaining)
		intervals = append(intervals, interval)
		size -= interval.Size
		blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
		innerBlockOffset = 0
	}
	return intervals
}

// TestEcOffByOneBug_Issue8947 directly demonstrates the off-by-one bug.
//
// It creates a .dat file whose size is an exact multiple of (largeBlockSize * DataShards),
// EC-encodes it, then shows that:
//   - The OLD buggy formula produces WRONG data (data corruption)
//   - The FIXED formula produces CORRECT data
func TestEcOffByOneBug_Issue8947(t *testing.T) {
	const (
		large = largeBlockSize // 10000
		small = smallBlockSize // 100
	)

	// datSize is exactly 2 large rows — each shard gets exactly 2*largeBlockSize bytes.
	// The encoder produces 2 large block rows and 0 small block rows.
	datSize := int64(2 * large * DataShardsCount) // 200000

	dir := t.TempDir()
	baseFileName := fmt.Sprintf("%s/bug_%d", dir, datSize)

	originalData := make([]byte, datSize)
	_, err := rand.Read(originalData)
	require.NoError(t, err)

	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
	require.NoError(t, err)

	ctx := NewDefaultECContext("", 0)
	_, err = generateEcFiles(baseFileName, int(small), large, small, ctx)
	require.NoError(t, err, "EC encoding")

	ecFiles, err := openEcFiles(baseFileName, true, ctx)
	require.NoError(t, err)
	defer closeEcFiles(ecFiles)

	// shardDatSize = datFileSize / DataShards = 2 * largeBlockSize
	// This is an EXACT multiple of largeBlockSize.
	shardDatSize := datSize / int64(ctx.DataShards) // = 2 * large = 20000

	// The encoder used 2 large block rows, 0 small block rows.
	// Correct: nLargeBlockRows = 20000 / 10000 = 2
	// Buggy:   nLargeBlockRows = (20000 - 1) / 10000 = 1  ← OFF BY ONE
	fixedRows := shardDatSize / large
	buggyRows := (shardDatSize - 1) / large
	assert.Equal(t, int64(2), fixedRows, "fixed formula should give 2 large block rows")
	assert.Equal(t, int64(1), buggyRows, "buggy formula gives only 1 (the bug)")

	// Test reading from the 2nd large block row (offsets 100000–199999).
	// With the buggy formula (nLargeBlockRows=1), this region is incorrectly
	// treated as small blocks, causing reads from the WRONG shard positions.
	readSize := types.Size(small / 2)

	// Pick an offset well into the 2nd large block row so that the buggy formula
	// computes a different (shard, offset) than the correct formula.
	// At the very start of the 2nd row, both formulas coincidentally hit the same
	// shard position. But further in, the small-block vs large-block addressing diverges.
	offset := int64(large*DataShardsCount) + large + 50 // 110050: in 2nd large row, shard 1

	// --- Fixed formula: reads correct data ---
	fixedIntervals := LocateData(large, small, shardDatSize, offset, readSize)
	fixedData, err := assembleFromIntervals(ecFiles, fixedIntervals, large, small)
	require.NoError(t, err, "fixed LocateData read")

	expected := originalData[offset : offset+int64(readSize)]
	assert.True(t, bytes.Equal(expected, fixedData),
		"FIXED formula should read correct data from 2nd large block row")

	// --- Buggy formula: reads WRONG data ---
	buggyIntervals := locateDataBuggy(large, small, shardDatSize, offset, readSize)
	buggyData, err := assembleFromIntervals(ecFiles, buggyIntervals, large, small)
	// The buggy formula might read from wrong offsets (possibly out of bounds),
	// so an error is also evidence of the bug.
	if err != nil {
		t.Logf("Buggy formula caused read error (expected): %v", err)
	} else {
		assert.False(t, bytes.Equal(expected, buggyData),
			"BUGGY formula should return WRONG data from 2nd large block row (demonstrating the corruption)")
		n := 8
		if len(expected) < n {
			n = len(expected)
		}
		t.Logf("Buggy formula returned wrong data: expected first bytes %x, got %x",
			expected[:n], buggyData[:n])
	}

	// Verify the bug mechanism: buggy formula misclassifies the 2nd large row as small blocks
	assert.True(t, fixedIntervals[0].IsLargeBlock,
		"fixed: offset %d should be in large blocks", offset)
	assert.False(t, buggyIntervals[0].IsLargeBlock,
		"buggy: offset %d is incorrectly classified as small blocks (the bug)", offset)

	t.Logf("Fixed: nLargeBlockRows=%d, interval=%+v", fixedRows, fixedIntervals[0])
	t.Logf("Buggy: nLargeBlockRows=%d, interval=%+v", buggyRows, buggyIntervals[0])
}

// TestEcDecodeDatRoundTrip tests the full WriteDatFile decode path using the production
// block sizes with a small .dat file that fits within the small block region.
func TestEcDecodeDatRoundTrip(t *testing.T) {
	// With production sizes, datFileSize must be < DataShardsCount * ErasureCodingLargeBlockSize (10GB)
	// to avoid needing huge test files. We test small block decode only.
	// Each shard gets datSize/DataShards bytes in small 1MB blocks.
	datSizes := []int64{
		1000, // tiny
		int64(DataShardsCount) * ErasureCodingSmallBlockSize,     // exactly 1 small row (10MB)
		int64(DataShardsCount)*ErasureCodingSmallBlockSize + 500, // 1 small row + partial
	}

	for _, datSize := range datSizes {
		t.Run(fmt.Sprintf("size_%d", datSize), func(t *testing.T) {
			testDecodeDat(t, datSize)
		})
	}
}

func testDecodeDat(t *testing.T, datSize int64) {
	t.Helper()

	dir := t.TempDir()
	baseFileName := fmt.Sprintf("%s/dec_%d", dir, datSize)

	// 1. Create .dat with random data
	originalData := make([]byte, datSize)
	_, err := rand.Read(originalData)
	require.NoError(t, err)

	err = os.WriteFile(baseFileName+".dat", originalData, 0644)
	require.NoError(t, err)

	ctx := NewDefaultECContext("", 0)

	// 2. EC encode with PRODUCTION block sizes
	_, err = generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
	require.NoError(t, err, "EC encoding")

	// 3. Decode via WriteDatFile
	decodedBase := baseFileName + "_decoded"
	shardFileNames := make([]string, DataShardsCount)
	for i := 0; i < DataShardsCount; i++ {
		shardFileNames[i] = fmt.Sprintf("%s%s", baseFileName, ctx.ToExt(i))
	}

	err = WriteDatFile(decodedBase, datSize, datSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
	require.NoError(t, err, "WriteDatFile")

	// The atomic publish must rename the temp file away, never leaving it behind.
	_, statErr := os.Stat(decodedBase + ".dat.tmp")
	require.True(t, os.IsNotExist(statErr), "WriteDatFile must not leave a .dat.tmp")

	// 4. Verify decoded .dat matches original
	decodedData, err := os.ReadFile(decodedBase + ".dat")
	require.NoError(t, err)

	assert.Equal(t, len(originalData), len(decodedData), "decoded .dat size mismatch")
	if !bytes.Equal(originalData, decodedData) {
		for i := 0; i < len(originalData) && i < len(decodedData); i++ {
			if originalData[i] != decodedData[i] {
				t.Fatalf("decoded .dat mismatch at byte %d (datSize=%d)", i, datSize)
			}
		}
	}
}

// collectTestOffsets generates offsets to test, focusing on the large/small block boundary.
func collectTestOffsets(datSize, readSize, boundaryOffset, large, small int64) []int64 {
	offsets := []int64{0}

	if datSize > readSize {
		offsets = append(offsets, datSize/2)
	}

	// Near the large/small block boundary
	if boundaryOffset > 0 && boundaryOffset < datSize {
		for _, delta := range []int64{-large, -small, -1, 0, 1, small, large} {
			off := boundaryOffset + delta
			if off >= 0 && off+readSize <= datSize {
				offsets = append(offsets, off)
			}
		}
	}

	// Near end of file
	if datSize > readSize {
		offsets = append(offsets, datSize-readSize)
	}

	return offsets
}

// assembleFromIntervals reads data from EC shard files according to the given intervals.
func assembleFromIntervals(ecFiles []*os.File, intervals []Interval, large, small int64) ([]byte, error) {
	var data []byte
	for _, interval := range intervals {
		shardId, shardOffset := interval.ToShardIdAndOffset(large, small)
		chunk := make([]byte, interval.Size)
		n, err := ecFiles[shardId].ReadAt(chunk, shardOffset)
		if err != nil {
			return nil, fmt.Errorf("read shard %d offset %d size %d: %v", shardId, shardOffset, interval.Size, err)
		}
		if n != int(interval.Size) {
			return nil, fmt.Errorf("short read from shard %d: got %d, want %d", shardId, n, interval.Size)
		}
		data = append(data, chunk...)
	}
	return data, nil
}

// TestWriteDatFileAfterTailDeletion decodes after deletions moved the live
// extent below the large-block row boundary. The shard block layout is fixed
// by the encode-time .dat size, so de-striping must not derive it from the
// shrunk live extent.
func TestWriteDatFileAfterTailDeletion(t *testing.T) {
	const (
		large = int64(largeBlockSize) // 10000
		small = int64(smallBlockSize) // 100
	)
	largeRowSize := large * DataShardsCount
	smallRowSize := small * DataShardsCount

	// one full large-block row plus a small-block tail
	datSize := largeRowSize + 2*smallRowSize + 530

	dir := t.TempDir()
	baseFileName := fmt.Sprintf("%s/tail_del", dir)

	originalData := make([]byte, datSize)
	_, err := rand.Read(originalData)
	require.NoError(t, err)
	require.NoError(t, os.WriteFile(baseFileName+".dat", originalData, 0644))

	ctx := NewDefaultECContext("", 0)
	_, err = generateEcFiles(baseFileName, 50, large, small, ctx)
	require.NoError(t, err, "EC encoding")

	shardFileNames := make([]string, DataShardsCount)
	for i := 0; i < DataShardsCount; i++ {
		shardFileNames[i] = baseFileName + ctx.ToExt(i)
	}
	shardFileInfo, err := os.Stat(shardFileNames[0])
	require.NoError(t, err)
	paddedSize := int64(DataShardsCount) * shardFileInfo.Size()

	decodedBase := baseFileName + "_decoded"
	decode := func(liveSize, encodedSize int64) []byte {
		require.NoError(t, writeDatFile(decodedBase, liveSize, encodedSize, shardFileNames, large, small))
		decoded, readErr := os.ReadFile(decodedBase + ".dat")
		require.NoError(t, readErr)
		require.NoError(t, os.Remove(decodedBase+".dat"))
		return decoded
	}

	liveSizes := []int64{
		large - 1,              // within the first large block
		large + 42,             // partial second large block
		largeRowSize / 2,       // mid large row
		largeRowSize,           // exactly the large row
		largeRowSize + 5*small, // into the small-block tail
		datSize,                // nothing deleted
	}
	for _, liveSize := range liveSizes {
		assert.Equal(t, originalData[:liveSize], decode(liveSize, datSize), "live size %d, encode-time layout", liveSize)
		assert.Equal(t, originalData[:liveSize], decode(liveSize, paddedSize), "live size %d, padded layout", liveSize)
	}

	// deriving the layout from the shrunk live extent reorders the data
	assert.NotEqual(t, originalData[:largeRowSize/2], decode(largeRowSize/2, largeRowSize/2))

	// the live extent can never exceed the encode-time size
	require.Error(t, writeDatFile(decodedBase, datSize+1, datSize, shardFileNames, large, small))
}

// TestWriteDatFileFallbackLayout covers decoding when .vif does not record the
// encode-time size: the layout is inferred from the shard size, except when
// that is an exact large-block multiple and the live extent reaches the
// ambiguous region.
func TestWriteDatFileFallbackLayout(t *testing.T) {
	const (
		large = int64(largeBlockSize) // 10000
		small = int64(smallBlockSize) // 100
	)
	largeRowSize := large * DataShardsCount

	dir := t.TempDir()

	encode := func(name string, datSize int64) (string, []string, []byte) {
		baseFileName := fmt.Sprintf("%s/%s", dir, name)
		originalData := make([]byte, datSize)
		_, err := rand.Read(originalData)
		require.NoError(t, err)
		require.NoError(t, os.WriteFile(baseFileName+".dat", originalData, 0644))
		ctx := NewDefaultECContext("", 0)
		_, err = generateEcFiles(baseFileName, 50, large, small, ctx)
		require.NoError(t, err, "EC encoding")
		shardFileNames := make([]string, DataShardsCount)
		for i := 0; i < DataShardsCount; i++ {
			shardFileNames[i] = baseFileName + ctx.ToExt(i)
		}
		return baseFileName, shardFileNames, originalData
	}

	// a small-block tail that is not a large-block multiple is unambiguous
	base, shards, original := encode("plain", largeRowSize+2530)
	liveSize := largeRowSize / 2
	require.NoError(t, writeDatFile(base+"_d", liveSize, 0, shards, large, small))
	decoded, err := os.ReadFile(base + "_d.dat")
	require.NoError(t, err)
	assert.Equal(t, original[:liveSize], decoded)

	// datSize just under one large row: the full small-block region makes each
	// shard exactly largeBlockSize, indistinguishable from one large row
	base, shards, _ = encode("ambiguous1", largeRowSize-1)
	err = writeDatFile(base+"_d", largeRowSize/2, 0, shards, large, small)
	require.ErrorContains(t, err, "does not identify the block layout")

	// two-row equivalent: decoding within the agreed prefix still works
	base, shards, original = encode("ambiguous2", 2*largeRowSize-1)
	require.NoError(t, writeDatFile(base+"_d", largeRowSize, 0, shards, large, small))
	decoded, err = os.ReadFile(base + "_d.dat")
	require.NoError(t, err)
	assert.Equal(t, original[:largeRowSize], decoded)
	err = writeDatFile(base+"_d2", largeRowSize+1, 0, shards, large, small)
	require.ErrorContains(t, err, "does not identify the block layout")
}