upstream-seaweedfs/weed/mount/page_writer/upload_pipeline_test.go

package page_writer

import (
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/util"
)

func TestUploadPipeline(t *testing.T) {

	uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16, "", nil)

	writeRange(uploadPipeline, 0, 131072)
	writeRange(uploadPipeline, 131072, 262144)
	writeRange(uploadPipeline, 262144, 1025536)

	confirmRange(t, uploadPipeline, 0, 1025536)

	writeRange(uploadPipeline, 1025536, 1296896)

	confirmRange(t, uploadPipeline, 1025536, 1296896)

	writeRange(uploadPipeline, 1296896, 2162688)

	confirmRange(t, uploadPipeline, 1296896, 2162688)

	confirmRange(t, uploadPipeline, 1296896, 2162688)
}

// startOff and stopOff must be divided by 4
func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
	p := make([]byte, 4)
	for i := startOff / 4; i < stopOff/4; i += 4 {
		util.Uint32toBytes(p, uint32(i))
		uploadPipeline.SaveDataAt(p, i, false, 0)
	}
}

func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
	p := make([]byte, 4)
	for i := startOff; i < stopOff/4; i += 4 {
		uploadPipeline.MaybeReadDataAt(p, i, 0)
		x := util.BytesToUint32(p)
		if x != uint32(i) {
			t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)
		}
	}
}

// Pressure-driven eviction must not seal a chunk with a leading or
// internal gap (issue #9330).
func TestEvictOneWritableChunk_SkipsGappyChunks(t *testing.T) {
	const cs int64 = 2 * 1024 * 1024
	// nil saveToStorage so the async upload SaveContent is a no-op.
	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)

	block := make([]byte, cs/4)

	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
		t.Fatal(err)
	}

	if !up.EvictOneWritableChunk() {
		t.Fatalf("EvictOneWritableChunk returned false; expected contiguous chunk 2 to be evictable")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
		t.Errorf("chunk 2 should have moved to sealed")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
		t.Errorf("chunk 0 (internal gap) must remain writable")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
		t.Errorf("chunk 1 (leading gap) must remain writable")
	}

	// Filling holes makes IsComplete true; maybeMoveToSealed auto-seals.
	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
		t.Fatal(err)
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
	}
}

// When every chunk is gappy, the fallback must still seal one so
// accountant.Reserve can wake; oldest-LastWriteTsNs wins.
func TestEvictOneWritableChunk_FallbackPicksOldestGappy(t *testing.T) {
	const cs int64 = 2 * 1024 * 1024
	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)

	block := make([]byte, cs/4)

	// Strictly increasing tsNs so chunk 0 is oldest; equal WrittenSize.
	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/4, true, 200); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/2, true, 201); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs, true, 300); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs+3*cs/4, true, 301); err != nil {
		t.Fatal(err)
	}

	if !up.EvictOneWritableChunk() {
		t.Fatalf("fallback must seal a gappy chunk to free a Reserve slot")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
		t.Errorf("oldest gappy chunk (0) should have been picked by the fallback")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
		t.Errorf("chunk 1 should remain writable")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; !stillWritable {
		t.Errorf("chunk 2 should remain writable")
	}
}

// Strict pass preempts the fallback even when a gappy chunk is older.
func TestEvictOneWritableChunk_PrefersStrictOverFallback(t *testing.T) {
	const cs int64 = 2 * 1024 * 1024
	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)

	block := make([]byte, cs/4)

	// chunk 0 (older, gappy), chunk 1 (newer, contiguous from 0).
	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs, true, 200); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/4, true, 201); err != nil {
		t.Fatal(err)
	}

	if !up.EvictOneWritableChunk() {
		t.Fatalf("EvictOneWritableChunk returned false")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
		t.Errorf("contiguous chunk 1 must be picked over older gappy chunk 0")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
		t.Errorf("gappy chunk 0 must remain writable")
	}
}

// ProactiveFlush also skips gappy chunks (issue #9330). No liveness
// fallback here — unlike EvictOneWritableChunk, returning false is just
// a missed optimization.
func TestProactiveFlush_SkipsGappyChunks(t *testing.T) {
	const cs int64 = 2 * 1024 * 1024
	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)

	block := make([]byte, cs/4)

	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
		t.Fatal(err)
	}

	// nowNs >> chunk timestamps so age clears the idle/maxHold gates;
	// fillRatio < WrittenSize so nearlyFull is true. Leaves
	// IsContiguouslyWritten as the only differentiator.
	const (
		nowNs           int64 = 1_000_000_000
		idleThresholdNs int64 = 100
		maxHoldNs       int64 = 200
		fillRatio       int64 = cs / 8
	)
	if !up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
		t.Fatalf("ProactiveFlush returned false; expected contiguous chunk 2 to be sealed")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
		t.Errorf("chunk 2 should have moved to sealed")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
		t.Errorf("chunk 0 (internal gap) must remain writable")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
		t.Errorf("chunk 1 (leading gap) must remain writable")
	}

	if up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
		t.Errorf("ProactiveFlush returned true with only gappy chunks left")
	}

	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
		t.Fatal(err)
	}
	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
		t.Fatal(err)
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
	}
	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
	}
}