upstream-seaweedfs/weed/storage/erasure_coding/ec_volume_counts_test.go

package erasure_coding_test

import (
	"fmt"
	"os"
	"path/filepath"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

// makeEntry builds one .ecx index entry (needle id, offset, size).
func makeEntry(key types.NeedleId, offset types.Offset, size types.Size) []byte {
	b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
	types.NeedleIdToBytes(b[0:types.NeedleIdSize], key)
	types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], offset)
	types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:], size)
	return b
}

// encodeEcjIds packs a slice of needle ids into the binary .ecj on-disk format.
func encodeEcjIds(ids []types.NeedleId) []byte {
	buf := make([]byte, 0, len(ids)*types.NeedleIdSize)
	for _, id := range ids {
		b := make([]byte, types.NeedleIdSize)
		types.NeedleIdToBytes(b, id)
		buf = append(buf, b...)
	}
	return buf
}

func writeFixture(t *testing.T, dir, collection string, vid int, ecxData []byte, ecjIds []types.NeedleId) *erasure_coding.EcVolume {
	t.Helper()
	base := filepath.Join(dir, fmt.Sprintf("%s_%d", collection, vid))

	if err := os.WriteFile(base+".ecx", ecxData, 0644); err != nil {
		t.Fatalf("write ecx: %v", err)
	}
	if err := os.WriteFile(base+".ecj", encodeEcjIds(ecjIds), 0644); err != nil {
		t.Fatalf("write ecj: %v", err)
	}
	if err := os.WriteFile(base+".ec00", make([]byte, 8), 0644); err != nil {
		t.Fatalf("write ec00: %v", err)
	}
	if err := os.WriteFile(base+".vif", []byte{}, 0644); err != nil {
		t.Fatalf("write vif: %v", err)
	}

	ev, err := erasure_coding.NewEcVolume("hdd", dir, dir, collection, needle.VolumeId(vid))
	if err != nil {
		t.Fatalf("NewEcVolume: %v", err)
	}
	return ev
}

// TestEcVolumeFileAndDeleteCountInitial verifies that FileAndDeleteCount is
// derived from file sizes at load: fileCount = .ecx size / NeedleMapEntrySize
// and deleteCount = .ecj size / NeedleIdSize. No index walk is performed.
func TestEcVolumeFileAndDeleteCountInitial(t *testing.T) {
	dir := t.TempDir()

	// 3 needles in .ecx, 2 deletions already recorded in .ecj.
	ecx := []byte{}
	ecx = append(ecx, makeEntry(1, types.ToOffset(64), 100)...)
	ecx = append(ecx, makeEntry(2, types.ToOffset(128), 200)...)
	ecx = append(ecx, makeEntry(3, types.ToOffset(256), 300)...)
	ecj := []types.NeedleId{2, 3}

	ev := writeFixture(t, dir, "test", 1, ecx, ecj)
	defer ev.Close()

	fileCount, deleteCount := ev.FileAndDeleteCount()
	if fileCount != 3 {
		t.Errorf("fileCount: got %d, want 3 (.ecx entries)", fileCount)
	}
	if deleteCount != 2 {
		t.Errorf("deleteCount: got %d, want 2 (.ecj entries)", deleteCount)
	}
}

// TestEcVolumeFileAndDeleteCountAfterDelete verifies that DeleteNeedleFromEcx
// increments the derived delete count by appending to .ecj, while leaving
// fileCount (derived from the sealed .ecx) unchanged. Idempotent re-deletes
// and deletes of missing needles must not drift the count.
func TestEcVolumeFileAndDeleteCountAfterDelete(t *testing.T) {
	dir := t.TempDir()

	ecx := []byte{}
	ecx = append(ecx, makeEntry(1, types.ToOffset(64), 100)...)
	ecx = append(ecx, makeEntry(2, types.ToOffset(128), 200)...)

	ev := writeFixture(t, dir, "test", 1, ecx, nil)
	defer ev.Close()

	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 0 {
		t.Fatalf("initial: got (%d, %d), want (2, 0)", fc, dc)
	}

	if err := ev.DeleteNeedleFromEcx(2); err != nil {
		t.Fatalf("DeleteNeedleFromEcx: %v", err)
	}
	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
		t.Errorf("after first delete: got (%d, %d), want (2, 1)", fc, dc)
	}

	// Re-deleting an already tombstoned needle is a no-op on .ecj, so
	// deleteCount must stay at 1.
	if err := ev.DeleteNeedleFromEcx(2); err != nil {
		t.Fatalf("idempotent DeleteNeedleFromEcx: %v", err)
	}
	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
		t.Errorf("after idempotent delete: got (%d, %d), want (2, 1)", fc, dc)
	}

	// Deleting a non-existent needle is a no-op: search returns NotFound,
	// no .ecj append.
	if err := ev.DeleteNeedleFromEcx(99); err != nil {
		t.Fatalf("missing DeleteNeedleFromEcx: %v", err)
	}
	if fc, dc := ev.FileAndDeleteCount(); fc != 2 || dc != 1 {
		t.Errorf("after missing delete: got (%d, %d), want (2, 1)", fc, dc)
	}
}