snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/erasure_coding/ec_decoder.go

@@ -0,0 +1,352 @@
+package erasure_coding
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/idx"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// EcNoLiveEntriesSubstring is used for server/client coordination when ec.decode determines that
+// decoding should be a no-op (all entries are deleted).
+const EcNoLiveEntriesSubstring = "has no live entries"
+
+// HasLiveNeedles returns whether the EC index (.ecx) contains at least one live (non-deleted) entry.
+// This is used by ec.decode to avoid generating an empty normal volume when all entries were deleted.
+func HasLiveNeedles(indexBaseFileName string) (hasLive bool, err error) {
+	err = iterateEcxFile(indexBaseFileName, func(_ types.NeedleId, _ types.Offset, size types.Size) error {
+		if !size.IsDeleted() {
+			hasLive = true
+			return io.EOF // stop early
+		}
+		return nil
+	})
+	return
+}
+
+// write .idx file from .ecx and .ecj files
+func WriteIdxFileFromEcIndex(baseFileName string) (err error) {
+
+	ecxFile, openErr := os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecx: %v", baseFileName, openErr)
+	}
+	defer ecxFile.Close()
+
+	// Write to a temp file and atomically rename into place, so a crash mid-write
+	// never leaves a partial .idx at the final name beside the source shards.
+	idxFileName := baseFileName + ".idx"
+	tmpFileName := idxFileName + ".tmp"
+	idxFile, openErr := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open %s: %v", tmpFileName, openErr)
+	}
+	committed := false
+	defer func() {
+		idxFile.Close()
+		if !committed {
+			os.Remove(tmpFileName)
+		}
+	}()
+
+	if _, err = io.Copy(idxFile, ecxFile); err != nil {
+		return fmt.Errorf("copy ecx to idx for %s: %v", baseFileName, err)
+	}
+
+	err = iterateEcjFile(baseFileName, func(key types.NeedleId) error {
+		bytes := needle_map.ToBytes(key, types.Offset{}, types.TombstoneFileSize)
+		if _, writeErr := idxFile.Write(bytes); writeErr != nil {
+			return writeErr
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// fsync, rename, then fsync the dir so the decoded .idx is durable and
+	// atomically published before the caller deletes the source shards.
+	if err = idxFile.Sync(); err != nil {
+		return fmt.Errorf("sync idx for %s: %v", baseFileName, err)
+	}
+	if err = idxFile.Close(); err != nil {
+		return fmt.Errorf("close idx for %s: %v", baseFileName, err)
+	}
+	if err = os.Rename(tmpFileName, idxFileName); err != nil {
+		return fmt.Errorf("rename idx for %s: %v", baseFileName, err)
+	}
+	if err = util.FsyncDir(filepath.Dir(idxFileName)); err != nil {
+		return fmt.Errorf("fsync dir for %s: %v", baseFileName, err)
+	}
+	committed = true
+	return nil
+}
+
+// FindDatFileSize calculate .dat file size from max offset entry
+// there may be extra deletions after that entry
+// but they are deletions anyway
+// shard0FileName is the actual path of the .ec00 shard file, which on a
+// multi-disk server may sit on a different disk than the EcVolume's own base
+// path — the store registers shards per disk, so the caller must pass the
+// path CollectEcShards resolved rather than deriving it from a base name.
+func FindDatFileSize(shard0FileName, indexBaseFileName string) (datSize int64, err error) {
+
+	version, err := readEcVolumeVersion(shard0FileName)
+	if err != nil {
+		return 0, fmt.Errorf("read ec volume %s version: %v", shard0FileName, err)
+	}
+
+	// Safety: ensure datSize is at least SuperBlockSize. While the caller typically
+	// checks HasLiveNeedles first, this protects against direct calls to FindDatFileSize
+	// when all needles are deleted (see issue #7748).
+	datSize = int64(super_block.SuperBlockSize)
+
+	err = iterateEcxFile(indexBaseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error {
+
+		if size.IsDeleted() {
+			return nil
+		}
+
+		entryStopOffset := offset.ToActualOffset() + needle.GetActualSize(size, version)
+		if datSize < entryStopOffset {
+			datSize = entryStopOffset
+		}
+
+		return nil
+	})
+
+	return
+}
+
+// VerifyDecodedDatFile checks that a reconstructed .dat is long enough to hold
+// every needle its index references. datFileSize is the extent the EC index
+// describes (see FindDatFileSize), so a shorter file cannot serve the needles
+// past the cut -- and the caller is about to delete the shards that are their
+// only other copy, which turns a short write into data loss rather than a
+// failed decode.
+func VerifyDecodedDatFile(dataBaseFileName string, datFileSize int64) error {
+	datPath := dataBaseFileName + ".dat"
+	stat, err := os.Stat(datPath)
+	if err != nil {
+		return fmt.Errorf("stat decoded %s: %w", datPath, err)
+	}
+	if stat.Size() < datFileSize {
+		return fmt.Errorf("decoded %s is %d bytes, short of the %d its ec index references", datPath, stat.Size(), datFileSize)
+	}
+	return nil
+}
+
+func readEcVolumeVersion(shard0FileName string) (version needle.Version, err error) {
+
+	// find volume version
+	datFile, err := os.OpenFile(shard0FileName, os.O_RDONLY, 0644)
+	if err != nil {
+		return 0, fmt.Errorf("open ec volume %s superblock: %v", shard0FileName, err)
+	}
+	datBackend := backend.NewDiskFile(datFile)
+
+	superBlock, err := super_block.ReadSuperBlock(datBackend)
+	datBackend.Close()
+	if err != nil {
+		return 0, fmt.Errorf("read ec volume %s superblock: %v", shard0FileName, err)
+	}
+
+	return superBlock.Version, nil
+
+}
+
+func iterateEcxFile(baseFileName string, processNeedleFn func(key types.NeedleId, offset types.Offset, size types.Size) error) error {
+	ecxFile, openErr := os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecx: %v", baseFileName, openErr)
+	}
+	defer ecxFile.Close()
+
+	buf := make([]byte, types.NeedleMapEntrySize)
+	for {
+		// .ecx is a sealed index: a partial trailing record means corruption, not a torn append.
+		_, err := io.ReadFull(ecxFile, buf)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("read ecx %s.ecx: %w", baseFileName, err)
+		}
+		key, offset, size := idx.IdxFileEntry(buf)
+		if processNeedleFn != nil {
+			err = processNeedleFn(key, offset, size)
+		}
+		if err != nil {
+			if err != io.EOF {
+				return err
+			}
+			return nil
+		}
+	}
+
+}
+
+func iterateEcjFile(baseFileName string, processNeedleFn func(key types.NeedleId) error) error {
+	if !util.FileExists(baseFileName + ".ecj") {
+		return nil
+	}
+	ecjFile, openErr := os.OpenFile(baseFileName+".ecj", os.O_RDONLY, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot open ec index %s.ecj: %v", baseFileName, openErr)
+	}
+	defer ecjFile.Close()
+
+	buf := make([]byte, types.NeedleIdSize)
+	for {
+		n, err := ecjFile.Read(buf)
+		if n != types.NeedleIdSize {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		if processNeedleFn != nil {
+			err = processNeedleFn(types.BytesToNeedleId(buf))
+		}
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+	}
+
+}
+
+// WriteDatFile generates .dat from EC shard files (e.g., .ec00 ~ .ec09 for 10+4).
+// datFileSize is the number of bytes to write, i.e. the live data extent from
+// FindDatFileSize. encodedDatFileSize is the .dat size at encode time, which
+// fixed the shard block layout: deletions can move the live extent below the
+// large-block row boundary, and deriving the layout from the shrunk extent
+// would read the shards in the wrong block order. Pass zero when the .vif does
+// not record the encode-time size to infer the layout from the shard size.
+// largeBlockSize/smallBlockSize are the volume's shard block layout, e.g.
+// ctx.LargeBlockSize()/ctx.SmallBlockSize() from its .vif EC config.
+func WriteDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
+	return writeDatFile(baseFileName, datFileSize, encodedDatFileSize, shardFileNames, largeBlockSize, smallBlockSize)
+}
+
+func writeDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
+
+	if len(shardFileNames) == 0 {
+		return fmt.Errorf("no data shard files")
+	}
+
+	// Write to a temp file and atomically rename into place, so a crash mid-write
+	// never leaves a partial .dat at the final name beside the source shards.
+	datFileName := baseFileName + ".dat"
+	tmpFileName := datFileName + ".tmp"
+	datFile, openErr := os.OpenFile(tmpFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if openErr != nil {
+		return fmt.Errorf("cannot write volume %s: %v", tmpFileName, openErr)
+	}
+
+	// Use the actual number of data shards passed in rather than the global
+	// constant, so the de-striping matches the caller's shard set.
+	dataShards := len(shardFileNames)
+	inputFiles := make([]*os.File, dataShards)
+
+	committed := false
+	defer func() {
+		datFile.Close()
+		for shardId := 0; shardId < dataShards; shardId++ {
+			if inputFiles[shardId] != nil {
+				inputFiles[shardId].Close()
+			}
+		}
+		if !committed {
+			os.Remove(tmpFileName)
+		}
+	}()
+
+	for shardId := 0; shardId < dataShards; shardId++ {
+		inputFiles[shardId], openErr = os.OpenFile(shardFileNames[shardId], os.O_RDONLY, 0)
+		if openErr != nil {
+			return openErr
+		}
+	}
+
+	if encodedDatFileSize <= 0 {
+		// .vif without the encode-time size: infer the padded layout from the
+		// physical shard size, which reads the shards in the same block order.
+		shardFileInfo, statErr := inputFiles[0].Stat()
+		if statErr != nil {
+			return fmt.Errorf("stat %s: %v", shardFileNames[0], statErr)
+		}
+		shardSize := shardFileInfo.Size()
+		// A shard size that is an exact multiple of the large block size is
+		// ambiguous: N large rows, or N-1 large rows plus a full small-block
+		// region. The two layouts only agree below the last large row.
+		// A uniform layout has no such ambiguity — its large and small blocks
+		// are the same size, so every reading of the shard is the same one, and
+		// without this precondition the check fires on every volume.
+		if largeBlockSize != smallBlockSize &&
+			shardSize%largeBlockSize == 0 && datFileSize > (shardSize/largeBlockSize-1)*largeBlockSize*int64(dataShards) {
+			return fmt.Errorf("shard size %d of %s does not identify the block layout; re-encode to record the dat size in .vif", shardSize, baseFileName)
+		}
+		encodedDatFileSize = int64(dataShards) * shardSize
+	}
+	if datFileSize > encodedDatFileSize {
+		return fmt.Errorf("dat file size %d exceeds encoded dat file size %d", datFileSize, encodedDatFileSize)
+	}
+
+	for encodedDatFileSize >= int64(dataShards)*largeBlockSize && datFileSize > 0 {
+		for shardId := 0; shardId < dataShards && datFileSize > 0; shardId++ {
+			toRead := min(datFileSize, largeBlockSize)
+			w, err := io.CopyN(datFile, inputFiles[shardId], toRead)
+			if w != toRead {
+				return fmt.Errorf("copy %s large block on shardId %d: %v", baseFileName, shardId, err)
+			}
+			datFileSize -= toRead
+		}
+		encodedDatFileSize -= int64(dataShards) * largeBlockSize
+	}
+
+	for datFileSize > 0 {
+		for shardId := 0; shardId < dataShards && datFileSize > 0; shardId++ {
+			toRead := min(datFileSize, smallBlockSize)
+			w, err := io.CopyN(datFile, inputFiles[shardId], toRead)
+			if w != toRead {
+				return fmt.Errorf("copy %s small block %d: %v", baseFileName, shardId, err)
+			}
+			datFileSize -= toRead
+		}
+	}
+
+	// fsync, rename, then fsync the dir so the decoded .dat is durable and
+	// atomically published before the caller deletes the source shards.
+	if err := datFile.Sync(); err != nil {
+		return fmt.Errorf("sync dat for %s: %v", baseFileName, err)
+	}
+	if err := datFile.Close(); err != nil {
+		return fmt.Errorf("close dat for %s: %v", baseFileName, err)
+	}
+	if err := os.Rename(tmpFileName, datFileName); err != nil {
+		return fmt.Errorf("rename dat for %s: %v", baseFileName, err)
+	}
+	if err := util.FsyncDir(filepath.Dir(datFileName)); err != nil {
+		return fmt.Errorf("fsync dir for %s: %v", baseFileName, err)
+	}
+	committed = true
+	return nil
+}
+
+func min(x, y int64) int64 {
+	if x > y {
+		return y
+	}
+	return x
+}