snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/volume.go

@@ -0,0 +1,531 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+type Volume struct {
+	Id                 needle.VolumeId
+	dir                string
+	dirIdx             string
+	Collection         string
+	DataBackend        backend.BackendStorageFile
+	nm                 NeedleMapper
+	tmpNm              TempNeedleMapper
+	needleMapKind      NeedleMapKind
+	noWriteOrDelete    bool // if readonly, either noWriteOrDelete or noWriteCanDelete
+	noWriteCanDelete   bool // if readonly, either noWriteOrDelete or noWriteCanDelete
+	noWriteLock        sync.RWMutex
+	hasRemoteFile      atomic.Bool // if the volume is tiered: data lives in a remote backend
+	MemoryMapMaxSizeMb uint32
+
+	super_block.SuperBlock
+
+	dataFileAccessLock   sync.RWMutex
+	superBlockAccessLock sync.Mutex
+
+	// The batch worker exists only once the volume takes a durable write. Most
+	// never do -- read-only, remote-tiered, or written without fsync -- and a
+	// parked worker costs its goroutine stack plus a 128-slot channel, which a
+	// server holding millions of volumes cannot pay for all of them.
+	asyncWorkerLock   sync.Mutex
+	asyncRequestsChan chan *needle.AsyncRequest
+	asyncWorkerClosed bool
+
+	lastModifiedTsSeconds uint64 // unix time in seconds
+	lastAppendAtNs        uint64 // unix time in nanoseconds
+
+	lastCompactIndexOffset uint64
+	lastCompactRevision    uint16
+	ldbTimeout             int64
+
+	isCompactionInProgress atomic.Bool
+	lastDiskCheckNs        atomic.Int64 // unix time in nanoseconds for phantom volume detection
+
+	volumeInfoRWLock sync.RWMutex
+	volumeInfo       *volume_server_pb.VolumeInfo
+	location         *DiskLocation
+	diskId           uint32 // ID of this volume's disk in Store.Locations array
+
+	IoErrorTracker
+}
+
+func NewVolume(dirname string, dirIdx string, collection string, id needle.VolumeId, needleMapKind NeedleMapKind, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, ver needle.Version, memoryMapMaxSizeMb uint32, ldbTimeout int64) (v *Volume, e error) {
+	// if replicaPlacement is nil, the superblock will be loaded from disk
+	v = &Volume{dir: dirname, dirIdx: dirIdx, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb}
+	v.SuperBlock = super_block.SuperBlock{ReplicaPlacement: replicaPlacement, Ttl: ttl}
+	v.needleMapKind = needleMapKind
+	v.ldbTimeout = ldbTimeout
+	e = v.load(true, true, needleMapKind, preallocate, ver)
+	return
+}
+
+func (v *Volume) String() string {
+	v.noWriteLock.RLock()
+	defer v.noWriteLock.RUnlock()
+	return fmt.Sprintf("Id:%v dir:%s dirIdx:%s Collection:%s dataFile:%v nm:%v noWrite:%v canDelete:%v", v.Id, v.dir, v.dirIdx, v.Collection, v.DataBackend, v.nm, v.noWriteOrDelete || v.noWriteCanDelete, v.noWriteCanDelete)
+}
+
+func VolumeFileName(dir string, collection string, id int) (fileName string) {
+	idString := strconv.Itoa(id)
+	if collection == "" {
+		fileName = path.Join(dir, idString)
+	} else {
+		fileName = path.Join(dir, collection+"_"+idString)
+	}
+	return
+}
+
+func (v *Volume) DataFileName() (fileName string) {
+	return VolumeFileName(v.dir, v.Collection, int(v.Id))
+}
+
+func (v *Volume) IndexFileName() (fileName string) {
+	return VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
+}
+
+func (v *Volume) FileName(ext string) (fileName string) {
+	switch ext {
+	case ".idx", ".cpx", ".ldb", ".cpldb", ".rdb":
+		return VolumeFileName(v.dirIdx, v.Collection, int(v.Id)) + ext
+	}
+	// .dat, .cpd, .vif
+	return VolumeFileName(v.dir, v.Collection, int(v.Id)) + ext
+}
+
+// RelocateIndexTo moves the volume's index to newIdxDir and reopens the volume
+// against it in place, without unmounting. It takes the data-file write lock —
+// so a concurrent read blocks briefly instead of failing — closes the needle
+// map and data backend, moves the .idx (and the derived .sdx best-effort), then
+// retargets dirIdx and reloads, mirroring CommitCompact's close-swap-load. A
+// decode co-locates the rebuilt index with the data so the on-demand mount can
+// find the volume; this returns it to the -dir.idx tier once the EC shards are
+// gone. A no-op when the index already lives in newIdxDir. A derived .ldb is
+// not moved: the reload rebuilds it in newIdxDir from the .idx.
+func (v *Volume) RelocateIndexTo(newIdxDir string) error {
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+
+	if v.dirIdx == newIdxDir {
+		return nil
+	}
+	oldBase := VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
+	if _, err := os.Stat(oldBase + ".idx"); err != nil {
+		return nil // nothing co-located to move
+	}
+	newBase := VolumeFileName(newIdxDir, v.Collection, int(v.Id))
+
+	if v.nm != nil {
+		_ = v.nm.Sync()
+		v.nm.Close()
+		v.nm = nil
+	}
+	if v.DataBackend != nil {
+		_ = v.DataBackend.Sync()
+		_ = v.DataBackend.Close()
+		v.DataBackend = nil
+	}
+
+	if err := RenameOrCopyFile(oldBase+".idx", newBase+".idx"); err != nil {
+		// Reopen against the old dir so the volume is not left down; surface a
+		// failed reopen since it leaves the volume unusable until the next load.
+		if reopenErr := v.load(true, false, v.needleMapKind, 0, v.Version()); reopenErr != nil {
+			glog.Errorf("relocate volume %d: reopen after failed .idx move: %v", v.Id, reopenErr)
+		}
+		return fmt.Errorf("relocate index for volume %d: move .idx: %w", v.Id, err)
+	}
+	// The .sdx is a derived sorted index; move it when present, but a failure is
+	// not fatal — drop the stale copy so the reload rebuilds it in the new dir.
+	if _, err := os.Stat(oldBase + ".sdx"); err == nil {
+		if err := RenameOrCopyFile(oldBase+".sdx", newBase+".sdx"); err != nil {
+			glog.Warningf("relocate volume %d: move .sdx: %v (will rebuild)", v.Id, err)
+			_ = os.Remove(oldBase + ".sdx")
+		}
+	}
+	v.dirIdx = newIdxDir
+	return v.load(true, false, v.needleMapKind, 0, v.Version())
+}
+
+func (v *Volume) Version() needle.Version {
+	v.superBlockAccessLock.Lock()
+	defer v.superBlockAccessLock.Unlock()
+	if v.volumeInfo.Version != 0 {
+		v.SuperBlock.Version = needle.Version(v.volumeInfo.Version)
+	}
+	return v.SuperBlock.Version
+}
+
+func (v *Volume) FileStat() (datSize uint64, idxSize uint64, modTime time.Time) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+
+	if v.DataBackend == nil {
+		return
+	}
+
+	datFileSize, modTime, e := v.DataBackend.GetStat()
+	if e == nil {
+		return uint64(datFileSize), v.nm.IndexFileSize(), modTime
+	}
+	glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
+	return // -1 causes integer overflow and the volume to become unwritable.
+}
+
+func (v *Volume) ContentSize() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return v.nm.ContentSize()
+}
+
+func (v *Volume) doIsEmpty() (bool, error) {
+	// check v.DataBackend.GetStat()
+	if v.DataBackend == nil {
+		return false, fmt.Errorf("v.DataBackend is nil")
+	} else {
+		datFileSize, _, e := v.DataBackend.GetStat()
+		if e != nil {
+			glog.V(0).Infof("Failed to read file size %s %v", v.DataBackend.Name(), e)
+			return false, fmt.Errorf("v.DataBackend.GetStat(): %v", e)
+		}
+		if datFileSize > super_block.SuperBlockSize {
+			return false, nil
+		}
+	}
+	// check v.nm.ContentSize()
+	if v.nm != nil {
+		if v.nm.ContentSize() > 0 {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func (v *Volume) DeletedSize() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return v.nm.DeletedSize()
+}
+
+func (v *Volume) FileCount() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return uint64(v.nm.FileCount())
+}
+
+func (v *Volume) DeletedCount() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return uint64(v.nm.DeletedCount())
+}
+
+func (v *Volume) MaxFileKey() types.NeedleId {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return v.nm.MaxFileKey()
+}
+
+func (v *Volume) IndexFileSize() uint64 {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+	if v.nm == nil {
+		return 0
+	}
+	return v.nm.IndexFileSize()
+}
+
+func (v *Volume) DiskType() types.DiskType {
+	return v.location.DiskType
+}
+
+func (v *Volume) SyncToDisk() {
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+	if v.nm != nil {
+		if err := v.nm.Sync(); err != nil {
+			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
+		}
+	}
+	if v.DataBackend != nil {
+		if err := v.DataBackend.Sync(); err != nil {
+			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
+		}
+	}
+}
+
+// Close cleanly shuts down this volume
+func (v *Volume) Close() {
+	// Wait for any in-progress compaction to finish and claim the flag so no
+	// new compaction can start. This must happen BEFORE acquiring
+	// dataFileAccessLock to avoid deadlocking with CommitCompact which holds
+	// the flag while waiting for the lock.
+	for !v.isCompactionInProgress.CompareAndSwap(false, true) {
+		time.Sleep(521 * time.Millisecond)
+		glog.Warningf("Volume Close wait for compaction %d", v.Id)
+	}
+	defer v.isCompactionInProgress.Store(false)
+
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+
+	v.doClose()
+}
+
+// SwapDataBackend atomically replaces the data backend and updates the
+// remote-tier flag under dataFileAccessLock, closing the old backend. Both tier
+// directions go through here so hasRemoteFile always matches the live backend:
+// tier-down passes hasRemoteFile=false (now serving a local .dat), tier-up
+// passes true. Keeping the swap and the flag under one lock means the heartbeat
+// never observes a half-swapped backend or a flag that disagrees with it — a
+// stale-false flag would make doDeleteRequest skip the new .dat's tombstones and
+// disable the phantom-.dat guard.
+func (v *Volume) SwapDataBackend(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+	v.swapDataBackendLocked(newBackend, hasRemoteFile)
+}
+
+// swapDataBackendLocked is the body of SwapDataBackend for callers that already
+// hold dataFileAccessLock (e.g. load() reached while CommitCompact holds the
+// lock). Reusing it from those under-lock paths avoids re-entering the
+// non-reentrant lock, which would deadlock.
+func (v *Volume) swapDataBackendLocked(newBackend backend.BackendStorageFile, hasRemoteFile bool) {
+	if v.DataBackend != nil {
+		v.DataBackend.Close()
+	}
+	v.DataBackend = newBackend
+	v.hasRemoteFile.Store(hasRemoteFile)
+}
+
+func (v *Volume) doClose() {
+	if v.nm != nil {
+		if err := v.nm.Sync(); err != nil {
+			glog.Warningf("Volume Close fail to sync volume idx %d", v.Id)
+		}
+		v.nm.Close()
+		v.nm = nil
+	}
+	if v.DataBackend != nil {
+		if err := v.DataBackend.Close(); err != nil {
+			glog.Warningf("Volume Close fail to sync volume %d", v.Id)
+		}
+		v.DataBackend = nil
+		stats.VolumeServerVolumeGauge.WithLabelValues(v.Collection, "volume").Dec()
+	}
+}
+
+func (v *Volume) NeedToReplicate() bool {
+	return v.ReplicaPlacement.GetCopyCount() > 1
+}
+
+// volume is expired if modified time + volume ttl < now
+// except when volume is empty
+// or when the volume does not have a ttl
+// or when volumeSizeLimit is 0 when server just starts
+func (v *Volume) expired(contentSize uint64, volumeSizeLimit uint64) bool {
+	if volumeSizeLimit == 0 {
+		// skip if we don't know size limit
+		return false
+	}
+	if contentSize <= super_block.SuperBlockSize {
+		return false
+	}
+	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
+		return false
+	}
+	glog.V(2).Infof("volume %d now:%v lastModified:%v", v.Id, time.Now().Unix(), v.lastModifiedTsSeconds)
+	livedMinutes := (time.Now().Unix() - int64(v.lastModifiedTsSeconds)) / 60
+	glog.V(2).Infof("volume %d ttl:%v lived:%v", v.Id, v.Ttl, livedMinutes)
+	if int64(v.Ttl.Minutes()) < livedMinutes {
+		return true
+	}
+	return false
+}
+
+// ExpireAtSec is when this volume's data becomes garbage, counted from its last
+// write. Counting from the current time instead let every .vif rewrite -- a
+// read-only mark, a tier upload, an EC encode -- hand an already expiring volume
+// another full TTL. Zero when the volume has no TTL.
+func (v *Volume) ExpireAtSec() uint64 {
+	if v.Ttl == nil {
+		return 0
+	}
+	ttlSeconds := v.Ttl.ToSeconds()
+	if ttlSeconds == 0 {
+		return 0
+	}
+	lastWriteSec := v.lastModifiedTsSeconds
+	if lastWriteSec == 0 {
+		lastWriteSec = uint64(time.Now().Unix())
+	}
+	return lastWriteSec + ttlSeconds
+}
+
+// wait either maxDelayMinutes or 10% of ttl minutes
+func (v *Volume) expiredLongEnough(maxDelayMinutes uint32) bool {
+	if v.Ttl == nil || v.Ttl.Minutes() == 0 {
+		return false
+	}
+	removalDelay := v.Ttl.Minutes() / 10
+	if removalDelay > maxDelayMinutes {
+		removalDelay = maxDelayMinutes
+	}
+
+	if uint64(v.Ttl.Minutes()+removalDelay)*60+v.lastModifiedTsSeconds < uint64(time.Now().Unix()) {
+		return true
+	}
+	return false
+}
+
+func (v *Volume) collectStatus() (maxFileKey types.NeedleId, datFileSize int64, modTime time.Time, fileCount, deletedCount, deletedSize uint64, ok bool) {
+	v.dataFileAccessLock.RLock()
+	defer v.dataFileAccessLock.RUnlock()
+
+	if v.nm == nil || v.DataBackend == nil {
+		return
+	}
+
+	ok = true
+
+	maxFileKey = v.nm.MaxFileKey()
+	datFileSize, modTime, _ = v.DataBackend.GetStat()
+	fileCount = uint64(v.nm.FileCount())
+	deletedCount = uint64(v.nm.DeletedCount())
+	deletedSize = v.nm.DeletedSize()
+
+	return
+}
+
+// ToVolumeInformationMessage fills into with what the master is told about this
+// volume, allocating a message when into is nil. A heartbeat that keeps only
+// the volumes it reports fills the same message for all the rest.
+func (v *Volume) ToVolumeInformationMessage(into *master_pb.VolumeInformationMessage) (types.NeedleId, *master_pb.VolumeInformationMessage) {
+
+	maxFileKey, volumeSize, modTime, fileCount, deletedCount, deletedSize, ok := v.collectStatus()
+
+	if !ok {
+		return 0, nil
+	}
+
+	// Detect phantom volumes: the .dat was unlinked from disk but is still held
+	// open as a deleted FD, so the volume keeps serving and heartbeating while no
+	// disk-path operation can ever succeed. Skip remote-tiered volumes, whose .dat
+	// legitimately lives in cloud storage. Only a present .dat is cached for 30s; a
+	// missing one is re-checked every heartbeat so the volume stays suppressed until
+	// the file returns. See github.com/seaweedfs/seaweedfs/issues/10004
+	if fileCount > 0 && !v.HasRemoteFile() {
+		const diskCheckIntervalNs = 30 * int64(time.Second)
+		now := time.Now().UnixNano()
+		if now-v.lastDiskCheckNs.Load() > diskCheckIntervalNs {
+			if _, err := os.Stat(v.FileName(".dat")); os.IsNotExist(err) {
+				glog.Warningf("Volume %d: data file %s missing (held open as deleted FD) - not reporting to master", v.Id, v.FileName(".dat"))
+				return 0, nil
+			}
+			v.lastDiskCheckNs.Store(now)
+		}
+	}
+
+	volumeInfo := into
+	if volumeInfo == nil {
+		volumeInfo = &master_pb.VolumeInformationMessage{}
+	}
+	volumeInfo.Id = uint32(v.Id)
+	volumeInfo.Size = uint64(volumeSize)
+	volumeInfo.Collection = v.Collection
+	volumeInfo.FileCount = fileCount
+	volumeInfo.DeleteCount = deletedCount
+	volumeInfo.DeletedByteCount = deletedSize
+	volumeInfo.ReadOnly, _, volumeInfo.ReadOnlyCanDelete, _ = v.ReadOnlyReasons()
+	volumeInfo.ReplicaPlacement = uint32(v.ReplicaPlacement.Byte())
+	volumeInfo.Version = uint32(v.Version())
+	volumeInfo.Ttl = v.Ttl.ToUint32()
+	volumeInfo.CompactRevision = uint32(v.SuperBlock.CompactionRevision)
+	volumeInfo.ModifiedAtSecond = modTime.Unix()
+	volumeInfo.DiskType = string(v.location.DiskType)
+	volumeInfo.DiskId = v.diskId
+	volumeInfo.RemoteStorageName, volumeInfo.RemoteStorageKey = v.RemoteStorageNameKey()
+
+	return maxFileKey, volumeInfo
+}
+
+func (v *Volume) RemoteStorageNameKey() (storageName, storageKey string) {
+	if v.volumeInfo == nil {
+		return
+	}
+	if len(v.volumeInfo.GetFiles()) == 0 {
+		return
+	}
+	return v.volumeInfo.GetFiles()[0].BackendName(), v.volumeInfo.GetFiles()[0].GetKey()
+}
+
+func (v *Volume) IsReadOnly() bool {
+	readOnly, _, _, _ := v.ReadOnlyReasons()
+	return readOnly
+}
+
+// ReadOnlyReasons reports whether the volume refuses writes and why, reading the
+// flags once so the reasons cannot disagree with the verdict.
+func (v *Volume) ReadOnlyReasons() (readOnly, noWriteOrDelete, noWriteCanDelete, diskSpaceLow bool) {
+	v.noWriteLock.RLock()
+	noWriteOrDelete, noWriteCanDelete = v.noWriteOrDelete, v.noWriteCanDelete
+	v.noWriteLock.RUnlock()
+	// The location is attached when the volume joins a disk location, which is
+	// after NewVolume hands it back.
+	diskSpaceLow = v.location != nil && v.location.isDiskSpaceLow.Load()
+	return noWriteOrDelete || noWriteCanDelete || diskSpaceLow, noWriteOrDelete, noWriteCanDelete, diskSpaceLow
+}
+
+func (v *Volume) PersistReadOnly(readOnly bool, canDelete bool) error {
+	v.volumeInfoRWLock.Lock()
+	defer v.volumeInfoRWLock.Unlock()
+	prevReadOnly := v.volumeInfo.ReadOnly
+	prevReadOnlyCanDelete := v.volumeInfo.ReadOnlyCanDelete
+	v.volumeInfo.ReadOnly = readOnly
+	v.volumeInfo.ReadOnlyCanDelete = readOnly && canDelete
+	if err := v.SaveVolumeInfo(); err != nil {
+		// A pre-commit failure (write/sync/close/rename) leaves the old
+		// .vif intact, so roll back in-memory state to match it. A
+		// NotCrashDurableError means the rename already committed the
+		// new mode to disk; rolling back would split in-memory state
+		// from the durable file, so keep the new state and propagate.
+		var ndErr *volume_info.NotCrashDurableError
+		if !errors.As(err, &ndErr) {
+			v.volumeInfo.ReadOnly = prevReadOnly
+			v.volumeInfo.ReadOnlyCanDelete = prevReadOnlyCanDelete
+		}
+		return fmt.Errorf("persist volume read-only state: %w", err)
+	}
+	return nil
+}