snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/page_writer/upload_pipeline.go

@@ -0,0 +1,402 @@
+package page_writer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type LogicChunkIndex int
+
+type UploadPipeline struct {
+	uploaderCount       int32
+	uploaderCountCond   *sync.Cond
+	filepath            util.FullPath
+	ChunkSize           int64
+	uploaders           *util.LimitedConcurrentExecutor
+	saveToStorageFn     SaveToStorageFunc
+	writableChunkLimit  int
+	concurrentWriterMax int32
+	swapFile            *SwapFile
+	chunksLock          sync.Mutex
+	writableChunks      map[LogicChunkIndex]PageChunk
+	sealedChunks        map[LogicChunkIndex]*SealedChunk
+	activeReadChunks    map[LogicChunkIndex]int
+	readerCountCond     *sync.Cond
+	accountant          *WriteBufferAccountant
+	lastWriteChunkIndex int64 // atomic: highest LogicChunkIndex written
+}
+
+type SealedChunk struct {
+	chunk            PageChunk
+	referenceCounter int // track uploading or reading processes
+	// accountant and chunkSize are captured when the chunk is sealed so
+	// FreeReference can release the global write-budget slot exactly once
+	// regardless of which code path triggers the final deref (normal
+	// upload completion, Shutdown, or replace-in-place in moveToSealed).
+	accountant *WriteBufferAccountant
+	chunkSize  int64
+}
+
+func (sc *SealedChunk) FreeReference(messageOnFree string) {
+	// Early-return guard so repeated calls (Shutdown racing the async
+	// uploader, or any future caller that loses track of ownership) are
+	// strict no-ops rather than driving referenceCounter negative.
+	if sc.referenceCounter <= 0 {
+		return
+	}
+	sc.referenceCounter--
+	if sc.referenceCounter == 0 {
+		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
+		sc.chunk.FreeResource()
+		sc.accountant.Release(sc.chunkSize)
+	}
+}
+
+// NewUploadPipeline constructs an UploadPipeline. accountant may be nil,
+// in which case no global write-buffer cap is enforced. When non-nil,
+// creating a new page chunk (memory or swap) first reserves ChunkSize
+// bytes against it, blocking the writer if the global cap is reached.
+// The accountant is captured at construction so the pipeline's hot paths
+// (SaveDataAt, moveToSealed, Shutdown) can read up.accountant without
+// any synchronization.
+func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string, accountant *WriteBufferAccountant) *UploadPipeline {
+	t := &UploadPipeline{
+		ChunkSize:           chunkSize,
+		writableChunks:      make(map[LogicChunkIndex]PageChunk),
+		sealedChunks:        make(map[LogicChunkIndex]*SealedChunk),
+		uploaders:           writers,
+		uploaderCountCond:   sync.NewCond(&sync.Mutex{}),
+		saveToStorageFn:     saveToStorageFn,
+		activeReadChunks:    make(map[LogicChunkIndex]int),
+		writableChunkLimit:  bufferChunkLimit,
+		concurrentWriterMax: int32(bufferChunkLimit),
+		swapFile:            NewSwapFile(swapFileDir, chunkSize),
+		accountant:          accountant,
+	}
+	t.readerCountCond = sync.NewCond(&t.chunksLock)
+	return t
+}
+
+func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsNs int64) (n int, err error) {
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	// track write frontier for proactive flushing (CAS to avoid regression)
+	for {
+		old := atomic.LoadInt64(&up.lastWriteChunkIndex)
+		if int64(logicChunkIndex) <= old {
+			break
+		}
+		if atomic.CompareAndSwapInt64(&up.lastWriteChunkIndex, old, int64(logicChunkIndex)) {
+			break
+		}
+	}
+
+	pageChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		// Reserve a chunk-sized slot against the global write budget before
+		// allocating. Reserve may block when volumes are full and sealed
+		// chunks can't drain, so we must release chunksLock first — the
+		// uploader goroutines that eventually call Release take chunksLock.
+		if up.accountant != nil {
+			up.chunksLock.Unlock()
+			up.accountant.Reserve(up.ChunkSize)
+			up.chunksLock.Lock()
+			// Re-check: another writer on the same file may have created
+			// the chunk while we were blocked. If so, give the slot back.
+			if pageChunk, found = up.writableChunks[logicChunkIndex]; found {
+				up.accountant.Release(up.ChunkSize)
+			}
+		}
+	}
+	if !found {
+		if len(up.writableChunks) > up.writableChunkLimit {
+			// Per-pipeline soft cap. Seal the fullest gap-free chunk;
+			// gappy chunks must wait so in-flight FUSE writeback can fill
+			// the gap (issue #9330). If none qualifies, fall through —
+			// the memChunkCounter ceiling below redirects to swap.
+			candidateChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
+			for lci, mc := range up.writableChunks {
+				if !mc.IsContiguouslyWritten() {
+					continue
+				}
+				if b := mc.WrittenSize(); b > fullness {
+					candidateChunkIndex = lci
+					fullness = b
+				}
+			}
+			if candidateChunkIndex >= 0 {
+				up.moveToSealed(up.writableChunks[candidateChunkIndex], candidateChunkIndex)
+			}
+		}
+		// fmt.Printf("isSequential:%v len(up.writableChunks):%v memChunkCounter:%v", isSequential, len(up.writableChunks), memChunkCounter)
+		if isSequential &&
+			len(up.writableChunks) < up.writableChunkLimit &&
+			atomic.LoadInt64(&memChunkCounter) < 4*int64(up.writableChunkLimit) {
+			pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
+			// fmt.Printf(" create mem  chunk %d\n", logicChunkIndex)
+		} else {
+			pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
+			// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
+			if pageChunk == nil {
+				up.accountant.Release(up.ChunkSize)
+				return 0, fmt.Errorf("failed to create swap file chunk")
+			}
+		}
+		up.writableChunks[logicChunkIndex] = pageChunk
+	}
+	//if _, foundSealed := up.sealedChunks[logicChunkIndex]; foundSealed {
+	//	println("found already sealed chunk", logicChunkIndex)
+	//}
+	//if _, foundReading := up.activeReadChunks[logicChunkIndex]; foundReading {
+	//	println("found active read chunk", logicChunkIndex)
+	//}
+	n = pageChunk.WriteDataAt(p, off, tsNs)
+	up.maybeMoveToSealed(pageChunk, logicChunkIndex)
+
+	return
+}
+
+func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	up.chunksLock.Lock()
+	defer func() {
+		up.readerCountCond.Signal()
+		up.chunksLock.Unlock()
+	}()
+
+	// read from sealed chunks first
+	sealedChunk, found := up.sealedChunks[logicChunkIndex]
+	if found {
+		maxStop = sealedChunk.chunk.ReadDataAt(p, off, tsNs)
+		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
+	}
+
+	// read from writable chunks last
+	writableChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		return
+	}
+	writableMaxStop := writableChunk.ReadDataAt(p, off, tsNs)
+	glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop)
+	maxStop = max(maxStop, writableMaxStop)
+
+	return
+}
+
+func (up *UploadPipeline) FlushAll() {
+	up.flushChunks()
+	up.waitForCurrentWritersToComplete()
+}
+
+func (up *UploadPipeline) flushChunks() {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	for logicChunkIndex, memChunk := range up.writableChunks {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	if memChunk.IsComplete() {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	newCount := atomic.AddInt32(&up.uploaderCount, 1)
+	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, newCount-1, newCount)
+
+	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
+		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
+	}
+	sealedChunk := &SealedChunk{
+		chunk: memChunk,
+		// One for the sealedChunks slot, one for the upload below, which
+		// drops its own. Execute() returns before SaveContent reads the
+		// chunk, so a later seal must not free the buffer under it.
+		referenceCounter: 2,
+		accountant:       up.accountant,
+		chunkSize:        up.ChunkSize,
+	}
+	up.sealedChunks[logicChunkIndex] = sealedChunk
+	delete(up.writableChunks, logicChunkIndex)
+
+	// unlock before submitting the uploading jobs
+	up.chunksLock.Unlock()
+	up.uploaders.Execute(func() {
+		// first add to the file chunks
+		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
+
+		// notify waiting process
+		newCount := atomic.AddInt32(&up.uploaderCount, -1)
+		glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, newCount+1, newCount)
+		// Lock and Unlock are not required,
+		// but it may signal multiple times during one wakeup,
+		// and the waiting goroutine may miss some of them!
+		up.uploaderCountCond.L.Lock()
+		up.uploaderCountCond.Broadcast()
+		up.uploaderCountCond.L.Unlock()
+
+		// wait for readers
+		up.chunksLock.Lock()
+		defer up.chunksLock.Unlock()
+		for up.IsLocked(logicChunkIndex) {
+			up.readerCountCond.Wait()
+		}
+
+		// then remove from sealed chunks, unless a later seal took over the
+		// index — deleting that one would hide its dirty pages from readers
+		if up.sealedChunks[logicChunkIndex] == sealedChunk {
+			delete(up.sealedChunks, logicChunkIndex)
+			sealedChunk.FreeReference(fmt.Sprintf("%s unindex chunk %d", up.filepath, logicChunkIndex))
+		}
+		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
+
+	})
+	up.chunksLock.Lock()
+}
+
+// EvictOneWritableChunk force-seals one writable chunk so the accountant's
+// Reserve loop can make progress. Strict pass picks the fullest gap-free
+// chunk (issue #9330). Fallback picks the oldest non-empty writer when
+// nothing is gap-free; without it Reserve deadlocks (cond.Wait only wakes
+// on Release, Release only fires on upload completion). Callers must not
+// hold up.chunksLock.
+func (up *UploadPipeline) EvictOneWritableChunk() bool {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	bestIndex := LogicChunkIndex(-1)
+	var bestBytes int64
+	for lci, wc := range up.writableChunks {
+		if !wc.IsContiguouslyWritten() {
+			continue
+		}
+		if b := wc.WrittenSize(); b > bestBytes {
+			bestIndex = lci
+			bestBytes = b
+		}
+	}
+
+	if bestIndex < 0 {
+		oldestTsNs := int64(math.MaxInt64)
+		var fallbackBytes int64
+		for lci, wc := range up.writableChunks {
+			b := wc.WrittenSize()
+			if b == 0 {
+				continue
+			}
+			ts := wc.LastWriteTsNs()
+			if ts == 0 {
+				continue
+			}
+			if ts < oldestTsNs || (ts == oldestTsNs && b > fallbackBytes) {
+				bestIndex = lci
+				oldestTsNs = ts
+				fallbackBytes = b
+			}
+		}
+	}
+
+	if bestIndex < 0 {
+		return false
+	}
+	up.moveToSealed(up.writableChunks[bestIndex], bestIndex)
+	return true
+}
+
+// ProactiveFlush seals at most one idle writable chunk that is unlikely to
+// receive further writes, submitting it for async upload. Returns true if a
+// chunk was sealed. The caller (ChunkFlusher) invokes this periodically so
+// that partially-written chunks drain continuously instead of piling up
+// until fsync.
+func (up *UploadPipeline) ProactiveFlush(nowNs int64, idleThresholdNs int64, maxHoldNs int64, fillRatio int64, frontierLag int, isSequential bool) bool {
+	if up.concurrentWriterMax <= 0 || atomic.LoadInt32(&up.uploaderCount)*2 >= up.concurrentWriterMax {
+		return false
+	}
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	frontier := atomic.LoadInt64(&up.lastWriteChunkIndex)
+	isSeq := isSequential
+
+	var bestIdx LogicChunkIndex = -1
+	var bestBytes int64 = -1
+
+	for lci, chunk := range up.writableChunks {
+		lastWrite := chunk.LastWriteTsNs()
+		if lastWrite == 0 {
+			continue
+		}
+		age := nowNs - lastWrite
+		if age < idleThresholdNs {
+			continue
+		}
+		// Skip gappy chunks; FUSE writeback may still be in flight for the
+		// hole and SaveContent would bake it into split volume chunks
+		// (issue #9330). Failing here is just a missed flush — FlushAll
+		// at close still seals everything.
+		if !chunk.IsContiguouslyWritten() {
+			continue
+		}
+		written := chunk.WrittenSize()
+		nearlyFull := written >= fillRatio
+		behindFrontier := isSeq && int64(lci) <= frontier-int64(frontierLag)
+		stale := age >= maxHoldNs
+
+		if !nearlyFull && !behindFrontier && !stale {
+			continue
+		}
+		if written > bestBytes {
+			bestIdx = lci
+			bestBytes = written
+		}
+	}
+	if bestIdx < 0 {
+		return false
+	}
+	glog.V(3).Infof("%s proactive flush chunk %d (%d bytes written)", up.filepath, bestIdx, bestBytes)
+	up.moveToSealed(up.writableChunks[bestIdx], bestIdx)
+	return true
+}
+
+func (up *UploadPipeline) Shutdown() {
+	up.swapFile.FreeResource()
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	// Free any writable chunks that were reserved but never sealed — on the
+	// Destroy() path (truncate / metadata invalidation) there is no
+	// preceding FlushData(), so dirty writable chunks would otherwise leak
+	// both their memory and their write-budget slots.
+	for logicChunkIndex, writableChunk := range up.writableChunks {
+		glog.V(4).Infof("%s uploadpipeline shutdown writable chunk %d", up.filepath, logicChunkIndex)
+		writableChunk.FreeResource()
+		up.accountant.Release(up.ChunkSize)
+		delete(up.writableChunks, logicChunkIndex)
+	}
+	for logicChunkIndex, sealedChunk := range up.sealedChunks {
+		// only the slot reference; an in-flight upload frees the chunk itself
+		sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
+		delete(up.sealedChunks, logicChunkIndex)
+	}
+}