snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/mount/page_writer/activity_score.go

@@ -0,0 +1,52 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"time"
+)
+
+type ActivityScore struct {
+	lastActiveTsNs         int64 // atomic
+	decayedActivenessScore int64 // atomic
+}
+
+func NewActivityScore() *ActivityScore {
+	return &ActivityScore{}
+}
+
+func (as *ActivityScore) MarkRead() {
+	now := time.Now().UnixNano()
+	last := atomic.SwapInt64(&as.lastActiveTsNs, now)
+	deltaTime := (now - last) >> 30 // about number of seconds
+	for {
+		old := atomic.LoadInt64(&as.decayedActivenessScore)
+		score := old>>deltaTime + 256
+		if score < 0 {
+			score = 0
+		}
+		if atomic.CompareAndSwapInt64(&as.decayedActivenessScore, old, score) {
+			break
+		}
+	}
+}
+
+func (as *ActivityScore) MarkWrite() {
+	now := time.Now().UnixNano()
+	last := atomic.SwapInt64(&as.lastActiveTsNs, now)
+	deltaTime := (now - last) >> 30 // about number of seconds
+	for {
+		old := atomic.LoadInt64(&as.decayedActivenessScore)
+		score := old>>deltaTime + 1024
+		if score < 0 {
+			score = 0
+		}
+		if atomic.CompareAndSwapInt64(&as.decayedActivenessScore, old, score) {
+			break
+		}
+	}
+}
+
+func (as *ActivityScore) ActivityScore() int64 {
+	deltaTime := (time.Now().UnixNano() - atomic.LoadInt64(&as.lastActiveTsNs)) >> 30
+	return atomic.LoadInt64(&as.decayedActivenessScore) >> deltaTime
+}

weed/mount/page_writer/chunk_interval_list.go

@@ -0,0 +1,165 @@
+package page_writer
+
+import (
+	"math"
+)
+
+// ChunkWrittenInterval mark one written interval within one page chunk
+type ChunkWrittenInterval struct {
+	StartOffset int64
+	stopOffset  int64
+	TsNs        int64
+	prev        *ChunkWrittenInterval
+	next        *ChunkWrittenInterval
+}
+
+func (interval *ChunkWrittenInterval) Size() int64 {
+	return interval.stopOffset - interval.StartOffset
+}
+
+func (interval *ChunkWrittenInterval) isComplete(chunkSize int64) bool {
+	return interval.stopOffset-interval.StartOffset == chunkSize
+}
+
+// ChunkWrittenIntervalList mark written intervals within one page chunk
+type ChunkWrittenIntervalList struct {
+	head *ChunkWrittenInterval
+	tail *ChunkWrittenInterval
+}
+
+func newChunkWrittenIntervalList() *ChunkWrittenIntervalList {
+	list := &ChunkWrittenIntervalList{
+		head: &ChunkWrittenInterval{
+			StartOffset: -1,
+			stopOffset:  -1,
+		},
+		tail: &ChunkWrittenInterval{
+			StartOffset: math.MaxInt64,
+			stopOffset:  math.MaxInt64,
+		},
+	}
+	list.head.next = list.tail
+	list.tail.prev = list.head
+	return list
+}
+
+func (list *ChunkWrittenIntervalList) MarkWritten(startOffset, stopOffset, tsNs int64) {
+	if startOffset >= stopOffset {
+		return
+	}
+	interval := &ChunkWrittenInterval{
+		StartOffset: startOffset,
+		stopOffset:  stopOffset,
+		TsNs:        tsNs,
+	}
+	list.addInterval(interval)
+}
+
+// IsComplete reports whether every byte of [0, chunkSize) has been
+// written, possibly via multiple adjacent or overlapping intervals.
+// addInterval does not merge adjacent intervals — a chunk filled by
+// two 1 MiB writes ends up as {[0,1M], [1M,2M]}, not one [0,2M] —
+// so checking list.size()==1 misses the "filled by adjacent writes"
+// case, leaving the chunk pinned in writableChunks even though all
+// its bytes are present. That latent bug became a hard deadlock once
+// -writeBufferSizeMB started reserving a global slot per writable
+// chunk: the chunks never got sealed, no uploader ran, no slot was
+// ever released, and the FUSE writer blocked in Reserve forever
+// (seaweedfs issue #8777 / PR #9066). Walking the list and tracking
+// the furthest covered offset detects adjacency correctly.
+func (list *ChunkWrittenIntervalList) IsComplete(chunkSize int64) bool {
+	var covered int64
+	for t := list.head.next; t != list.tail; t = t.next {
+		if t.StartOffset > covered {
+			return false // gap before this interval
+		}
+		if t.stopOffset > covered {
+			covered = t.stopOffset
+		}
+	}
+	return covered >= chunkSize
+}
+func (list *ChunkWrittenIntervalList) WrittenSize() (writtenByteCount int64) {
+	for t := list.head; t != nil; t = t.next {
+		writtenByteCount += t.Size()
+	}
+	return
+}
+
+// IsContiguouslyWritten reports whether the written bytes form one
+// unbroken run starting at offset 0. Pressure-driven sealing uses this
+// to avoid racing in-flight FUSE writeback on a gap range — sealing a
+// gappy chunk would emit volume chunks with no coverage for the gap and
+// reads would silently zero-fill it (issue #9330).
+func (list *ChunkWrittenIntervalList) IsContiguouslyWritten() bool {
+	first := list.head.next
+	if first == list.tail || first.StartOffset != 0 {
+		return false
+	}
+	for t := first; t.next != list.tail; t = t.next {
+		if t.stopOffset != t.next.StartOffset {
+			return false
+		}
+	}
+	return true
+}
+
+func (list *ChunkWrittenIntervalList) addInterval(interval *ChunkWrittenInterval) {
+
+	//t := list.head
+	//for ; t.next != nil; t = t.next {
+	//	if t.TsNs > interval.TsNs {
+	//		println("writes is out of order", t.TsNs-interval.TsNs, "ns")
+	//	}
+	//}
+
+	p := list.head
+	for ; p.next != nil && p.next.stopOffset <= interval.StartOffset; p = p.next {
+	}
+	q := list.tail
+	for ; q.prev != nil && q.prev.StartOffset >= interval.stopOffset; q = q.prev {
+	}
+
+	// left side
+	// interval after p.next start
+	if p.next.StartOffset < interval.StartOffset {
+		t := &ChunkWrittenInterval{
+			StartOffset: p.next.StartOffset,
+			stopOffset:  interval.StartOffset,
+			TsNs:        p.next.TsNs,
+		}
+		p.next = t
+		t.prev = p
+		t.next = interval
+		interval.prev = t
+	} else {
+		p.next = interval
+		interval.prev = p
+	}
+
+	// right side
+	// interval ends before p.prev
+	if interval.stopOffset < q.prev.stopOffset {
+		t := &ChunkWrittenInterval{
+			StartOffset: interval.stopOffset,
+			stopOffset:  q.prev.stopOffset,
+			TsNs:        q.prev.TsNs,
+		}
+		q.prev = t
+		t.next = q
+		interval.next = t
+		t.prev = interval
+	} else {
+		q.prev = interval
+		interval.next = q
+	}
+
+}
+
+func (list *ChunkWrittenIntervalList) size() int {
+	var count int
+	for t := list.head; t != nil; t = t.next {
+		count++
+	}
+	return count - 2
+}

weed/mount/page_writer/chunk_interval_list_test.go

@@ -0,0 +1,189 @@
+package page_writer
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_PageChunkWrittenIntervalList(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+
+	assert.Equal(t, 0, list.size(), "empty list")
+
+	list.MarkWritten(0, 5, 1)
+	assert.Equal(t, 1, list.size(), "one interval")
+
+	list.MarkWritten(0, 5, 2)
+	assert.Equal(t, 1, list.size(), "duplicated interval2")
+
+	list.MarkWritten(95, 100, 3)
+	assert.Equal(t, 2, list.size(), "two intervals")
+
+	list.MarkWritten(50, 60, 4)
+	assert.Equal(t, 3, list.size(), "three intervals")
+
+	list.MarkWritten(50, 55, 5)
+	assert.Equal(t, 4, list.size(), "three intervals merge")
+
+	list.MarkWritten(40, 50, 6)
+	assert.Equal(t, 5, list.size(), "three intervals grow forward")
+
+	list.MarkWritten(50, 65, 7)
+	assert.Equal(t, 4, list.size(), "three intervals grow backward")
+
+	list.MarkWritten(70, 80, 8)
+	assert.Equal(t, 5, list.size(), "four intervals")
+
+	list.MarkWritten(60, 70, 9)
+	assert.Equal(t, 6, list.size(), "three intervals merged")
+
+	list.MarkWritten(59, 71, 10)
+	assert.Equal(t, 6, list.size(), "covered three intervals")
+
+	list.MarkWritten(5, 59, 11)
+	assert.Equal(t, 5, list.size(), "covered two intervals")
+
+	list.MarkWritten(70, 99, 12)
+	assert.Equal(t, 5, list.size(), "covered one intervals")
+
+}
+
+type interval struct {
+	start    int64
+	stop     int64
+	expected bool
+}
+
+func Test_PageChunkWrittenIntervalList1(t *testing.T) {
+	list := newChunkWrittenIntervalList()
+	inputs := []interval{
+		{1, 5, true},
+		{2, 3, true},
+	}
+	for i, input := range inputs {
+		list.MarkWritten(input.start, input.stop, int64(i)+1)
+		actual := hasData(list, 0, 4)
+		if actual != input.expected {
+			t.Errorf("input [%d,%d) expected %v actual %v", input.start, input.stop, input.expected, actual)
+		}
+	}
+}
+
+func hasData(usage *ChunkWrittenIntervalList, chunkStartOffset, x int64) bool {
+	for t := usage.head.next; t != usage.tail; t = t.next {
+		logicStart := chunkStartOffset + t.StartOffset
+		logicStop := chunkStartOffset + t.stopOffset
+		if logicStart <= x && x < logicStop {
+			return true
+		}
+	}
+	return false
+}
+
+func TestIsContiguouslyWritten(t *testing.T) {
+	const chunkSize int64 = 2 * 1024 * 1024
+	cases := []struct {
+		name     string
+		writes   [][2]int64
+		expected bool
+	}{
+		{name: "empty", writes: nil, expected: false},
+		{name: "single full", writes: [][2]int64{{0, chunkSize}}, expected: true},
+		{name: "single partial at start", writes: [][2]int64{{0, chunkSize / 4}}, expected: true},
+		{name: "two adjacent halves in order", writes: [][2]int64{{0, chunkSize / 2}, {chunkSize / 2, chunkSize}}, expected: true},
+		{name: "two adjacent halves out of order", writes: [][2]int64{{chunkSize / 2, chunkSize}, {0, chunkSize / 2}}, expected: true},
+		{name: "overlapping covers everything", writes: [][2]int64{{0, chunkSize*3/4 + 1}, {chunkSize / 2, chunkSize}}, expected: true},
+		{name: "three adjacent thirds", writes: [][2]int64{{0, chunkSize / 3}, {chunkSize / 3, 2 * chunkSize / 3}, {2 * chunkSize / 3, chunkSize}}, expected: true},
+		{name: "single partial in middle", writes: [][2]int64{{chunkSize / 4, chunkSize / 2}}, expected: false},
+		{name: "first byte missing", writes: [][2]int64{{1, chunkSize}}, expected: false},
+		{name: "gap in middle", writes: [][2]int64{{0, chunkSize / 4}, {chunkSize / 2, chunkSize}}, expected: false},
+		{name: "two intervals with 1-byte gap", writes: [][2]int64{{0, 100}, {101, 200}}, expected: false},
+		{name: "three intervals with middle gap", writes: [][2]int64{{0, chunkSize / 4}, {chunkSize / 3, chunkSize / 2}, {chunkSize / 2, chunkSize}}, expected: false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			list := newChunkWrittenIntervalList()
+			for i, w := range tc.writes {
+				list.MarkWritten(w[0], w[1], int64(i+1))
+			}
+			assert.Equal(t, tc.expected, list.IsContiguouslyWritten(), "IsContiguouslyWritten mismatch")
+		})
+	}
+}
+
+func TestIsComplete_AdjacentIntervals(t *testing.T) {
+	// Linux FUSE delivers writes up to FUSE_MAX_PAGES_PER_REQ
+	// (typically 1 MiB) per op, so a 2 MiB chunk filled by sequential
+	// writes arrives as two adjacent 1 MiB writes. addInterval does
+	// not merge adjacent intervals into one, but IsComplete must
+	// still report the chunk as fully covered — otherwise
+	// maybeMoveToSealed never fires, the chunk stays writable
+	// forever, and the write buffer cap cannot drain. Regression for
+	// the TestWriteBufferCap Linux CI deadlock on PR #9066.
+	const chunkSize int64 = 2 * 1024 * 1024
+
+	cases := []struct {
+		name     string
+		writes   [][2]int64
+		expected bool
+	}{
+		{
+			name:     "single full write",
+			writes:   [][2]int64{{0, chunkSize}},
+			expected: true,
+		},
+		{
+			name:     "two adjacent halves in order",
+			writes:   [][2]int64{{0, chunkSize / 2}, {chunkSize / 2, chunkSize}},
+			expected: true,
+		},
+		{
+			name:     "two adjacent halves out of order",
+			writes:   [][2]int64{{chunkSize / 2, chunkSize}, {0, chunkSize / 2}},
+			expected: true,
+		},
+		{
+			name: "eight adjacent eighths",
+			writes: func() [][2]int64 {
+				const n = 8
+				step := chunkSize / n
+				out := make([][2]int64, 0, n)
+				for i := int64(0); i < n; i++ {
+					out = append(out, [2]int64{i * step, (i + 1) * step})
+				}
+				return out
+			}(),
+			expected: true,
+		},
+		{
+			name:     "gap in the middle",
+			writes:   [][2]int64{{0, chunkSize / 4}, {chunkSize / 2, chunkSize}},
+			expected: false,
+		},
+		{
+			name:     "missing last byte",
+			writes:   [][2]int64{{0, chunkSize - 1}},
+			expected: false,
+		},
+		{
+			name:     "missing first byte",
+			writes:   [][2]int64{{1, chunkSize}},
+			expected: false,
+		},
+		{
+			name:     "overlapping intervals covering everything",
+			writes:   [][2]int64{{0, chunkSize*3/4 + 1}, {chunkSize / 2, chunkSize}},
+			expected: true,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			list := newChunkWrittenIntervalList()
+			for i, w := range tc.writes {
+				list.MarkWritten(w[0], w[1], int64(i+1))
+			}
+			assert.Equal(t, tc.expected, list.IsComplete(chunkSize), "IsComplete mismatch")
+		})
+	}
+}

weed/mount/page_writer/dirty_pages.go

@@ -0,0 +1,26 @@
+package page_writer
+
+type DirtyPages interface {
+	AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error
+	FlushData() error
+	HasWrites() bool
+	ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64)
+	Destroy()
+	LockForRead(startOffset, stopOffset int64)
+	UnlockForRead(startOffset, stopOffset int64)
+	EvictOneWritableChunk() bool
+	ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio int64, frontierLag int, isSequential bool) bool
+}
+
+func max(x, y int64) int64 {
+	if x > y {
+		return x
+	}
+	return y
+}
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/mount/page_writer/page_chunk.go

@@ -0,0 +1,19 @@
+package page_writer
+
+import (
+	"io"
+)
+
+type SaveToStorageFunc func(reader io.Reader, offset int64, size int64, modifiedTsNs int64, cleanupFn func())
+
+type PageChunk interface {
+	FreeResource()
+	WriteDataAt(src []byte, offset int64, tsNs int64) (n int)
+	ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64)
+	IsComplete() bool
+	IsContiguouslyWritten() bool
+	ActivityScore() int64
+	WrittenSize() int64
+	LastWriteTsNs() int64
+	SaveContent(saveFn SaveToStorageFunc)
+}

weed/mount/page_writer/page_chunk_mem.go

@@ -0,0 +1,127 @@
+package page_writer
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+var (
+	_ = PageChunk(&MemChunk{})
+
+	memChunkCounter int64
+)
+
+type MemChunk struct {
+	sync.RWMutex
+	buf             []byte
+	usage           *ChunkWrittenIntervalList
+	chunkSize       int64
+	logicChunkIndex LogicChunkIndex
+	activityScore   *ActivityScore
+	lastWriteTsNs   atomic.Int64
+}
+
+func NewMemChunk(logicChunkIndex LogicChunkIndex, chunkSize int64) *MemChunk {
+	atomic.AddInt64(&memChunkCounter, 1)
+	return &MemChunk{
+		logicChunkIndex: logicChunkIndex,
+		chunkSize:       chunkSize,
+		buf:             mem.Allocate(int(chunkSize)),
+		usage:           newChunkWrittenIntervalList(),
+		activityScore:   NewActivityScore(),
+	}
+}
+
+func (mc *MemChunk) FreeResource() {
+	mc.Lock()
+	defer mc.Unlock()
+
+	atomic.AddInt64(&memChunkCounter, -1)
+	mem.Free(mc.buf)
+}
+
+func (mc *MemChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+	mc.Lock()
+	defer mc.Unlock()
+
+	innerOffset := offset % mc.chunkSize
+	n = copy(mc.buf[innerOffset:], src)
+	mc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
+	mc.activityScore.MarkWrite()
+	mc.lastWriteTsNs.Store(tsNs)
+
+	return
+}
+
+func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	memChunkBaseOffset := int64(mc.logicChunkIndex) * mc.chunkSize
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		logicStart := max(off, memChunkBaseOffset+t.StartOffset)
+		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
+		if logicStart < logicStop {
+			copy(p[logicStart-off:logicStop-off], mc.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	mc.activityScore.MarkRead()
+
+	return
+}
+
+func (mc *MemChunk) IsComplete() bool {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.IsComplete(mc.chunkSize)
+}
+
+func (mc *MemChunk) IsContiguouslyWritten() bool {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.IsContiguouslyWritten()
+}
+
+func (mc *MemChunk) ActivityScore() int64 {
+	return mc.activityScore.ActivityScore()
+}
+
+func (mc *MemChunk) WrittenSize() int64 {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	return mc.usage.WrittenSize()
+}
+
+func (mc *MemChunk) LastWriteTsNs() int64 {
+	return mc.lastWriteTsNs.Load()
+}
+
+func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
+	mc.RLock()
+	defer mc.RUnlock()
+
+	if saveFn == nil {
+		return
+	}
+
+	for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
+		startOffset := t.StartOffset
+		stopOffset := t.stopOffset
+		tsNs := t.TsNs
+		for t != mc.usage.tail && t.next.StartOffset == stopOffset {
+			stopOffset = t.next.stopOffset
+			t = t.next
+			tsNs = max(tsNs, t.TsNs)
+		}
+		reader := util.NewBytesReader(mc.buf[startOffset:stopOffset])
+		saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+startOffset, stopOffset-startOffset, tsNs, func() {
+		})
+	}
+}

weed/mount/page_writer/page_chunk_swapfile.go

@@ -0,0 +1,223 @@
+package page_writer
+
+import (
+	"io"
+	"os"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/mem"
+)
+
+var (
+	_ = PageChunk(&SwapFileChunk{})
+)
+
+type ActualChunkIndex int
+
+type SwapFile struct {
+	dir                 string
+	file                *os.File
+	chunkSize           int64
+	chunkTrackingLock   sync.Mutex
+	activeChunkCount    int
+	freeActualChunkList []ActualChunkIndex
+}
+
+type SwapFileChunk struct {
+	sync.RWMutex
+	swapfile         *SwapFile
+	usage            *ChunkWrittenIntervalList
+	logicChunkIndex  LogicChunkIndex
+	actualChunkIndex ActualChunkIndex
+	activityScore    *ActivityScore
+	lastWriteTsNs    atomic.Int64
+	//memChunk         *MemChunk
+}
+
+func NewSwapFile(dir string, chunkSize int64) *SwapFile {
+	return &SwapFile{
+		dir:       dir,
+		file:      nil,
+		chunkSize: chunkSize,
+	}
+}
+func (sf *SwapFile) FreeResource() {
+	sf.chunkTrackingLock.Lock()
+	defer sf.chunkTrackingLock.Unlock()
+	if sf.file != nil {
+		sf.file.Close()
+		os.Remove(sf.file.Name())
+		sf.file = nil
+	}
+}
+
+func (sf *SwapFile) NewSwapFileChunk(logicChunkIndex LogicChunkIndex) (tc *SwapFileChunk) {
+	sf.chunkTrackingLock.Lock()
+	defer sf.chunkTrackingLock.Unlock()
+
+	if sf.file == nil {
+		var err error
+		sf.file, err = os.CreateTemp(sf.dir, "")
+		if os.IsNotExist(err) {
+			if mkdirErr := os.MkdirAll(sf.dir, 0700); mkdirErr != nil {
+				glog.Errorf("create/recreate swap directory %s: %v", sf.dir, mkdirErr)
+				return nil
+			}
+			sf.file, err = os.CreateTemp(sf.dir, "")
+		}
+		if err != nil {
+			glog.Errorf("create swap file in %s: %v", sf.dir, err)
+			return nil
+		}
+	}
+
+	sf.activeChunkCount++
+
+	// assign a new physical chunk
+	var actualChunkIndex ActualChunkIndex
+	if len(sf.freeActualChunkList) > 0 {
+		actualChunkIndex = sf.freeActualChunkList[0]
+		sf.freeActualChunkList = sf.freeActualChunkList[1:]
+	} else {
+		actualChunkIndex = ActualChunkIndex(sf.activeChunkCount)
+	}
+
+	swapFileChunk := &SwapFileChunk{
+		swapfile:         sf,
+		usage:            newChunkWrittenIntervalList(),
+		logicChunkIndex:  logicChunkIndex,
+		actualChunkIndex: actualChunkIndex,
+		activityScore:    NewActivityScore(),
+		// memChunk:         NewMemChunk(logicChunkIndex, sf.chunkSize),
+	}
+
+	// println(logicChunkIndex, "|", "++++", swapFileChunk.actualChunkIndex, swapFileChunk, sf)
+	return swapFileChunk
+}
+
+func (sc *SwapFileChunk) FreeResource() {
+
+	sc.Lock()
+	defer sc.Unlock()
+
+	sc.swapfile.chunkTrackingLock.Lock()
+	defer sc.swapfile.chunkTrackingLock.Unlock()
+
+	sc.swapfile.freeActualChunkList = append(sc.swapfile.freeActualChunkList, sc.actualChunkIndex)
+	sc.swapfile.activeChunkCount--
+	// println(sc.logicChunkIndex, "|", "----", sc.actualChunkIndex, sc, sc.swapfile)
+}
+
+func (sc *SwapFileChunk) WriteDataAt(src []byte, offset int64, tsNs int64) (n int) {
+	sc.Lock()
+	defer sc.Unlock()
+
+	// println(sc.logicChunkIndex, "|", tsNs, "write at", offset, len(src), sc.actualChunkIndex)
+
+	innerOffset := offset % sc.swapfile.chunkSize
+	var err error
+	n, err = sc.swapfile.file.WriteAt(src, int64(sc.actualChunkIndex)*sc.swapfile.chunkSize+innerOffset)
+	sc.usage.MarkWritten(innerOffset, innerOffset+int64(n), tsNs)
+	if err != nil {
+		glog.Errorf("failed to write swap file %s: %v", sc.swapfile.file.Name(), err)
+	}
+	//sc.memChunk.WriteDataAt(src, offset, tsNs)
+	sc.activityScore.MarkWrite()
+	sc.lastWriteTsNs.Store(tsNs)
+
+	return
+}
+
+func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	sc.RLock()
+	defer sc.RUnlock()
+
+	// println(sc.logicChunkIndex, "|", tsNs, "read at", off, len(p), sc.actualChunkIndex)
+
+	//memCopy := make([]byte, len(p))
+	//copy(memCopy, p)
+
+	chunkStartOffset := int64(sc.logicChunkIndex) * sc.swapfile.chunkSize
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		logicStart := max(off, chunkStartOffset+t.StartOffset)
+		logicStop := min(off+int64(len(p)), chunkStartOffset+t.stopOffset)
+		if logicStart < logicStop {
+			actualStart := logicStart - chunkStartOffset + int64(sc.actualChunkIndex)*sc.swapfile.chunkSize
+			if n, err := sc.swapfile.file.ReadAt(p[logicStart-off:logicStop-off], actualStart); err != nil {
+				if err == io.EOF && n == int(logicStop-logicStart) {
+					err = nil
+				}
+				glog.Errorf("failed to reading swap file %s: %v", sc.swapfile.file.Name(), err)
+				break
+			}
+			maxStop = max(maxStop, logicStop)
+		}
+	}
+	//sc.memChunk.ReadDataAt(memCopy, off, tsNs)
+	//if bytes.Compare(memCopy, p) != 0 {
+	//	println("read wrong data from swap file", off, sc.logicChunkIndex)
+	//}
+
+	sc.activityScore.MarkRead()
+
+	return
+}
+
+func (sc *SwapFileChunk) IsComplete() bool {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.IsComplete(sc.swapfile.chunkSize)
+}
+
+func (sc *SwapFileChunk) IsContiguouslyWritten() bool {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.IsContiguouslyWritten()
+}
+
+func (sc *SwapFileChunk) ActivityScore() int64 {
+	return sc.activityScore.ActivityScore()
+}
+
+func (sc *SwapFileChunk) WrittenSize() int64 {
+	sc.RLock()
+	defer sc.RUnlock()
+	return sc.usage.WrittenSize()
+}
+
+func (sc *SwapFileChunk) LastWriteTsNs() int64 {
+	return sc.lastWriteTsNs.Load()
+}
+
+func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
+	sc.RLock()
+	defer sc.RUnlock()
+
+	if saveFn == nil {
+		return
+	}
+	// println(sc.logicChunkIndex, "|", "save")
+	for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
+		startOffset := t.StartOffset
+		stopOffset := t.stopOffset
+		tsNs := t.TsNs
+		for t != sc.usage.tail && t.next.StartOffset == stopOffset {
+			stopOffset = t.next.stopOffset
+			t = t.next
+			tsNs = max(tsNs, t.TsNs)
+		}
+
+		data := mem.Allocate(int(stopOffset - startOffset))
+		n, _ := sc.swapfile.file.ReadAt(data, startOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
+		if n > 0 {
+			reader := util.NewBytesReader(data[:n])
+			saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+startOffset, int64(n), tsNs, func() {
+			})
+		}
+		mem.Free(data)
+	}
+
+}

weed/mount/page_writer/page_chunk_swapfile_test.go

@@ -0,0 +1,105 @@
+package page_writer
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+func TestSwapFile_NewSwapFileChunk_Concurrent(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	sf := NewSwapFile(filepath.Join(tempDir, "swap"), 1024)
+	defer sf.FreeResource()
+
+	var wg sync.WaitGroup
+	var failures atomic.Uint32
+	numConcurrent := 10
+	for i := 0; i < numConcurrent; i++ {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			chunk := sf.NewSwapFileChunk(LogicChunkIndex(idx))
+			if chunk == nil {
+				failures.Add(1)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	if failures.Load() > 0 {
+		t.Errorf("failed to create %d chunks", failures.Load())
+	}
+
+	if sf.file == nil {
+		t.Error("sf.file should not be nil")
+	}
+}
+
+func TestSwapFile_MkdirAll_Permissions(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_perm_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	swapDir := filepath.Join(tempDir, "swap_subdir")
+	sf := NewSwapFile(swapDir, 1024)
+	defer sf.FreeResource()
+
+	// This should trigger os.MkdirAll
+	if sf.NewSwapFileChunk(0) == nil {
+		t.Fatal("NewSwapFileChunk failed to create a chunk")
+	}
+
+	info, err := os.Stat(swapDir)
+	if err != nil {
+		t.Fatalf("failed to stat swap dir: %v", err)
+	}
+
+	if !info.IsDir() {
+		t.Errorf("expected %s to be a directory", swapDir)
+	}
+
+	// Check permissions - should be 0700
+	if info.Mode().Perm() != 0700 {
+		t.Errorf("expected permissions 0700, got %o", info.Mode().Perm())
+	}
+}
+
+func TestSwapFile_RecreateDir(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "seaweedfs_swap_recreate_test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	swapDir := filepath.Join(tempDir, "swap_recreate")
+	sf := NewSwapFile(swapDir, 1024)
+
+	// Create first chunk
+	if sf.NewSwapFileChunk(0) == nil {
+		t.Fatal("NewSwapFileChunk failed to create the first chunk")
+	}
+	sf.FreeResource()
+
+	// Delete the directory
+	os.RemoveAll(swapDir)
+
+	// Create second chunk - should recreate directory
+	chunk := sf.NewSwapFileChunk(1)
+	if chunk == nil {
+		t.Error("failed to create chunk after directory deletion")
+	}
+
+	if _, err := os.Stat(swapDir); os.IsNotExist(err) {
+		t.Error("swap directory was not recreated")
+	}
+	sf.FreeResource()
+}

weed/mount/page_writer/upload_pipeline.go

@@ -0,0 +1,402 @@
+package page_writer
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"sync/atomic"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type LogicChunkIndex int
+
+type UploadPipeline struct {
+	uploaderCount       int32
+	uploaderCountCond   *sync.Cond
+	filepath            util.FullPath
+	ChunkSize           int64
+	uploaders           *util.LimitedConcurrentExecutor
+	saveToStorageFn     SaveToStorageFunc
+	writableChunkLimit  int
+	concurrentWriterMax int32
+	swapFile            *SwapFile
+	chunksLock          sync.Mutex
+	writableChunks      map[LogicChunkIndex]PageChunk
+	sealedChunks        map[LogicChunkIndex]*SealedChunk
+	activeReadChunks    map[LogicChunkIndex]int
+	readerCountCond     *sync.Cond
+	accountant          *WriteBufferAccountant
+	lastWriteChunkIndex int64 // atomic: highest LogicChunkIndex written
+}
+
+type SealedChunk struct {
+	chunk            PageChunk
+	referenceCounter int // track uploading or reading processes
+	// accountant and chunkSize are captured when the chunk is sealed so
+	// FreeReference can release the global write-budget slot exactly once
+	// regardless of which code path triggers the final deref (normal
+	// upload completion, Shutdown, or replace-in-place in moveToSealed).
+	accountant *WriteBufferAccountant
+	chunkSize  int64
+}
+
+func (sc *SealedChunk) FreeReference(messageOnFree string) {
+	// Early-return guard so repeated calls (Shutdown racing the async
+	// uploader, or any future caller that loses track of ownership) are
+	// strict no-ops rather than driving referenceCounter negative.
+	if sc.referenceCounter <= 0 {
+		return
+	}
+	sc.referenceCounter--
+	if sc.referenceCounter == 0 {
+		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
+		sc.chunk.FreeResource()
+		sc.accountant.Release(sc.chunkSize)
+	}
+}
+
+// NewUploadPipeline constructs an UploadPipeline. accountant may be nil,
+// in which case no global write-buffer cap is enforced. When non-nil,
+// creating a new page chunk (memory or swap) first reserves ChunkSize
+// bytes against it, blocking the writer if the global cap is reached.
+// The accountant is captured at construction so the pipeline's hot paths
+// (SaveDataAt, moveToSealed, Shutdown) can read up.accountant without
+// any synchronization.
+func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string, accountant *WriteBufferAccountant) *UploadPipeline {
+	t := &UploadPipeline{
+		ChunkSize:           chunkSize,
+		writableChunks:      make(map[LogicChunkIndex]PageChunk),
+		sealedChunks:        make(map[LogicChunkIndex]*SealedChunk),
+		uploaders:           writers,
+		uploaderCountCond:   sync.NewCond(&sync.Mutex{}),
+		saveToStorageFn:     saveToStorageFn,
+		activeReadChunks:    make(map[LogicChunkIndex]int),
+		writableChunkLimit:  bufferChunkLimit,
+		concurrentWriterMax: int32(bufferChunkLimit),
+		swapFile:            NewSwapFile(swapFileDir, chunkSize),
+		accountant:          accountant,
+	}
+	t.readerCountCond = sync.NewCond(&t.chunksLock)
+	return t
+}
+
+func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsNs int64) (n int, err error) {
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	// track write frontier for proactive flushing (CAS to avoid regression)
+	for {
+		old := atomic.LoadInt64(&up.lastWriteChunkIndex)
+		if int64(logicChunkIndex) <= old {
+			break
+		}
+		if atomic.CompareAndSwapInt64(&up.lastWriteChunkIndex, old, int64(logicChunkIndex)) {
+			break
+		}
+	}
+
+	pageChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		// Reserve a chunk-sized slot against the global write budget before
+		// allocating. Reserve may block when volumes are full and sealed
+		// chunks can't drain, so we must release chunksLock first — the
+		// uploader goroutines that eventually call Release take chunksLock.
+		if up.accountant != nil {
+			up.chunksLock.Unlock()
+			up.accountant.Reserve(up.ChunkSize)
+			up.chunksLock.Lock()
+			// Re-check: another writer on the same file may have created
+			// the chunk while we were blocked. If so, give the slot back.
+			if pageChunk, found = up.writableChunks[logicChunkIndex]; found {
+				up.accountant.Release(up.ChunkSize)
+			}
+		}
+	}
+	if !found {
+		if len(up.writableChunks) > up.writableChunkLimit {
+			// Per-pipeline soft cap. Seal the fullest gap-free chunk;
+			// gappy chunks must wait so in-flight FUSE writeback can fill
+			// the gap (issue #9330). If none qualifies, fall through —
+			// the memChunkCounter ceiling below redirects to swap.
+			candidateChunkIndex, fullness := LogicChunkIndex(-1), int64(0)
+			for lci, mc := range up.writableChunks {
+				if !mc.IsContiguouslyWritten() {
+					continue
+				}
+				if b := mc.WrittenSize(); b > fullness {
+					candidateChunkIndex = lci
+					fullness = b
+				}
+			}
+			if candidateChunkIndex >= 0 {
+				up.moveToSealed(up.writableChunks[candidateChunkIndex], candidateChunkIndex)
+			}
+		}
+		// fmt.Printf("isSequential:%v len(up.writableChunks):%v memChunkCounter:%v", isSequential, len(up.writableChunks), memChunkCounter)
+		if isSequential &&
+			len(up.writableChunks) < up.writableChunkLimit &&
+			atomic.LoadInt64(&memChunkCounter) < 4*int64(up.writableChunkLimit) {
+			pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
+			// fmt.Printf(" create mem  chunk %d\n", logicChunkIndex)
+		} else {
+			pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
+			// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
+			if pageChunk == nil {
+				up.accountant.Release(up.ChunkSize)
+				return 0, fmt.Errorf("failed to create swap file chunk")
+			}
+		}
+		up.writableChunks[logicChunkIndex] = pageChunk
+	}
+	//if _, foundSealed := up.sealedChunks[logicChunkIndex]; foundSealed {
+	//	println("found already sealed chunk", logicChunkIndex)
+	//}
+	//if _, foundReading := up.activeReadChunks[logicChunkIndex]; foundReading {
+	//	println("found active read chunk", logicChunkIndex)
+	//}
+	n = pageChunk.WriteDataAt(p, off, tsNs)
+	up.maybeMoveToSealed(pageChunk, logicChunkIndex)
+
+	return
+}
+
+func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) {
+	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
+
+	up.chunksLock.Lock()
+	defer func() {
+		up.readerCountCond.Signal()
+		up.chunksLock.Unlock()
+	}()
+
+	// read from sealed chunks first
+	sealedChunk, found := up.sealedChunks[logicChunkIndex]
+	if found {
+		maxStop = sealedChunk.chunk.ReadDataAt(p, off, tsNs)
+		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
+	}
+
+	// read from writable chunks last
+	writableChunk, found := up.writableChunks[logicChunkIndex]
+	if !found {
+		return
+	}
+	writableMaxStop := writableChunk.ReadDataAt(p, off, tsNs)
+	glog.V(4).Infof("%s read writable memchunk [%d,%d)", up.filepath, off, writableMaxStop)
+	maxStop = max(maxStop, writableMaxStop)
+
+	return
+}
+
+func (up *UploadPipeline) FlushAll() {
+	up.flushChunks()
+	up.waitForCurrentWritersToComplete()
+}
+
+func (up *UploadPipeline) flushChunks() {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	for logicChunkIndex, memChunk := range up.writableChunks {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) maybeMoveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	if memChunk.IsComplete() {
+		up.moveToSealed(memChunk, logicChunkIndex)
+	}
+}
+
+func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex LogicChunkIndex) {
+	newCount := atomic.AddInt32(&up.uploaderCount, 1)
+	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, newCount-1, newCount)
+
+	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
+		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
+	}
+	sealedChunk := &SealedChunk{
+		chunk: memChunk,
+		// One for the sealedChunks slot, one for the upload below, which
+		// drops its own. Execute() returns before SaveContent reads the
+		// chunk, so a later seal must not free the buffer under it.
+		referenceCounter: 2,
+		accountant:       up.accountant,
+		chunkSize:        up.ChunkSize,
+	}
+	up.sealedChunks[logicChunkIndex] = sealedChunk
+	delete(up.writableChunks, logicChunkIndex)
+
+	// unlock before submitting the uploading jobs
+	up.chunksLock.Unlock()
+	up.uploaders.Execute(func() {
+		// first add to the file chunks
+		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
+
+		// notify waiting process
+		newCount := atomic.AddInt32(&up.uploaderCount, -1)
+		glog.V(4).Infof("%s uploaderCount %d --> %d", up.filepath, newCount+1, newCount)
+		// Lock and Unlock are not required,
+		// but it may signal multiple times during one wakeup,
+		// and the waiting goroutine may miss some of them!
+		up.uploaderCountCond.L.Lock()
+		up.uploaderCountCond.Broadcast()
+		up.uploaderCountCond.L.Unlock()
+
+		// wait for readers
+		up.chunksLock.Lock()
+		defer up.chunksLock.Unlock()
+		for up.IsLocked(logicChunkIndex) {
+			up.readerCountCond.Wait()
+		}
+
+		// then remove from sealed chunks, unless a later seal took over the
+		// index — deleting that one would hide its dirty pages from readers
+		if up.sealedChunks[logicChunkIndex] == sealedChunk {
+			delete(up.sealedChunks, logicChunkIndex)
+			sealedChunk.FreeReference(fmt.Sprintf("%s unindex chunk %d", up.filepath, logicChunkIndex))
+		}
+		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
+
+	})
+	up.chunksLock.Lock()
+}
+
+// EvictOneWritableChunk force-seals one writable chunk so the accountant's
+// Reserve loop can make progress. Strict pass picks the fullest gap-free
+// chunk (issue #9330). Fallback picks the oldest non-empty writer when
+// nothing is gap-free; without it Reserve deadlocks (cond.Wait only wakes
+// on Release, Release only fires on upload completion). Callers must not
+// hold up.chunksLock.
+func (up *UploadPipeline) EvictOneWritableChunk() bool {
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	bestIndex := LogicChunkIndex(-1)
+	var bestBytes int64
+	for lci, wc := range up.writableChunks {
+		if !wc.IsContiguouslyWritten() {
+			continue
+		}
+		if b := wc.WrittenSize(); b > bestBytes {
+			bestIndex = lci
+			bestBytes = b
+		}
+	}
+
+	if bestIndex < 0 {
+		oldestTsNs := int64(math.MaxInt64)
+		var fallbackBytes int64
+		for lci, wc := range up.writableChunks {
+			b := wc.WrittenSize()
+			if b == 0 {
+				continue
+			}
+			ts := wc.LastWriteTsNs()
+			if ts == 0 {
+				continue
+			}
+			if ts < oldestTsNs || (ts == oldestTsNs && b > fallbackBytes) {
+				bestIndex = lci
+				oldestTsNs = ts
+				fallbackBytes = b
+			}
+		}
+	}
+
+	if bestIndex < 0 {
+		return false
+	}
+	up.moveToSealed(up.writableChunks[bestIndex], bestIndex)
+	return true
+}
+
+// ProactiveFlush seals at most one idle writable chunk that is unlikely to
+// receive further writes, submitting it for async upload. Returns true if a
+// chunk was sealed. The caller (ChunkFlusher) invokes this periodically so
+// that partially-written chunks drain continuously instead of piling up
+// until fsync.
+func (up *UploadPipeline) ProactiveFlush(nowNs int64, idleThresholdNs int64, maxHoldNs int64, fillRatio int64, frontierLag int, isSequential bool) bool {
+	if up.concurrentWriterMax <= 0 || atomic.LoadInt32(&up.uploaderCount)*2 >= up.concurrentWriterMax {
+		return false
+	}
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+
+	if len(up.writableChunks) == 0 {
+		return false
+	}
+
+	frontier := atomic.LoadInt64(&up.lastWriteChunkIndex)
+	isSeq := isSequential
+
+	var bestIdx LogicChunkIndex = -1
+	var bestBytes int64 = -1
+
+	for lci, chunk := range up.writableChunks {
+		lastWrite := chunk.LastWriteTsNs()
+		if lastWrite == 0 {
+			continue
+		}
+		age := nowNs - lastWrite
+		if age < idleThresholdNs {
+			continue
+		}
+		// Skip gappy chunks; FUSE writeback may still be in flight for the
+		// hole and SaveContent would bake it into split volume chunks
+		// (issue #9330). Failing here is just a missed flush — FlushAll
+		// at close still seals everything.
+		if !chunk.IsContiguouslyWritten() {
+			continue
+		}
+		written := chunk.WrittenSize()
+		nearlyFull := written >= fillRatio
+		behindFrontier := isSeq && int64(lci) <= frontier-int64(frontierLag)
+		stale := age >= maxHoldNs
+
+		if !nearlyFull && !behindFrontier && !stale {
+			continue
+		}
+		if written > bestBytes {
+			bestIdx = lci
+			bestBytes = written
+		}
+	}
+	if bestIdx < 0 {
+		return false
+	}
+	glog.V(3).Infof("%s proactive flush chunk %d (%d bytes written)", up.filepath, bestIdx, bestBytes)
+	up.moveToSealed(up.writableChunks[bestIdx], bestIdx)
+	return true
+}
+
+func (up *UploadPipeline) Shutdown() {
+	up.swapFile.FreeResource()
+
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	// Free any writable chunks that were reserved but never sealed — on the
+	// Destroy() path (truncate / metadata invalidation) there is no
+	// preceding FlushData(), so dirty writable chunks would otherwise leak
+	// both their memory and their write-budget slots.
+	for logicChunkIndex, writableChunk := range up.writableChunks {
+		glog.V(4).Infof("%s uploadpipeline shutdown writable chunk %d", up.filepath, logicChunkIndex)
+		writableChunk.FreeResource()
+		up.accountant.Release(up.ChunkSize)
+		delete(up.writableChunks, logicChunkIndex)
+	}
+	for logicChunkIndex, sealedChunk := range up.sealedChunks {
+		// only the slot reference; an in-flight upload frees the chunk itself
+		sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
+		delete(up.sealedChunks, logicChunkIndex)
+	}
+}

weed/mount/page_writer/upload_pipeline_lock.go

@@ -0,0 +1,61 @@
+package page_writer
+
+import (
+	"sync/atomic"
+)
+
+func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			up.activeReadChunks[i] = count + 1
+		} else {
+			up.activeReadChunks[i] = 1
+		}
+	}
+}
+
+func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
+	startLogicChunkIndex := LogicChunkIndex(startOffset / up.ChunkSize)
+	stopLogicChunkIndex := LogicChunkIndex(stopOffset / up.ChunkSize)
+	if stopOffset%up.ChunkSize > 0 {
+		stopLogicChunkIndex += 1
+	}
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
+	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
+		if count, found := up.activeReadChunks[i]; found {
+			if count == 1 {
+				delete(up.activeReadChunks, i)
+			} else {
+				up.activeReadChunks[i] = count - 1
+			}
+		}
+	}
+}
+
+func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
+	if count, found := up.activeReadChunks[logicChunkIndex]; found {
+		return count > 0
+	}
+	return false
+}
+
+func (up *UploadPipeline) waitForCurrentWritersToComplete() {
+	up.uploaderCountCond.L.Lock()
+	t := int32(100)
+	for {
+		t = atomic.LoadInt32(&up.uploaderCount)
+		if t <= 0 {
+			break
+		}
+		up.uploaderCountCond.Wait()
+	}
+	up.uploaderCountCond.L.Unlock()
+}

weed/mount/page_writer/upload_pipeline_reseal_test.go

@@ -0,0 +1,121 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// gatedChunk parks in SaveContent until the test opens its gate: the state
+// moveToSealed leaves an upload in, submitted but not yet reading its buffer.
+type gatedChunk struct {
+	gate        chan struct{}
+	saved       chan struct{}
+	freedBefore bool // FreeResource had already run when SaveContent began
+	freeCount   atomic.Int32
+}
+
+func newGatedChunk() *gatedChunk {
+	return &gatedChunk{
+		gate:  make(chan struct{}),
+		saved: make(chan struct{}),
+	}
+}
+
+func (c *gatedChunk) FreeResource()                         { c.freeCount.Add(1) }
+func (c *gatedChunk) WriteDataAt([]byte, int64, int64) int  { return 0 }
+func (c *gatedChunk) ReadDataAt([]byte, int64, int64) int64 { return 0 }
+func (c *gatedChunk) IsComplete() bool                      { return false }
+func (c *gatedChunk) IsContiguouslyWritten() bool           { return true }
+func (c *gatedChunk) ActivityScore() int64                  { return 0 }
+func (c *gatedChunk) WrittenSize() int64                    { return 0 }
+func (c *gatedChunk) LastWriteTsNs() int64                  { return 0 }
+func (c *gatedChunk) SaveContent(saveFn SaveToStorageFunc) {
+	<-c.gate
+	c.freedBefore = c.freeCount.Load() > 0
+	close(c.saved)
+}
+
+// Sealing the same index twice must not free the first chunk: freeing it
+// returns a live buffer to the slot pool and the in-flight upload then ships
+// whatever the next NewMemChunk writes there.
+func TestMoveToSealed_ReplaceKeepsUploadReference(t *testing.T) {
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), 2*1024*1024, nil, 16, "", nil)
+
+	first, second := newGatedChunk(), newGatedChunk()
+
+	up.chunksLock.Lock()
+	up.moveToSealed(first, 0)
+	up.moveToSealed(second, 0) // takes over index 0 while first is uploading
+	up.chunksLock.Unlock()
+
+	// both seals are done, so any premature free has already happened
+	close(first.gate)
+	select {
+	case <-first.saved:
+	case <-time.After(5 * time.Second):
+		t.Fatal("first chunk's SaveContent never ran")
+	}
+	if first.freedBefore {
+		t.Error("first chunk was freed before its upload read it")
+	}
+
+	// a finished upload must leave the newer chunk indexed for readers
+	waitForFree(t, first)
+	up.chunksLock.Lock()
+	indexed := up.sealedChunks[0]
+	up.chunksLock.Unlock()
+	if indexed == nil || indexed.chunk != second {
+		t.Errorf("index 0 should still hold the second chunk, got %v", indexed)
+	}
+
+	close(second.gate)
+	waitForFree(t, second)
+	if got := first.freeCount.Load(); got != 1 {
+		t.Errorf("first chunk freed %d times, want 1", got)
+	}
+	if got := second.freeCount.Load(); got != 1 {
+		t.Errorf("second chunk freed %d times, want 1", got)
+	}
+}
+
+// Shutdown drops only the slot reference; the in-flight upload keeps the
+// chunk alive until it is done, then frees it exactly once.
+func TestShutdown_LeavesInFlightUploadItsReference(t *testing.T) {
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), 2*1024*1024, nil, 16, t.TempDir(), nil)
+
+	chunk := newGatedChunk()
+	up.chunksLock.Lock()
+	up.moveToSealed(chunk, 0)
+	up.chunksLock.Unlock()
+
+	up.Shutdown()
+
+	close(chunk.gate)
+	select {
+	case <-chunk.saved:
+	case <-time.After(5 * time.Second):
+		t.Fatal("SaveContent never ran")
+	}
+	if chunk.freedBefore {
+		t.Error("Shutdown freed the chunk before its upload read it")
+	}
+	waitForFree(t, chunk)
+	if got := chunk.freeCount.Load(); got != 1 {
+		t.Errorf("chunk freed %d times, want 1", got)
+	}
+}
+
+func waitForFree(t *testing.T, c *gatedChunk) {
+	t.Helper()
+	<-c.saved
+	deadline := time.Now().Add(5 * time.Second)
+	for c.freeCount.Load() == 0 {
+		if time.Now().After(deadline) {
+			t.Fatal("chunk was never freed after its upload finished")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}

weed/mount/page_writer/upload_pipeline_shutdown_test.go

@@ -0,0 +1,61 @@
+package page_writer
+
+import (
+	"io"
+	"testing"
+)
+
+// TestShutdown_ReleasesWritableReservation verifies that calling Shutdown
+// on a pipeline with dirty-but-unsealed writable chunks returns those
+// chunks' reservations to the accountant. Regression for the Destroy()
+// path (truncate / metadata invalidation) which never flushes before
+// dropping the pipeline, so writable chunks would otherwise leak their
+// write-budget slots permanently.
+func TestShutdown_ReleasesWritableReservation(t *testing.T) {
+	acc := NewWriteBufferAccountant(1024 * 1024)
+	saveFn := func(io.Reader, int64, int64, int64, func()) {}
+	up := NewUploadPipeline(nil, 64*1024, saveFn, 16, t.TempDir(), acc)
+
+	// A small partial write lands in writableChunks as a mem chunk and
+	// stays there (the chunk is not complete so maybeMoveToSealed is a
+	// no-op) — exactly the state Destroy() can hit.
+	buf := make([]byte, 128)
+	if _, err := up.SaveDataAt(buf, 0, true, 1); err != nil {
+		t.Fatalf("SaveDataAt: %v", err)
+	}
+	if got := acc.Used(); got != 64*1024 {
+		t.Fatalf("expected 64KiB reserved after one write, got %d", got)
+	}
+
+	up.Shutdown()
+
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after Shutdown, got %d (writable reservation leaked)", got)
+	}
+}
+
+// TestSealedChunk_FreeReferenceIsIdempotent verifies that FreeReference
+// releases the accountant slot exactly once even if it is invoked more
+// than once. This guards the path where Shutdown and the async uploader
+// goroutine both call FreeReference on the same sealed chunk: the second
+// call must be a no-op, not a double-release.
+func TestSealedChunk_FreeReferenceIsIdempotent(t *testing.T) {
+	acc := NewWriteBufferAccountant(1024 * 1024)
+	acc.Reserve(64 * 1024)
+
+	sc := &SealedChunk{
+		chunk:            NewMemChunk(0, 64*1024),
+		referenceCounter: 1,
+		accountant:       acc,
+		chunkSize:        64 * 1024,
+	}
+
+	sc.FreeReference("first")
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after first FreeReference, got %d", got)
+	}
+	sc.FreeReference("second")
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after second FreeReference, got %d (double release)", got)
+	}
+}

weed/mount/page_writer/upload_pipeline_test.go

@@ -0,0 +1,260 @@
+package page_writer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestUploadPipeline(t *testing.T) {
+
+	uploadPipeline := NewUploadPipeline(nil, 2*1024*1024, nil, 16, "", nil)
+
+	writeRange(uploadPipeline, 0, 131072)
+	writeRange(uploadPipeline, 131072, 262144)
+	writeRange(uploadPipeline, 262144, 1025536)
+
+	confirmRange(t, uploadPipeline, 0, 1025536)
+
+	writeRange(uploadPipeline, 1025536, 1296896)
+
+	confirmRange(t, uploadPipeline, 1025536, 1296896)
+
+	writeRange(uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+
+	confirmRange(t, uploadPipeline, 1296896, 2162688)
+}
+
+// startOff and stopOff must be divided by 4
+func writeRange(uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff / 4; i < stopOff/4; i += 4 {
+		util.Uint32toBytes(p, uint32(i))
+		uploadPipeline.SaveDataAt(p, i, false, 0)
+	}
+}
+
+func confirmRange(t *testing.T, uploadPipeline *UploadPipeline, startOff, stopOff int64) {
+	p := make([]byte, 4)
+	for i := startOff; i < stopOff/4; i += 4 {
+		uploadPipeline.MaybeReadDataAt(p, i, 0)
+		x := util.BytesToUint32(p)
+		if x != uint32(i) {
+			t.Errorf("expecting %d found %d at offset [%d,%d)", i, x, i, i+4)
+		}
+	}
+}
+
+// Pressure-driven eviction must not seal a chunk with a leading or
+// internal gap (issue #9330).
+func TestEvictOneWritableChunk_SkipsGappyChunks(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	// nil saveToStorage so the async upload SaveContent is a no-op.
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
+	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("EvictOneWritableChunk returned false; expected contiguous chunk 2 to be evictable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
+		t.Errorf("chunk 2 should have moved to sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("chunk 0 (internal gap) must remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 (leading gap) must remain writable")
+	}
+
+	// Filling holes makes IsComplete true; maybeMoveToSealed auto-seals.
+	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
+		t.Fatal(err)
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
+	}
+}
+
+// When every chunk is gappy, the fallback must still seal one so
+// accountant.Reserve can wake; oldest-LastWriteTsNs wins.
+func TestEvictOneWritableChunk_FallbackPicksOldestGappy(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// Strictly increasing tsNs so chunk 0 is oldest; equal WrittenSize.
+	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 200); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 201); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 300); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+3*cs/4, true, 301); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("fallback must seal a gappy chunk to free a Reserve slot")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("oldest gappy chunk (0) should have been picked by the fallback")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 should remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; !stillWritable {
+		t.Errorf("chunk 2 should remain writable")
+	}
+}
+
+// Strict pass preempts the fallback even when a gappy chunk is older.
+func TestEvictOneWritableChunk_PrefersStrictOverFallback(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0 (older, gappy), chunk 1 (newer, contiguous from 0).
+	if _, err := up.SaveDataAt(block, 0, true, 100); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 101); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 200); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 201); err != nil {
+		t.Fatal(err)
+	}
+
+	if !up.EvictOneWritableChunk() {
+		t.Fatalf("EvictOneWritableChunk returned false")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("contiguous chunk 1 must be picked over older gappy chunk 0")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("gappy chunk 0 must remain writable")
+	}
+}
+
+// ProactiveFlush also skips gappy chunks (issue #9330). No liveness
+// fallback here — unlike EvictOneWritableChunk, returning false is just
+// a missed optimization.
+func TestProactiveFlush_SkipsGappyChunks(t *testing.T) {
+	const cs int64 = 2 * 1024 * 1024
+	up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(2), cs, nil, 16, "", nil)
+
+	block := make([]byte, cs/4)
+
+	// chunk 0: internal gap; chunk 1: leading gap; chunk 2: contiguous from 0.
+	if _, err := up.SaveDataAt(block, 0, true, 1); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 3*cs/4, true, 2); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/4, true, 3); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+cs/2, true, 4); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs, true, 5); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, 2*cs+cs/4, true, 6); err != nil {
+		t.Fatal(err)
+	}
+
+	// nowNs >> chunk timestamps so age clears the idle/maxHold gates;
+	// fillRatio < WrittenSize so nearlyFull is true. Leaves
+	// IsContiguouslyWritten as the only differentiator.
+	const (
+		nowNs           int64 = 1_000_000_000
+		idleThresholdNs int64 = 100
+		maxHoldNs       int64 = 200
+		fillRatio       int64 = cs / 8
+	)
+	if !up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
+		t.Fatalf("ProactiveFlush returned false; expected contiguous chunk 2 to be sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(2)]; stillWritable {
+		t.Errorf("chunk 2 should have moved to sealed")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; !stillWritable {
+		t.Errorf("chunk 0 (internal gap) must remain writable")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; !stillWritable {
+		t.Errorf("chunk 1 (leading gap) must remain writable")
+	}
+
+	if up.ProactiveFlush(nowNs, idleThresholdNs, maxHoldNs, fillRatio, 0, false) {
+		t.Errorf("ProactiveFlush returned true with only gappy chunks left")
+	}
+
+	if _, err := up.SaveDataAt(block, cs/4, true, 7); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs/2, true, 8); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs, true, 9); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := up.SaveDataAt(block, cs+3*cs/4, true, 10); err != nil {
+		t.Fatal(err)
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(0)]; stillWritable {
+		t.Errorf("chunk 0 should have auto-sealed after gap was filled")
+	}
+	if _, stillWritable := up.writableChunks[LogicChunkIndex(1)]; stillWritable {
+		t.Errorf("chunk 1 should have auto-sealed after leading range was filled")
+	}
+}

weed/mount/page_writer/write_buffer_accountant.go

@@ -0,0 +1,180 @@
+package page_writer
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	// softThresholdRatio is the fraction of cap at which soft throttling begins.
+	// At this level, writes sleep briefly to let uploaders drain.
+	softThresholdRatio = 0.8
+	// hardThresholdRatio is the fraction of cap at which hard throttling kicks in.
+	// Writes sleep longer to aggressively slow intake.
+	hardThresholdRatio = 0.95
+	// softThrottleDelay is the sleep duration when usage exceeds the soft threshold.
+	softThrottleDelay = 10 * time.Millisecond
+	// hardThrottleDelay is the sleep duration when usage exceeds the hard threshold.
+	hardThrottleDelay = 50 * time.Millisecond
+)
+
+// WriteBufferAccountant enforces a global byte budget across all
+// UploadPipeline instances. Callers Reserve chunk-sized slots before
+// allocating a page chunk (in memory or on swap) and Release them when
+// the chunk is freed. Reserve blocks when the cap would be exceeded,
+// providing natural backpressure to the FUSE write path when volume
+// uploads stall (e.g. all assigned volumes are full) instead of letting
+// the swap file grow without bound.
+//
+// A nil receiver is treated as "unlimited" for backward compatibility.
+type WriteBufferAccountant struct {
+	mu            sync.Mutex
+	cond          *sync.Cond
+	cap           int64 // 0 means unlimited
+	used          int64
+	softThreshold int64 // pre-computed: cap * softThresholdRatio
+	hardThreshold int64 // pre-computed: cap * hardThresholdRatio
+	evictor       func(needBytes int64) bool
+	evicting      bool
+
+	softThrottleCount atomic.Int64
+	hardThrottleCount atomic.Int64
+}
+
+func NewWriteBufferAccountant(capBytes int64) *WriteBufferAccountant {
+	a := &WriteBufferAccountant{
+		cap:           capBytes,
+		softThreshold: int64(float64(capBytes) * softThresholdRatio),
+		hardThreshold: int64(float64(capBytes) * hardThresholdRatio),
+	}
+	a.cond = sync.NewCond(&a.mu)
+	return a
+}
+
+// SetEvictor registers a callback that Reserve invokes when the cap would
+// otherwise block. The evictor is expected to force-seal at least one
+// writable chunk in some UploadPipeline, which turns a pinned-forever
+// writable chunk into a sealed chunk that the async uploader drains and
+// Releases. Without this hook, workloads that hold many files open for
+// write with less-than-chunkSize data in each (e.g. fio 4k randwrite with
+// nrfiles * chunkSize > cap) deadlock permanently, because writable chunks
+// only seal on close or when they fill.
+//
+// The evictor must not call Reserve on the same accountant or re-enter
+// Reserve transitively — it would deadlock on accountant.mu.
+func (a *WriteBufferAccountant) SetEvictor(fn func(needBytes int64) bool) {
+	if a == nil {
+		return
+	}
+	a.mu.Lock()
+	a.evictor = fn
+	a.mu.Unlock()
+}
+
+// Reserve blocks until n bytes can be accounted for under the cap.
+// It must not be called while holding any UploadPipeline lock, or the
+// uploader goroutines that eventually call Release will deadlock.
+func (a *WriteBufferAccountant) Reserve(n int64) {
+	if a == nil || a.cap <= 0 {
+		return
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	// Graduated backpressure: slow writers before hitting the hard cap.
+	// Use projected usage (used + n) so that a single reservation crossing
+	// a threshold is throttled immediately rather than on the next call.
+	//
+	// This runs once per Reserve, not inside the blocking loop below.
+	// Once usage reaches the cap, the evictor + cond.Wait is the correct
+	// mechanism: it blocks until a sealed chunk upload frees space, which
+	// is strictly more efficient than repeated time-based sleeps that
+	// cannot free capacity on their own.
+	projected := a.used + n
+	if projected >= a.hardThreshold && a.used > 0 {
+		a.hardThrottleCount.Add(1)
+		a.mu.Unlock()
+		time.Sleep(hardThrottleDelay)
+		a.mu.Lock()
+	} else if projected >= a.softThreshold && a.used > 0 {
+		a.softThrottleCount.Add(1)
+		a.mu.Unlock()
+		time.Sleep(softThrottleDelay)
+		a.mu.Lock()
+	}
+
+	for a.used+n > a.cap && a.used > 0 {
+		// Before blocking, try to force-seal a writable chunk somewhere so
+		// its async upload path will eventually Release a slot. Single-flight
+		// on `evicting` so a stampede of blocked reservers doesn't iterate
+		// the fhMap concurrently.
+		if a.evictor != nil && !a.evicting {
+			a.runEvictorLocked(n)
+			// A concurrent Release may have brought used back under the
+			// cap during the evict window — re-check before waiting so we
+			// do not block on a broadcast that has already fired.
+			if a.used+n <= a.cap || a.used == 0 {
+				break
+			}
+		}
+		a.cond.Wait()
+	}
+	a.used += n
+}
+
+// runEvictorLocked is called with a.mu held and !a.evicting. It drops the
+// lock around the evictor invocation so uploader goroutines (which call
+// Release under the same lock) can make progress, and uses defer to
+// guarantee the `evicting` flag and the lock are restored even if the
+// evictor panics.
+func (a *WriteBufferAccountant) runEvictorLocked(n int64) {
+	evictor := a.evictor
+	a.evicting = true
+	a.mu.Unlock()
+	defer func() {
+		a.mu.Lock()
+		a.evicting = false
+		a.cond.Broadcast()
+	}()
+	evictor(n)
+}
+
+func (a *WriteBufferAccountant) Release(n int64) {
+	if a == nil || a.cap <= 0 {
+		return
+	}
+	a.mu.Lock()
+	a.used -= n
+	if a.used < 0 {
+		a.used = 0
+	}
+	a.cond.Broadcast()
+	a.mu.Unlock()
+}
+
+// Used returns the currently reserved byte count (for tests/metrics).
+func (a *WriteBufferAccountant) Used() int64 {
+	if a == nil || a.cap <= 0 {
+		return 0
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	return a.used
+}
+
+// SoftThrottleCount returns the number of times soft throttling was triggered.
+func (a *WriteBufferAccountant) SoftThrottleCount() int64 {
+	if a == nil {
+		return 0
+	}
+	return a.softThrottleCount.Load()
+}
+
+// HardThrottleCount returns the number of times hard throttling was triggered.
+func (a *WriteBufferAccountant) HardThrottleCount() int64 {
+	if a == nil {
+		return 0
+	}
+	return a.hardThrottleCount.Load()
+}

weed/mount/page_writer/write_buffer_accountant_test.go

@@ -0,0 +1,199 @@
+package page_writer
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestWriteBufferAccountant_Unlimited(t *testing.T) {
+	a := NewWriteBufferAccountant(0)
+	a.Reserve(1 << 30)
+	a.Reserve(1 << 30)
+	if got := a.Used(); got != 0 {
+		t.Fatalf("unlimited accountant should not track usage, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_Nil(t *testing.T) {
+	var a *WriteBufferAccountant
+	a.Reserve(100) // must not panic
+	a.Release(100)
+	if a.Used() != 0 {
+		t.Fatal("nil Used should return 0")
+	}
+}
+
+func TestWriteBufferAccountant_ReserveAndRelease(t *testing.T) {
+	a := NewWriteBufferAccountant(300)
+	a.Reserve(100)
+	a.Reserve(100)
+	if got := a.Used(); got != 200 {
+		t.Fatalf("expected 200 used, got %d", got)
+	}
+	a.Release(100)
+	if got := a.Used(); got != 100 {
+		t.Fatalf("expected 100 used after release, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_BlocksWhenOverCap(t *testing.T) {
+	a := NewWriteBufferAccountant(100)
+	a.Reserve(100)
+
+	started := make(chan struct{})
+	var landed atomic.Bool
+	go func() {
+		close(started)
+		a.Reserve(50) // should block — used=100, cap=100
+		landed.Store(true)
+	}()
+
+	// Wait until the goroutine is about to enter Reserve. Once it does,
+	// Reserve cannot make progress until Release is called (cap is
+	// full), so landed is guaranteed to stay false until we release.
+	<-started
+	if landed.Load() {
+		t.Fatal("second reserve should be blocked while cap is full")
+	}
+
+	a.Release(100)
+
+	deadline := time.Now().Add(time.Second)
+	for !landed.Load() && time.Now().Before(deadline) {
+		time.Sleep(5 * time.Millisecond)
+	}
+	if !landed.Load() {
+		t.Fatal("second reserve should unblock after release")
+	}
+	if got := a.Used(); got != 50 {
+		t.Fatalf("expected 50 used after handoff, got %d", got)
+	}
+}
+
+func TestWriteBufferAccountant_SoftThrottle(t *testing.T) {
+	// Cap = 1000, soft threshold = 800. Reserve 850 (85%) then measure
+	// that the next Reserve is delayed by at least softThrottleDelay.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(850)
+
+	start := time.Now()
+	a.Reserve(10) // should trigger soft throttle (used=850 > 800)
+	elapsed := time.Since(start)
+
+	if elapsed < softThrottleDelay {
+		t.Fatalf("expected Reserve to take >= %v (soft throttle), took %v", softThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_HardThrottle(t *testing.T) {
+	// Cap = 1000, hard threshold = 950. Reserve 960 (96%) then measure
+	// that the next Reserve is delayed by at least hardThrottleDelay.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(960)
+
+	start := time.Now()
+	a.Reserve(10) // should trigger hard throttle (used=960 > 950)
+	elapsed := time.Since(start)
+
+	if elapsed < hardThrottleDelay {
+		t.Fatalf("expected Reserve to take >= %v (hard throttle), took %v", hardThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_NoThrottleBelowSoft(t *testing.T) {
+	// Cap = 1000, soft threshold = 800. Reserve 700 (70%) -- below soft
+	// threshold, so subsequent Reserve should return quickly.
+	a := NewWriteBufferAccountant(1000)
+	a.Reserve(700)
+
+	start := time.Now()
+	a.Reserve(10)
+	elapsed := time.Since(start)
+
+	if elapsed >= softThrottleDelay {
+		t.Fatalf("expected Reserve below soft threshold to avoid throttle delay (< %v), took %v", softThrottleDelay, elapsed)
+	}
+}
+
+func TestWriteBufferAccountant_ThrottleCounters(t *testing.T) {
+	a := NewWriteBufferAccountant(1000)
+
+	// Below soft -- no throttle
+	a.Reserve(700)
+	a.Reserve(10)
+	if a.SoftThrottleCount() != 0 || a.HardThrottleCount() != 0 {
+		t.Fatalf("expected no throttle counts below soft threshold, got soft=%d hard=%d",
+			a.SoftThrottleCount(), a.HardThrottleCount())
+	}
+
+	// Release back down, then fill to soft zone
+	a.Release(710)
+	a.Reserve(850) // now used=850, above soft (800)
+	a.Reserve(10)  // triggers soft throttle
+	if a.SoftThrottleCount() != 1 {
+		t.Fatalf("expected soft throttle count 1, got %d", a.SoftThrottleCount())
+	}
+	if a.HardThrottleCount() != 0 {
+		t.Fatalf("expected hard throttle count 0, got %d", a.HardThrottleCount())
+	}
+
+	// Release and fill to hard zone
+	a.Release(860)
+	a.Reserve(960) // now used=960, above hard (950)
+	a.Reserve(10)  // triggers hard throttle
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count 1, got %d", a.HardThrottleCount())
+	}
+}
+
+func TestWriteBufferAccountant_GraduatedRecovery(t *testing.T) {
+	// Fill to hard threshold, verify hard throttle. Release to soft zone,
+	// verify that throttle level decreases to soft.
+	a := NewWriteBufferAccountant(1000)
+
+	// Fill to hard zone (960 > 950)
+	a.Reserve(960)
+	start := time.Now()
+	a.Reserve(10) // hard throttle
+	elapsed := time.Since(start)
+	if elapsed < hardThrottleDelay {
+		t.Fatalf("expected hard throttle delay, got %v", elapsed)
+	}
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count 1, got %d", a.HardThrottleCount())
+	}
+
+	// Release down to soft zone: used = 970 - 130 = 840 (> 800 soft, < 950 hard)
+	a.Release(130)
+
+	start = time.Now()
+	a.Reserve(10) // should now be soft throttle
+	elapsed = time.Since(start)
+	if elapsed < softThrottleDelay {
+		t.Fatalf("expected soft throttle delay after recovery, got %v", elapsed)
+	}
+	if a.HardThrottleCount() != 1 {
+		t.Fatalf("expected hard throttle count to remain 1 after recovery, got %d", a.HardThrottleCount())
+	}
+	if a.SoftThrottleCount() != 1 {
+		t.Fatalf("expected soft throttle count 1 after recovery, got %d", a.SoftThrottleCount())
+	}
+}
+
+func TestWriteBufferAccountant_AllowsOversizeWhenEmpty(t *testing.T) {
+	// A single request larger than cap must succeed when nothing is in
+	// flight, otherwise a single file handle with a large chunk size would
+	// deadlock forever.
+	a := NewWriteBufferAccountant(100)
+	done := make(chan struct{})
+	go func() {
+		a.Reserve(500)
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second):
+		t.Fatal("oversize reserve on empty accountant should not block")
+	}
+}

weed/mount/page_writer/write_buffer_cap_integration_test.go

@@ -0,0 +1,175 @@
+package page_writer
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// TestWriteBufferCap_SharedAcrossPipelines exercises the full write-budget
+// plumbing end-to-end: one WriteBufferAccountant shared across several
+// UploadPipeline instances (modeling multiple open file handles in a
+// single mount), a deliberately stalled saveFn (modeling the reported
+// "volume server rejecting uploads" condition), and many concurrent
+// writers.  Without the cap this scenario is what filled /tmp to 1.8 TB;
+// with the cap, Used() must never exceed the configured budget and
+// writers must backpressure instead of allocating unboundedly.
+func TestWriteBufferCap_SharedAcrossPipelines(t *testing.T) {
+	const (
+		chunkSize    = 64 * 1024
+		capChunks    = 4 // global budget: 4 chunks across all pipelines
+		numPipelines = 3
+		numWriters   = 6
+		writesEach   = 8 // more than enough to exceed capChunks if unbounded
+	)
+	capBytes := int64(capChunks * chunkSize)
+	acc := NewWriteBufferAccountant(capBytes)
+
+	// Gate every upload so sealed chunks cannot drain until the test says
+	// so — this is what reproduces the "stalled volume server" scenario.
+	var inflight atomic.Int64
+	var maxInflight atomic.Int64
+	release := make(chan struct{})
+	saveFn := func(_ io.Reader, _, _, _ int64, cleanup func()) {
+		cur := inflight.Add(1)
+		for {
+			prev := maxInflight.Load()
+			if cur <= prev || maxInflight.CompareAndSwap(prev, cur) {
+				break
+			}
+		}
+		<-release
+		inflight.Add(-1)
+		if cleanup != nil {
+			cleanup()
+		}
+	}
+
+	// Sample Used() frequently on a background goroutine so we can catch
+	// any moment the accountant exceeds the cap.
+	var observedMax atomic.Int64
+	stopSampling := make(chan struct{})
+	var samplerWg sync.WaitGroup
+	samplerWg.Add(1)
+	go func() {
+		defer samplerWg.Done()
+		for {
+			select {
+			case <-stopSampling:
+				return
+			default:
+			}
+			u := acc.Used()
+			for {
+				prev := observedMax.Load()
+				if u <= prev || observedMax.CompareAndSwap(prev, u) {
+					break
+				}
+			}
+			time.Sleep(time.Millisecond)
+		}
+	}()
+
+	// Build N pipelines, each with its own uploader executor so chunks
+	// can actually enter the upload stage and hit the gated saveFn.
+	pipelines := make([]*UploadPipeline, numPipelines)
+	for i := range pipelines {
+		up := NewUploadPipeline(util.NewLimitedConcurrentExecutor(4), chunkSize, saveFn, 2, t.TempDir(), acc)
+		pipelines[i] = up
+	}
+
+	// Launch numWriters goroutines, round-robin across pipelines. Each
+	// writer writes writesEach full chunks, on distinct logical chunk
+	// indices so every write triggers a new reservation.
+	var writerWg sync.WaitGroup
+	writerStart := make(chan struct{})
+	writerWg.Add(numWriters)
+	for w := 0; w < numWriters; w++ {
+		w := w
+		go func() {
+			defer writerWg.Done()
+			<-writerStart
+			up := pipelines[w%numPipelines]
+			data := make([]byte, chunkSize)
+			for k := 0; k < writesEach; k++ {
+				// Offset each writer into a distinct region of the file
+				// so chunks don't collide on the same logicChunkIndex.
+				off := int64(w)*int64(writesEach)*chunkSize + int64(k)*chunkSize
+				if _, err := up.SaveDataAt(data, off, true, int64(k+1)); err != nil {
+					t.Errorf("writer %d: SaveDataAt: %v", w, err)
+					return
+				}
+			}
+		}()
+	}
+	close(writerStart)
+
+	// Give the writers time to pile up against the cap. Any unbounded
+	// allocation would show up as Used() > capBytes; the sampler catches
+	// that. Writers should be blocked on Reserve well before writesEach
+	// chunks are created per pipeline.
+	deadline := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		if acc.Used() >= capBytes {
+			break
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+	if got := acc.Used(); got < capBytes/2 {
+		t.Fatalf("expected writers to fill the budget, used=%d cap=%d", got, capBytes)
+	}
+
+	// While the gate is closed and writers are blocked, Used() must stay
+	// at or below the cap. Sample explicitly for a short window to catch
+	// any overshoot.
+	for i := 0; i < 50; i++ {
+		if u := acc.Used(); u > capBytes {
+			t.Fatalf("Used()=%d exceeded cap=%d while writers were stalled", u, capBytes)
+		}
+		time.Sleep(2 * time.Millisecond)
+	}
+
+	// Open the gate: every sealed chunk completes, writers drain.
+	close(release)
+
+	// Wait for all writers to finish.
+	writerDone := make(chan struct{})
+	go func() {
+		writerWg.Wait()
+		close(writerDone)
+	}()
+	select {
+	case <-writerDone:
+	case <-time.After(10 * time.Second):
+		close(stopSampling)
+		samplerWg.Wait()
+		t.Fatalf("writers did not drain after release; used=%d", acc.Used())
+	}
+
+	// Shut down all pipelines and stop sampling.
+	for _, up := range pipelines {
+		up.Shutdown()
+	}
+	close(stopSampling)
+	samplerWg.Wait()
+
+	if got := observedMax.Load(); got > capBytes {
+		t.Fatalf("observed Used()=%d exceeded cap=%d", got, capBytes)
+	}
+	// An uploader goroutine releases its budget slot only after reacquiring
+	// chunksLock post-Shutdown, so the last releases may land asynchronously.
+	drainDeadline := time.Now().Add(5 * time.Second)
+	for acc.Used() != 0 && time.Now().Before(drainDeadline) {
+		time.Sleep(2 * time.Millisecond)
+	}
+	if got := acc.Used(); got != 0 {
+		t.Fatalf("expected 0 used after shutdown, got %d", got)
+	}
+
+	t.Logf("peak inflight uploads=%d, peak Used()=%d bytes (cap=%d)",
+		maxInflight.Load(), observedMax.Load(), capBytes)
+}