snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/needle/README.md

@@ -0,0 +1,160 @@
+# Needle Layout Format
+
+This document describes the binary layout of the Needle structure as used in SeaweedFS storage, for all supported versions (v1, v2, v3).
+
+A Needle represents a file or data blob stored in a volume file. The layout determines how the Needle is serialized to disk for efficient storage and retrieval.
+
+---
+
+## Common Field Sizes
+
+| Field         | Size (bytes) |
+|---------------|--------------|
+| Cookie        | 4            |
+| NeedleId      | 8            |
+| Size          | 4            |
+| DataSize      | 4            |
+| Flags         | 1            |
+| NameSize      | 1            |
+| MimeSize      | 1            |
+| LastModified  | 5            |
+| Ttl           | 2            |
+| PairsSize     | 2            |
+| Checksum      | 4            |
+| Timestamp     | 8            |
+
+---
+
+## Needle Layouts by Version
+
+### Version 1
+
+| Offset   | Field    | Size (bytes) | Description                                   |
+|----------|----------|--------------|-----------------------------------------------|
+| 0        | Cookie   | 4            | Random number to mitigate brute force lookups |
+| 4        | Id       | 8            | Needle ID                                     |
+| 12       | Size     | 4            | Length of Data                                |
+| 16       | Data     | N            | File data (N = Size)                          |
+| 16+N     | Checksum | 4            | CRC32 of Data                                 |
+| 20+N     | Padding  | 0-7          | To align to 8 bytes                           |
+
+### Version 2
+
+| Offset   | Field        | Size (bytes) | Description                                   |
+|----------|-------------|--------------|-----------------------------------------------|
+| 0        | Cookie      | 4            | Random number                                 |
+| 4        | Id          | 8            | Needle ID                                     |
+| 12       | Size        | 4            | Total size of the following fields            |
+| 16       | DataSize    | 4            | Length of Data (N)                            |
+| 20       | Data        | N            | File data                                     |
+| 20+N     | Flags       | 1            | Bit flags                                     |
+| 21+N     | NameSize    | 1 (opt)      | Optional, if present                          |
+| 22+N     | Name        | M (opt)      | Optional, if present (M = NameSize)           |
+| ...      | MimeSize    | 1 (opt)      | Optional, if present                          |
+| ...      | Mime        | K (opt)      | Optional, if present (K = MimeSize)           |
+| ...      | LastModified| 5 (opt)      | Optional, if present                          |
+| ...      | Ttl         | 2 (opt)      | Optional, if present                          |
+| ...      | PairsSize   | 2 (opt)      | Optional, if present                          |
+| ...      | Pairs       | P (opt)      | Optional, if present (P = PairsSize)          |
+| ...      | Checksum    | 4            | CRC32                                         |
+| ...      | Padding     | 0-7          | To align to 8 bytes                           |
+
+### Version 3
+
+| Offset   | Field        | Size (bytes) | Description                                   |
+|----------|-------------|--------------|-----------------------------------------------|
+| 0        | Cookie      | 4            | Random number                                 |
+| 4        | Id          | 8            | Needle ID                                     |
+| 12       | Size        | 4            | Total size of the following fields            |
+| 16       | DataSize    | 4            | Length of Data (N)                            |
+| 20       | Data        | N            | File data                                     |
+| 20+N     | Flags       | 1            | Bit flags                                     |
+| 21+N     | NameSize    | 1 (opt)      | Optional, if present                          |
+| 22+N     | Name        | M (opt)      | Optional, if present (M = NameSize)           |
+| ...      | MimeSize    | 1 (opt)      | Optional, if present                          |
+| ...      | Mime        | K (opt)      | Optional, if present (K = MimeSize)           |
+| ...      | LastModified| 5 (opt)      | Optional, if present                          |
+| ...      | Ttl         | 2 (opt)      | Optional, if present                          |
+| ...      | PairsSize   | 2 (opt)      | Optional, if present                          |
+| ...      | Pairs       | P (opt)      | Optional, if present (P = PairsSize)          |
+| ...      | Checksum    | 4            | CRC32                                         |
+| ...      | Timestamp   | 8            | Append time in nanoseconds                    |
+| ...      | Padding     | 0-7          | To align to 8 bytes                           |
+
+- Offsets marked with `...` depend on the presence and size of previous optional fields.
+- Fields marked `(opt)` are optional and only present if the corresponding size or flag is non-zero.
+- N = DataSize, M = NameSize, K = MimeSize, P = PairsSize.
+
+---
+
+## Field Explanations
+
+- **Cookie**: 4 bytes, random value for security.
+- **Id**: 8 bytes, unique identifier for the Needle.
+- **Size**: 4 bytes, total size of the Needle data section (not including header, checksum, timestamp, or padding).
+- **DataSize**: 4 bytes, length of the Data field.
+- **Data**: File data (variable length).
+- **Flags**: 1 byte, bit flags for Needle properties.
+- **NameSize/Name**: 1 byte + variable, optional file name.
+- **MimeSize/Mime**: 1 byte + variable, optional MIME type.
+- **LastModified**: 5 bytes, optional last modified timestamp.
+- **Ttl**: 2 bytes, optional time-to-live.
+- **PairsSize/Pairs**: 2 bytes + variable, optional key-value pairs.
+- **Checksum**: 4 bytes, CRC32 checksum of the Needle data.
+- **Timestamp**: 8 bytes, append time (only in v3).
+- **Padding**: 0-7 bytes, to align the total Needle size to 8 bytes.
+
+---
+
+## Version Comparison Table
+
+| Field         | v1 | v2 | v3 |
+|---------------|----|----|----|
+| Cookie        | ✔  | ✔  | ✔  |
+| Id            | ✔  | ✔  | ✔  |
+| Size          | ✔  | ✔  | ✔  |
+| DataSize      |    | ✔  | ✔  |
+| Data          | ✔  | ✔  | ✔  |
+| Flags         |    | ✔  | ✔  |
+| NameSize/Name |    | ✔  | ✔  |
+| MimeSize/Mime |    | ✔  | ✔  |
+| LastModified  |    | ✔  | ✔  |
+| Ttl           |    | ✔  | ✔  |
+| PairsSize/Pairs|   | ✔  | ✔  |
+| Checksum      | ✔  | ✔  | ✔  |
+| Timestamp     |    |    | ✔  |
+| Padding       | ✔  | ✔  | ✔  |
+
+---
+
+## Flags Field Details
+
+The `Flags` field (present in v2 and v3) is a bitmask that encodes several boolean properties of the Needle. Each bit has a specific meaning:
+
+| Bit Value | Name                  | Meaning                                                      |
+|-----------|-----------------------|--------------------------------------------------------------|
+| 0x01      | FlagIsCompressed      | Data is compressed (isCompressed)                            |
+| 0x02      | FlagHasName           | Name field is present (NameSize/Name)                        |
+| 0x04      | FlagHasMime           | Mime field is present (MimeSize/Mime)                        |
+| 0x08      | FlagHasLastModifiedDate| LastModified field is present                                |
+| 0x10      | FlagHasTtl            | Ttl field is present                                         |
+| 0x20      | FlagHasPairs          | Pairs field is present (PairsSize/Pairs)                     |
+| 0x80      | FlagIsChunkManifest   | Data is a chunk manifest (for large files)                   |
+
+- If a flag is set, the corresponding field(s) will appear in the Needle layout at the appropriate position.
+- The `Flags` field is always present in v2 and v3, immediately after the Data field.
+
+## Optional Fields
+
+- Fields marked as optional in the layout tables are only present if the corresponding flag in the `Flags` field is set (except for Name/Mime/Pairs, which also depend on their size fields being non-zero).
+- The order of optional fields is fixed and matches the order of their flags.
+
+## Special Notes
+
+- **isCompressed**: If set, the Data field is compressed (typically using gzip). This is indicated by the lowest bit (0x01) in the Flags byte.
+- **isChunkManifest**: If set, the Data field contains a manifest describing chunks of a large file, not raw file data.
+- **All multi-byte fields** are stored in big-endian order.
+- **Padding** is always added at the end to align the total Needle size to 8 bytes.
+- **N = DataSize, M = NameSize, K = MimeSize, P = PairsSize** in the layout tables above.
+
+For more details, see the implementation in the corresponding Go files in this directory. 

weed/storage/needle/async_request.go

@@ -0,0 +1,53 @@
+package needle
+
+type AsyncRequest struct {
+	N              *Needle
+	IsWriteRequest bool
+	ActualSize     int64
+	offset         uint64
+	size           uint64
+	doneChan       chan interface{}
+	isUnchanged    bool
+	err            error
+}
+
+func NewAsyncRequest(n *Needle, isWriteRequest bool) *AsyncRequest {
+	return &AsyncRequest{
+		offset:         0,
+		size:           0,
+		ActualSize:     0,
+		doneChan:       make(chan interface{}),
+		N:              n,
+		isUnchanged:    false,
+		IsWriteRequest: isWriteRequest,
+		err:            nil,
+	}
+}
+
+func (r *AsyncRequest) WaitComplete() (uint64, uint64, bool, error) {
+	<-r.doneChan
+	return r.offset, r.size, r.isUnchanged, r.err
+}
+
+func (r *AsyncRequest) Complete(offset uint64, size uint64, isUnchanged bool, err error) {
+	r.offset = offset
+	r.size = size
+	r.isUnchanged = isUnchanged
+	r.err = err
+	close(r.doneChan)
+}
+
+func (r *AsyncRequest) UpdateResult(offset uint64, size uint64, isUnchanged bool, err error) {
+	r.offset = offset
+	r.size = size
+	r.isUnchanged = isUnchanged
+	r.err = err
+}
+
+func (r *AsyncRequest) Submit() {
+	close(r.doneChan)
+}
+
+func (r *AsyncRequest) IsSucceed() bool {
+	return r.err == nil
+}

weed/storage/needle/crc.go

@@ -0,0 +1,38 @@
+package needle
+
+import (
+	"fmt"
+	"io"
+
+	"hash/crc32"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+var table = crc32.MakeTable(crc32.Castagnoli)
+
+type CRC uint32
+
+func NewCRC(b []byte) CRC {
+	return CRC(0).Update(b)
+}
+
+func (c CRC) Update(b []byte) CRC {
+	return CRC(crc32.Update(uint32(c), table, b))
+}
+
+// Value Deprecated. Just use the raw uint32 value to compare.
+func (c CRC) Value() uint32 {
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}
+
+func (n *Needle) Etag() string {
+	bits := make([]byte, 4)
+	util.Uint32toBytes(bits, uint32(n.Checksum))
+	return fmt.Sprintf("%x", bits)
+}
+
+type CRCwriter struct {
+	crc CRC
+	w   io.Writer
+}

weed/storage/needle/file_id.go

@@ -0,0 +1,81 @@
+package needle
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strings"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+type FileId struct {
+	VolumeId VolumeId
+	Key      NeedleId
+	Cookie   Cookie
+}
+
+func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId {
+	return &FileId{VolumeId: VolumeId, Key: n.Id, Cookie: n.Cookie}
+}
+
+func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId {
+	return &FileId{VolumeId: VolumeId, Key: Uint64ToNeedleId(key), Cookie: Uint32ToCookie(cookie)}
+}
+
+// Deserialize the file id
+func ParseFileIdFromString(fid string) (*FileId, error) {
+	vid, needleKeyCookie, err := splitVolumeId(fid)
+	if err != nil {
+		return nil, err
+	}
+	volumeId, err := NewVolumeId(vid)
+	if err != nil {
+		return nil, err
+	}
+
+	nid, cookie, err := ParseNeedleIdCookie(needleKeyCookie)
+	if err != nil {
+		return nil, err
+	}
+	fileId := &FileId{VolumeId: volumeId, Key: nid, Cookie: cookie}
+	return fileId, nil
+}
+
+func (n *FileId) GetVolumeId() VolumeId {
+	return n.VolumeId
+}
+
+func (n *FileId) GetNeedleId() NeedleId {
+	return n.Key
+}
+
+func (n *FileId) GetCookie() Cookie {
+	return n.Cookie
+}
+
+func (n *FileId) GetNeedleIdCookie() string {
+	return formatNeedleIdCookie(n.Key, n.Cookie)
+}
+
+func (n *FileId) String() string {
+	return n.VolumeId.String() + "," + formatNeedleIdCookie(n.Key, n.Cookie)
+}
+
+func formatNeedleIdCookie(key NeedleId, cookie Cookie) string {
+	bytes := make([]byte, NeedleIdSize+CookieSize)
+	NeedleIdToBytes(bytes[0:NeedleIdSize], key)
+	CookieToBytes(bytes[NeedleIdSize:NeedleIdSize+CookieSize], cookie)
+	nonzero_index := 0
+	for ; bytes[nonzero_index] == 0 && nonzero_index < NeedleIdSize; nonzero_index++ {
+	}
+	return hex.EncodeToString(bytes[nonzero_index:])
+}
+
+// copied from operation/delete_content.go, to cut off cycle dependency
+func splitVolumeId(fid string) (vid string, key_cookie string, err error) {
+	commaIndex := strings.Index(fid, ",")
+	if commaIndex <= 0 {
+		return "", "", fmt.Errorf("wrong fid format")
+	}
+	return fid[:commaIndex], fid[commaIndex+1:], nil
+}

weed/storage/needle/file_id_test.go

@@ -0,0 +1,82 @@
+package needle
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestParseFileIdFromString(t *testing.T) {
+	fidStr1 := "100,12345678"
+	_, err := ParseFileIdFromString(fidStr1)
+	if err == nil {
+		t.Errorf("%s : KeyHash is too short", fidStr1)
+	}
+
+	fidStr1 = "100, 12345678"
+	_, err = ParseFileIdFromString(fidStr1)
+	if err == nil {
+		t.Errorf("%s : needleId invalid syntax", fidStr1)
+	}
+
+	fidStr1 = "100,123456789"
+	_, err = ParseFileIdFromString(fidStr1)
+	if err != nil {
+		t.Errorf("%s : should be OK", fidStr1)
+	}
+
+	var fileId *FileId
+	fidStr1 = "100,123456789012345678901234"
+	fileId, err = ParseFileIdFromString(fidStr1)
+	if err != nil {
+		t.Errorf("%s : should be OK", fidStr1)
+	}
+	if !(fileId.VolumeId == VolumeId(100) &&
+		fileId.Key == types.NeedleId(0x1234567890123456) &&
+		fileId.Cookie == types.Cookie(types.Uint32ToCookie(uint32(0x78901234)))) {
+		t.Errorf("src : %s, dest : %v", fidStr1, fileId)
+	}
+
+	fidStr1 = "100,abcd0000abcd"
+	fileId, err = ParseFileIdFromString(fidStr1)
+	if err != nil {
+		t.Errorf("%s : should be OK", fidStr1)
+	}
+	if !(fileId.VolumeId == VolumeId(100) &&
+		fileId.Key == types.NeedleId(0xabcd) &&
+		fileId.Cookie == types.Cookie(types.Uint32ToCookie(uint32(0xabcd)))) {
+		t.Errorf("src : %s, dest : %v", fidStr1, fileId)
+	}
+
+	fidStr1 = "100,1234567890123456789012345"
+	_, err = ParseFileIdFromString(fidStr1)
+	if err == nil {
+		t.Errorf("%s : needleId is too long", fidStr1)
+	}
+}
+
+func TestNeedleIdFileId(t *testing.T) {
+	tests := []struct {
+		key      types.NeedleId
+		volumeId uint32
+		want     string
+	}{
+		{types.NeedleId(1), 3, "3,0100000000"},
+		{types.NeedleId(0x123), 3, "3,012300000000"},
+		{types.NeedleId(0x1234), 7, "7,123400000000"},
+	}
+	for _, tt := range tests {
+		fid := tt.key.FileId(tt.volumeId)
+		if fid != tt.want {
+			t.Errorf("NeedleId(%d).FileId(%d) = %s, want %s", tt.key, tt.volumeId, fid, tt.want)
+		}
+		fileId, err := ParseFileIdFromString(fid)
+		if err != nil {
+			t.Errorf("%s : should be OK: %v", fid, err)
+			continue
+		}
+		if fileId.VolumeId != VolumeId(tt.volumeId) || fileId.Key != tt.key || fileId.Cookie != 0 {
+			t.Errorf("src : %s, dest : %v", fid, fileId)
+		}
+	}
+}

weed/storage/needle/needle.go

@@ -0,0 +1,186 @@
+package needle
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/images"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+const (
+	NeedleChecksumSize = 4
+	PairNamePrefix     = "Seaweed-"
+)
+
+/*
+* A Needle means a uploaded and stored file.
+* Needle file size is limited to 4GB for now.
+ */
+type Needle struct {
+	Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
+	Id     NeedleId `comment:"needle id"`
+	Size   Size     `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
+
+	DataSize     uint32 `comment:"Data size"` //version2
+	Data         []byte `comment:"The actual file data"`
+	Flags        byte   `comment:"boolean flags"` //version2
+	NameSize     uint8  //version2
+	Name         []byte `comment:"maximum 255 characters"` //version2
+	MimeSize     uint8  //version2
+	Mime         []byte `comment:"maximum 255 characters"` //version2
+	PairsSize    uint16 //version2
+	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
+	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
+	Ttl          *TTL
+
+	Checksum   CRC    `comment:"CRC32 to check integrity"`
+	AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
+	Padding    []byte `comment:"Aligned to 8 bytes"`
+}
+
+func (n *Needle) String() (str string) {
+	str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s Compressed:%v", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime, n.IsCompressed())
+	return
+}
+
+func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64, bytesBuffer *bytes.Buffer) (n *Needle, originalSize int, contentMd5 string, e error) {
+	n = new(Needle)
+	pu, e := ParseUpload(r, sizeLimit, bytesBuffer)
+	if e != nil {
+		return
+	}
+	n.Data = pu.Data
+	originalSize = pu.OriginalDataSize
+	n.LastModified = pu.ModifiedTime
+	n.Ttl = pu.Ttl
+	contentMd5 = pu.ContentMd5
+
+	if len(pu.FileName) < 256 {
+		n.Name = []byte(pu.FileName)
+		n.SetHasName()
+	}
+	if len(pu.MimeType) < 256 {
+		n.Mime = []byte(pu.MimeType)
+		n.SetHasMime()
+	}
+	if len(pu.PairMap) != 0 {
+		trimmedPairMap := make(map[string]string)
+		for k, v := range pu.PairMap {
+			trimmedPairMap[k[len(PairNamePrefix):]] = v
+		}
+
+		pairs, _ := json.Marshal(trimmedPairMap)
+		if len(pairs) < 65536 {
+			n.Pairs = pairs
+			n.PairsSize = uint16(len(pairs))
+			n.SetHasPairs()
+		}
+	}
+	if pu.IsGzipped {
+		// println(r.URL.Path, "is set to compressed", pu.FileName, pu.IsGzipped, "dataSize", pu.OriginalDataSize)
+		n.SetIsCompressed()
+	}
+	if n.LastModified == 0 {
+		n.LastModified = uint64(time.Now().Unix())
+	}
+	n.SetHasLastModifiedDate()
+	if n.Ttl != EMPTY_TTL {
+		n.SetHasTtl()
+	}
+
+	if pu.IsChunkedFile {
+		n.SetIsChunkManifest()
+	}
+
+	if fixJpgOrientation {
+		loweredName := strings.ToLower(pu.FileName)
+		if pu.MimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
+			n.Data = images.FixJpgOrientation(n.Data)
+		}
+	}
+
+	n.Checksum = NewCRC(n.Data)
+
+	commaSep := strings.LastIndex(r.URL.Path, ",")
+	dotSep := strings.LastIndex(r.URL.Path, ".")
+	fid := r.URL.Path[commaSep+1:]
+	// dot must be after comma; otherwise path[commaSep+1:dotSep] panics
+	if dotSep > commaSep {
+		fid = r.URL.Path[commaSep+1 : dotSep]
+	}
+
+	e = n.ParsePath(fid)
+
+	return
+}
+func (n *Needle) ParsePath(fid string) (err error) {
+	length := len(fid)
+	if length <= CookieSize*2 {
+		return fmt.Errorf("Invalid fid: %s", fid)
+	}
+	delta := ""
+	deltaIndex := strings.LastIndex(fid, "_")
+	if deltaIndex > 0 {
+		fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
+	}
+	n.Id, n.Cookie, err = ParseNeedleIdCookie(fid)
+	if err != nil {
+		return err
+	}
+	if delta != "" {
+		if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
+			if d > math.MaxUint64-uint64(n.Id) {
+				return fmt.Errorf("delta %d overflows needle id %x", d, n.Id)
+			}
+			n.Id += Uint64ToNeedleId(d)
+		} else {
+			return e
+		}
+	}
+	return err
+}
+
+func GetAppendAtNs(volumeLastAppendAtNs uint64) uint64 {
+	return max(uint64(time.Now().UnixNano()), volumeLastAppendAtNs+1)
+}
+
+func (n *Needle) UpdateAppendAtNs(volumeLastAppendAtNs uint64) {
+	n.AppendAtNs = max(uint64(time.Now().UnixNano()), volumeLastAppendAtNs+1)
+}
+
+func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
+	if len(key_hash_string) <= CookieSize*2 {
+		return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too short.")
+	}
+	if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
+		return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too long.")
+	}
+	split := len(key_hash_string) - CookieSize*2
+	needleId, err := ParseNeedleId(key_hash_string[:split])
+	if err != nil {
+		return NeedleIdEmpty, 0, fmt.Errorf("Parse needleId error: %w", err)
+	}
+	cookie, err := ParseCookie(key_hash_string[split:])
+	if err != nil {
+		return NeedleIdEmpty, 0, fmt.Errorf("Parse cookie error: %w", err)
+	}
+	return needleId, cookie, nil
+}
+
+func (n *Needle) LastModifiedString() string {
+	return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
+}
+
+func max(x, y uint64) uint64 {
+	if x <= y {
+		return y
+	}
+	return x
+}

weed/storage/needle/needle_parse_upload.go

@@ -0,0 +1,293 @@
+package needle
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"mime"
+	"net/http"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// maxEagerPreGrow caps how many bytes ParseUpload is willing to pre-allocate
+// from a request's announced Content-Length. Large enough to skip a few
+// rounds of bytes.Buffer.ReadFrom geometric grow on typical small uploads;
+// small enough that a misreported Content-Length or a slow/idle client can
+// only ever waste this much memory per request. Bigger uploads fall back to
+// the standard ReadFrom-grow path for the remainder.
+const maxEagerPreGrow = 4 * 1024 * 1024
+
+// eagerPreGrow ensures bytesBuffer has at least min(contentLength, sizeLimit,
+// maxEagerPreGrow) bytes of capacity, so the bytes.Buffer.ReadFrom pumps
+// inside parseUpload below skip the first round(s) of geometric grow on the
+// common upload sizes — see #6541. The cap policy is the load-bearing piece
+// here; it's extracted so unit tests can exercise the policy directly
+// without spinning a real http upload.
+func eagerPreGrow(bytesBuffer *bytes.Buffer, contentLength, sizeLimit int64) {
+	if contentLength <= 0 || contentLength > sizeLimit {
+		return
+	}
+	grow := contentLength
+	if grow > maxEagerPreGrow {
+		grow = maxEagerPreGrow
+	}
+	bytesBuffer.Grow(int(grow))
+}
+
+type ParsedUpload struct {
+	FileName    string
+	Data        []byte
+	bytesBuffer *bytes.Buffer
+	MimeType    string
+	PairMap     map[string]string
+	IsGzipped   bool
+	// IsZstd           bool
+	OriginalDataSize int
+	ModifiedTime     uint64
+	Ttl              *TTL
+	IsChunkedFile    bool
+	UncompressedData []byte
+	ContentMd5       string
+}
+
+func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (pu *ParsedUpload, e error) {
+	bytesBuffer.Reset()
+	eagerPreGrow(bytesBuffer, r.ContentLength, sizeLimit)
+	pu = &ParsedUpload{bytesBuffer: bytesBuffer}
+	pu.PairMap = make(map[string]string)
+	for k, v := range r.Header {
+		if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
+			pu.PairMap[k] = v[0]
+		}
+	}
+	// glog.V(4).Infof("ParseUpload: r.URL=%s, Content-MD5 header=%s", r.URL.String(), r.Header.Get("Content-MD5"))
+
+	e = parseUpload(r, sizeLimit, pu)
+
+	if e != nil {
+		return
+	}
+
+	pu.ModifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
+	pu.Ttl, _ = ReadTTL(r.FormValue("ttl"))
+
+	pu.OriginalDataSize = len(pu.Data)
+	pu.UncompressedData = pu.Data
+	// println("received data", len(pu.Data), "isGzipped", pu.IsGzipped, "mime", pu.MimeType, "name", pu.FileName)
+	if pu.IsGzipped {
+		// MD5 check needs the uncompressed bytes; otherwise just count
+		// the gunzip stream — see #6541.
+		needMD5 := r.Header.Get("Content-MD5") != "" || pu.ContentMd5 != ""
+		if needMD5 {
+			if unzipped, err := util.DecompressData(pu.Data); err == nil {
+				pu.OriginalDataSize = len(unzipped)
+				pu.UncompressedData = unzipped
+			}
+		} else if n, err := util.GunzipStream(io.Discard, bytes.NewReader(pu.Data)); err == nil {
+			pu.OriginalDataSize = int(n)
+		}
+	} else if r.URL.Query().Get("type") != "replicate" {
+		// replica writes must keep the source needle's compression state, not re-derive it
+		ext := filepath.Base(pu.FileName)
+		mimeType := pu.MimeType
+		if mimeType == "" {
+			mimeType = http.DetectContentType(pu.Data)
+		}
+		// println("detected mimetype to", pu.MimeType)
+		if mimeType == "application/octet-stream" {
+			mimeType = ""
+		}
+		if shouldBeCompressed, iAmSure := util.IsCompressableFileType(ext, mimeType); shouldBeCompressed && iAmSure {
+			// println("ext", ext, "iAmSure", iAmSure, "shouldBeCompressed", shouldBeCompressed, "mimeType", pu.MimeType)
+			if compressedData, err := util.GzipData(pu.Data); err == nil {
+				if len(compressedData)*10 < len(pu.Data)*9 {
+					pu.Data = compressedData
+					pu.IsGzipped = true
+				}
+				// println("gzipped data size", len(compressedData))
+			}
+		}
+	}
+
+	// verify Content-MD5
+	expectedChecksum := r.Header.Get("Content-MD5")
+	if expectedChecksum == "" {
+		expectedChecksum = pu.ContentMd5
+	}
+	if expectedChecksum != "" {
+		h := md5.New()
+		h.Write(pu.UncompressedData)
+		pu.ContentMd5 = base64.StdEncoding.EncodeToString(h.Sum(nil))
+		if expectedChecksum != pu.ContentMd5 {
+			e = fmt.Errorf("Content-MD5 did not match md5 of file data expected [%s] received [%s] size %d", expectedChecksum, pu.ContentMd5, len(pu.UncompressedData))
+			return
+		}
+	}
+
+	return
+}
+
+func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
+
+	defer func() {
+		if e != nil && r.Body != nil {
+			io.Copy(io.Discard, r.Body)
+			r.Body.Close()
+		}
+	}()
+
+	contentType := r.Header.Get("Content-Type")
+	var dataSize int64
+
+	if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) {
+		form, fe := r.MultipartReader()
+
+		if fe != nil {
+			glog.V(0).Infoln("MultipartReader [ERROR]", fe)
+			e = fe
+			return
+		}
+
+		// first multi-part item
+		part, fe := form.NextPart()
+		if fe != nil {
+			glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
+			e = fe
+			return
+		}
+
+		pu.FileName = part.FileName()
+		if pu.FileName != "" {
+			pu.FileName = util.CleanWindowsPathBase(pu.FileName)
+		}
+
+		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
+		if e != nil {
+			glog.V(0).Infoln("Reading Content [ERROR]", e)
+			return
+		}
+		if dataSize == sizeLimit+1 {
+			e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+			return
+		}
+		pu.Data = pu.bytesBuffer.Bytes()
+
+		contentType = part.Header.Get("Content-Type")
+		pu.ContentMd5 = part.Header.Get("Content-MD5")
+
+		// if the filename is empty string, do a search on the other multi-part items
+		for pu.FileName == "" {
+			part2, fe := form.NextPart()
+			if fe != nil {
+				break // no more or on error, just safely break
+			}
+
+			fName := part2.FileName()
+
+			// found the first <file type> multi-part has filename
+			if fName != "" {
+				pu.bytesBuffer.Reset()
+				dataSize2, fe2 := pu.bytesBuffer.ReadFrom(io.LimitReader(part2, sizeLimit+1))
+				if fe2 != nil {
+					glog.V(0).Infoln("Reading Content [ERROR]", fe2)
+					e = fe2
+					return
+				}
+				if dataSize2 == sizeLimit+1 {
+					e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+					return
+				}
+
+				// update
+				pu.Data = pu.bytesBuffer.Bytes()
+				pu.FileName = util.CleanWindowsPathBase(fName)
+				contentType = part2.Header.Get("Content-Type")
+				pu.ContentMd5 = part2.Header.Get("Content-MD5")
+				part = part2
+				break
+			}
+		}
+
+		pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
+		// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
+
+	} else {
+		disposition := r.Header.Get("Content-Disposition")
+
+		if strings.Contains(disposition, "name=") {
+
+			if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") {
+				disposition = "attachment; " + disposition
+			}
+
+			_, mediaTypeParams, err := mime.ParseMediaType(disposition)
+
+			if err == nil {
+				dpFilename, hasFilename := mediaTypeParams["filename"]
+				dpName, hasName := mediaTypeParams["name"]
+
+				if hasFilename {
+					pu.FileName = dpFilename
+				} else if hasName {
+					pu.FileName = dpName
+				}
+
+			}
+
+		} else {
+			pu.FileName = ""
+		}
+
+		if pu.FileName != "" {
+			pu.FileName = util.CleanWindowsPathBase(pu.FileName)
+		} else {
+			pu.FileName = path.Base(r.URL.Path)
+		}
+
+		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
+
+		if e != nil {
+			glog.V(0).Infoln("Reading Content [ERROR]", e)
+			return
+		}
+		if dataSize == sizeLimit+1 {
+			e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
+			return
+		}
+
+		pu.Data = pu.bytesBuffer.Bytes()
+		pu.MimeType = contentType
+		pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
+		// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
+	}
+
+	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
+
+	if !pu.IsChunkedFile {
+
+		dotIndex := strings.LastIndex(pu.FileName, ".")
+		ext, mtype := "", ""
+
+		if dotIndex > 0 {
+			ext = strings.ToLower(pu.FileName[dotIndex:])
+			mtype = mime.TypeByExtension(ext)
+		}
+
+		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
+			pu.MimeType = contentType // only return mime type if not deducible
+		} else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" {
+			pu.MimeType = mtype
+		}
+
+	}
+
+	return
+}

weed/storage/needle/needle_parse_upload_alloc_test.go

@@ -0,0 +1,104 @@
+package needle
+
+import (
+	"bytes"
+	"testing"
+)
+
+// TestEagerPreGrow is a regression guard for the volume-side amplifier in
+// https://github.com/seaweedfs/seaweedfs/issues/6541.
+//
+// ParseUpload reads the multipart part body via bytes.Buffer.ReadFrom; if the
+// receive buffer arrives with cap=0 (sync.Pool dropped the prior buffer),
+// ReadFrom doubles capacity on each grow and pays 2-4x the chunk size in
+// cumulative allocations. eagerPreGrow shaves the first few rounds off that
+// grow chain by pre-sizing from r.ContentLength.
+//
+// Two policy invariants the cases below pin:
+//
+//  1. The cap. We never grow more than maxEagerPreGrow per request, so a
+//     misreported Content-Length or a slow/idle client cannot force
+//     per-request preallocation up to the configured sizeLimit. This is the
+//     review concern that landed during PR review and is what changed the
+//     test from a TotalAlloc bound to this structural form.
+//  2. The validity gates. Negative / zero / oversized Content-Length and
+//     buffers that already have enough capacity should not trigger a Grow.
+func TestEagerPreGrow(t *testing.T) {
+	const sizeLimit = int64(256 * 1024 * 1024)
+
+	cases := []struct {
+		name       string
+		startCap   int
+		cl         int64
+		wantMinCap int
+		wantNoop   bool // post: cap stays at startCap
+	}{
+		{
+			name:       "small content-length grows exactly",
+			cl:         1 * 1024 * 1024,
+			wantMinCap: 1 * 1024 * 1024,
+		},
+		{
+			name:       "content-length at cap grows to cap",
+			cl:         maxEagerPreGrow,
+			wantMinCap: maxEagerPreGrow,
+		},
+		{
+			name:       "content-length above cap is clamped to cap",
+			cl:         200 * 1024 * 1024,
+			wantMinCap: maxEagerPreGrow,
+		},
+		{
+			name:     "content-length above sizeLimit is rejected",
+			cl:       sizeLimit + 1,
+			wantNoop: true,
+		},
+		{
+			name:     "zero content-length is a no-op",
+			cl:       0,
+			wantNoop: true,
+		},
+		{
+			name:     "negative content-length (chunked encoding) is a no-op",
+			cl:       -1,
+			wantNoop: true,
+		},
+		{
+			name:       "buffer with sufficient cap is left alone",
+			startCap:   maxEagerPreGrow,
+			cl:         1 * 1024 * 1024,
+			wantMinCap: maxEagerPreGrow,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			bb := &bytes.Buffer{}
+			if c.startCap > 0 {
+				bb.Grow(c.startCap)
+			}
+			startCap := bb.Cap()
+
+			eagerPreGrow(bb, c.cl, sizeLimit)
+
+			if c.wantNoop {
+				if got := bb.Cap(); got != startCap {
+					t.Fatalf("eagerPreGrow(cl=%d, sizeLimit=%d) should not have grown: "+
+						"cap=%d, want %d (regression: validity gate removed?)",
+						c.cl, sizeLimit, got, startCap)
+				}
+				return
+			}
+			if got := bb.Cap(); got < c.wantMinCap {
+				t.Fatalf("after eagerPreGrow(cl=%d): cap=%d, want >=%d", c.cl, got, c.wantMinCap)
+			}
+			// Hard upper bound: never above the cap (regardless of cl/sizeLimit).
+			// Note bytes.Buffer.Grow may round capacity up modestly, so allow
+			// a generous overshoot but still well below sizeLimit.
+			if got := int64(bb.Cap()); got > 2*int64(maxEagerPreGrow) {
+				t.Fatalf("after eagerPreGrow(cl=%d): cap=%d exceeds 2*maxEagerPreGrow=%d "+
+					"(regression: cap removed?)", c.cl, got, 2*maxEagerPreGrow)
+			}
+		})
+	}
+}

weed/storage/needle/needle_parse_upload_stream_test.go

@@ -0,0 +1,169 @@
+package needle
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/md5"
+	"encoding/base64"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"net/textproto"
+	"runtime"
+	"testing"
+)
+
+// TestParseUpload_GzipStreamCount: gzipped uploads without Content-MD5
+// must take the stream-count path and skip the uncompressed-slice
+// allocation. See #6541.
+func TestParseUpload_GzipStreamCount(t *testing.T) {
+	const uncompressedSize = 4 * 1024 * 1024 // 4 MiB
+	uncompressed := make([]byte, uncompressedSize)
+	for i := range uncompressed {
+		uncompressed[i] = byte(i*31 + 7)
+	}
+	gzipped := gzipBytes(t, uncompressed)
+	uncompressedMD5 := base64.StdEncoding.EncodeToString(md5sum(uncompressed))
+
+	cases := []struct {
+		name       string
+		md5OnPart  string
+		md5OnReq   string
+		wantStream bool
+	}{
+		{name: "no MD5: stream-count", wantStream: true},
+		{name: "MD5 on part: materialize", md5OnPart: uncompressedMD5},
+		{name: "MD5 on request: materialize", md5OnReq: uncompressedMD5},
+	}
+
+	// Warm the gzip.Reader sync.Pool so per-test alloc is steady-state.
+	if _, err := ParseUpload(buildGzipReq(t, gzipped, "", ""), 256<<20, &bytes.Buffer{}); err != nil {
+		t.Fatalf("warmup ParseUpload: %v", err)
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			req := buildGzipReq(t, gzipped, c.md5OnPart, c.md5OnReq)
+
+			runtime.GC()
+			var before, after runtime.MemStats
+			runtime.ReadMemStats(&before)
+
+			pu, err := ParseUpload(req, 256<<20, &bytes.Buffer{})
+
+			runtime.ReadMemStats(&after)
+			allocated := after.TotalAlloc - before.TotalAlloc
+
+			if err != nil {
+				t.Fatalf("ParseUpload: %v", err)
+			}
+			if pu.OriginalDataSize != uncompressedSize {
+				t.Errorf("OriginalDataSize=%d, want %d", pu.OriginalDataSize, uncompressedSize)
+			}
+
+			if c.wantStream {
+				if allocated > uint64(uncompressedSize) {
+					t.Errorf("stream path allocated %d, bound %d", allocated, uncompressedSize)
+				}
+			} else {
+				if allocated < uint64(uncompressedSize) {
+					t.Errorf("materialize path allocated %d, want >= %d", allocated, uncompressedSize)
+				}
+			}
+			t.Logf("allocated=%d bytes (gzipped=%d, uncompressed=%d)",
+				allocated, len(gzipped), uncompressedSize)
+		})
+	}
+}
+
+func TestParseUploadUsesDiscoveredFilePartContentType(t *testing.T) {
+	var body bytes.Buffer
+	mw := multipart.NewWriter(&body)
+
+	fieldHeader := make(textproto.MIMEHeader)
+	fieldHeader.Set("Content-Disposition", `form-data; name="description"`)
+	fieldHeader.Set("Content-Type", "text/plain")
+	field, err := mw.CreatePart(fieldHeader)
+	if err != nil {
+		t.Fatalf("create field part: %v", err)
+	}
+	if _, err := field.Write([]byte("metadata")); err != nil {
+		t.Fatalf("write field part: %v", err)
+	}
+
+	fileHeader := make(textproto.MIMEHeader)
+	fileHeader.Set("Content-Disposition", `form-data; name="file"; filename="file.dat"`)
+	fileHeader.Set("Content-Type", "application/x-seaweed-test")
+	filePart, err := mw.CreatePart(fileHeader)
+	if err != nil {
+		t.Fatalf("create file part: %v", err)
+	}
+	if _, err := filePart.Write([]byte("payload")); err != nil {
+		t.Fatalf("write file part: %v", err)
+	}
+	if err := mw.Close(); err != nil {
+		t.Fatalf("close multipart writer: %v", err)
+	}
+
+	req := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(body.Bytes()))
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+
+	pu, err := ParseUpload(req, 1024, &bytes.Buffer{})
+	if err != nil {
+		t.Fatalf("ParseUpload: %v", err)
+	}
+	if pu.FileName != "file.dat" {
+		t.Fatalf("FileName = %q, want file.dat", pu.FileName)
+	}
+	if pu.MimeType != "application/x-seaweed-test" {
+		t.Fatalf("MimeType = %q, want application/x-seaweed-test", pu.MimeType)
+	}
+}
+
+func gzipBytes(t *testing.T, in []byte) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	if _, err := gw.Write(in); err != nil {
+		t.Fatalf("gzip write: %v", err)
+	}
+	if err := gw.Close(); err != nil {
+		t.Fatalf("gzip close: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func md5sum(in []byte) []byte {
+	h := md5.New()
+	h.Write(in)
+	return h.Sum(nil)
+}
+
+func buildGzipReq(t *testing.T, body []byte, md5OnPart, md5OnReq string) *http.Request {
+	t.Helper()
+	var bodyBuf bytes.Buffer
+	mw := multipart.NewWriter(&bodyBuf)
+	h := make(textproto.MIMEHeader)
+	h.Set("Content-Disposition", `form-data; name="file"; filename="test.bin"`)
+	h.Set("Content-Encoding", "gzip")
+	if md5OnPart != "" {
+		h.Set("Content-MD5", md5OnPart)
+	}
+	fw, err := mw.CreatePart(h)
+	if err != nil {
+		t.Fatalf("CreatePart: %v", err)
+	}
+	if _, err := fw.Write(body); err != nil {
+		t.Fatalf("write part: %v", err)
+	}
+	if err := mw.Close(); err != nil {
+		t.Fatalf("close mw: %v", err)
+	}
+	req := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(bodyBuf.Bytes()))
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	req.ContentLength = int64(bodyBuf.Len())
+	if md5OnReq != "" {
+		req.Header.Set("Content-MD5", md5OnReq)
+	}
+	return req
+}

weed/storage/needle/needle_read.go

@@ -0,0 +1,294 @@
+package needle
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	FlagIsCompressed        = 0x01
+	FlagHasName             = 0x02
+	FlagHasMime             = 0x04
+	FlagHasLastModifiedDate = 0x08
+	FlagHasTtl              = 0x10
+	FlagHasPairs            = 0x20
+	FlagIsChunkManifest     = 0x80
+	LastModifiedBytesLength = 5
+	TtlBytesLength          = 2
+)
+
+var ErrorSizeMismatch = errors.New("size mismatch")
+var ErrorSizeInvalid = errors.New("size invalid")
+
+// ErrorCorrupted marks a needle whose on-disk bytes cannot be parsed because
+// of data corruption: bad CRC, malformed v2/v3/v4 headers, or out-of-range
+// fields. Wrap errors with %w so callers (e.g. vacuum compaction) can
+// distinguish "the bytes are bad" from a genuine I/O fault.
+var ErrorCorrupted = errors.New("needle data corrupted")
+
+func (n *Needle) DiskSize(version Version) int64 {
+	return GetActualSize(n.Size, version)
+}
+
+func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error) {
+
+	dataSize := GetActualSize(size, version)
+	dataSlice = make([]byte, int(dataSize))
+
+	var n int
+	n, err = r.ReadAt(dataSlice, offset)
+	if err != nil && int64(n) == dataSize {
+		err = nil
+	}
+	if err != nil {
+		fileSize, _, _ := r.GetStat()
+		glog.Errorf("%s read %d dataSize %d offset %d fileSize %d: %v", r.Name(), n, dataSize, offset, fileSize, err)
+	}
+	return dataSlice, err
+
+}
+
+// ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.
+func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error) {
+	n.ParseNeedleHeader(bytes)
+	if n.Size != size {
+		if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatchOffsetSize).Inc()
+			return ErrorSizeMismatch
+		}
+		stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorSizeMismatch).Inc()
+		return fmt.Errorf("%w: entry not found: offset %d found id %x size %d, expected size %d", ErrorSizeMismatch, offset, n.Id, n.Size, size)
+	}
+	if version == Version1 {
+		n.Data = bytes[NeedleHeaderSize : NeedleHeaderSize+size]
+	} else {
+		err := n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(size)])
+		if err != nil && err != io.EOF {
+			return err
+		}
+	}
+	err = n.readNeedleTail(bytes[NeedleHeaderSize+size:], version)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ReadData hydrates the needle from the file, with only n.Id is set.
+func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error) {
+	bytes, err := ReadNeedleBlob(r, offset, size, version)
+	if err != nil {
+		return err
+	}
+	err = n.ReadBytes(bytes, offset, size, version)
+	if err == ErrorSizeMismatch && OffsetSize == 4 {
+		offset = offset + int64(MaxPossibleVolumeSize)
+		bytes, err = ReadNeedleBlob(r, offset, size, version)
+		if err != nil {
+			return err
+		}
+		err = n.ReadBytes(bytes, offset, size, version)
+	}
+	return err
+}
+
+func (n *Needle) ParseNeedleHeader(bytes []byte) {
+	n.Cookie = BytesToCookie(bytes[0:CookieSize])
+	n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
+	n.Size = BytesToSize(bytes[CookieSize+NeedleIdSize : NeedleHeaderSize])
+}
+
+func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
+	index, lenBytes := 0, len(bytes)
+	if index < lenBytes {
+		n.DataSize = util.BytesToUint32(bytes[index : index+4])
+		index = index + 4
+		if int(n.DataSize)+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return fmt.Errorf("index out of range %d: %w", 1, ErrorCorrupted)
+		}
+		n.Data = bytes[index : index+int(n.DataSize)]
+		index = index + int(n.DataSize)
+	}
+	_, err = n.readNeedleDataVersion2NonData(bytes[index:])
+	return
+}
+func (n *Needle) readNeedleDataVersion2NonData(bytes []byte) (index int, err error) {
+	lenBytes := len(bytes)
+	if index < lenBytes {
+		n.Flags = bytes[index]
+		index = index + 1
+	}
+	if index < lenBytes && n.HasName() {
+		n.NameSize = uint8(bytes[index])
+		index = index + 1
+		if int(n.NameSize)+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 2, ErrorCorrupted)
+		}
+		n.Name = bytes[index : index+int(n.NameSize)]
+		index = index + int(n.NameSize)
+	}
+	if index < lenBytes && n.HasMime() {
+		n.MimeSize = uint8(bytes[index])
+		index = index + 1
+		if int(n.MimeSize)+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 3, ErrorCorrupted)
+		}
+		n.Mime = bytes[index : index+int(n.MimeSize)]
+		index = index + int(n.MimeSize)
+	}
+	if index < lenBytes && n.HasLastModifiedDate() {
+		if LastModifiedBytesLength+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 4, ErrorCorrupted)
+		}
+		n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
+		index = index + LastModifiedBytesLength
+	}
+	if index < lenBytes && n.HasTtl() {
+		if TtlBytesLength+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 5, ErrorCorrupted)
+		}
+		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
+		index = index + TtlBytesLength
+	}
+	if index < lenBytes && n.HasPairs() {
+		if 2+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 6, ErrorCorrupted)
+		}
+		n.PairsSize = util.BytesToUint16(bytes[index : index+2])
+		index += 2
+		if int(n.PairsSize)+index > lenBytes {
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorIndexOutOfRange).Inc()
+			return index, fmt.Errorf("index out of range %d: %w", 7, ErrorCorrupted)
+		}
+		end := index + int(n.PairsSize)
+		n.Pairs = bytes[index:end]
+		index = end
+	}
+	return index, nil
+}
+
+func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) {
+	n = new(Needle)
+
+	bytes = make([]byte, NeedleHeaderSize)
+
+	var count int
+	count, err = r.ReadAt(bytes, offset)
+	if err == io.EOF && count == NeedleHeaderSize {
+		err = nil
+	}
+	if count <= 0 || err != nil {
+		return nil, bytes, 0, err
+	}
+
+	n.ParseNeedleHeader(bytes)
+	bodyLength = NeedleBodyLength(n.Size, version)
+
+	return
+}
+
+// n should be a needle already read the header
+// the input stream will read until next file entry
+func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
+
+	if bodyLength <= 0 {
+		return nil, nil
+	}
+	bytes = make([]byte, bodyLength)
+	readCount, err := r.ReadAt(bytes, offset)
+	if err == io.EOF && int64(readCount) == bodyLength {
+		err = nil
+	}
+	if err != nil {
+		glog.Errorf("%s read %d bodyLength %d offset %d: %v", r.Name(), readCount, bodyLength, offset, err)
+		return
+	}
+
+	err = n.ReadNeedleBodyBytes(bytes, version)
+
+	return
+}
+
+func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error) {
+
+	if len(needleBody) <= 0 {
+		return nil
+	}
+	switch version {
+	case Version1:
+		n.Data = needleBody[:n.Size]
+		err = n.readNeedleTail(needleBody[n.Size:], version)
+	case Version2, Version3:
+		err = n.readNeedleDataVersion2(needleBody[0:n.Size])
+		if err == nil {
+			err = n.readNeedleTail(needleBody[n.Size:], version)
+		}
+	default:
+		err = fmt.Errorf("unsupported version %d!", version)
+	}
+	return
+}
+
+func (n *Needle) IsCompressed() bool {
+	return n.Flags&FlagIsCompressed > 0
+}
+func (n *Needle) SetIsCompressed() {
+	n.Flags = n.Flags | FlagIsCompressed
+}
+func (n *Needle) HasName() bool {
+	return n.Flags&FlagHasName > 0
+}
+func (n *Needle) SetHasName() {
+	n.Flags = n.Flags | FlagHasName
+}
+func (n *Needle) HasMime() bool {
+	return n.Flags&FlagHasMime > 0
+}
+func (n *Needle) SetHasMime() {
+	n.Flags = n.Flags | FlagHasMime
+}
+func (n *Needle) HasLastModifiedDate() bool {
+	return n.Flags&FlagHasLastModifiedDate > 0
+}
+func (n *Needle) SetHasLastModifiedDate() {
+	n.Flags = n.Flags | FlagHasLastModifiedDate
+}
+func (n *Needle) HasTtl() bool {
+	return n.Flags&FlagHasTtl > 0
+}
+func (n *Needle) SetHasTtl() {
+	n.Flags = n.Flags | FlagHasTtl
+}
+
+func (n *Needle) IsChunkedManifest() bool {
+	return n.Flags&FlagIsChunkManifest > 0
+}
+
+func (n *Needle) SetIsChunkManifest() {
+	n.Flags = n.Flags | FlagIsChunkManifest
+}
+
+func (n *Needle) HasPairs() bool {
+	return n.Flags&FlagHasPairs != 0
+}
+
+func (n *Needle) SetHasPairs() {
+	n.Flags = n.Flags | FlagHasPairs
+}
+
+func GetActualSize(size Size, version Version) int64 {
+	return NeedleHeaderSize + NeedleBodyLength(size, version)
+}

weed/storage/needle/needle_read_page.go

@@ -0,0 +1,96 @@
+package needle
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ReadNeedleData uses a needle without n.Data to read the content
+// volumeOffset: the offset within the volume
+// needleOffset: the offset within the needle Data
+func (n *Needle) ReadNeedleData(r backend.BackendStorageFile, volumeOffset int64, data []byte, needleOffset int64) (count int, err error) {
+
+	sizeToRead := min(int64(len(data)), int64(n.DataSize)-needleOffset)
+	if sizeToRead <= 0 {
+		return 0, io.EOF
+	}
+	startOffset := volumeOffset + NeedleHeaderSize + DataSizeSize + needleOffset
+
+	count, err = r.ReadAt(data[:sizeToRead], startOffset)
+	if err == io.EOF && int64(count) == sizeToRead {
+		err = nil
+	}
+	if err != nil {
+		fileSize, _, _ := r.GetStat()
+		glog.Errorf("%s read %d %d size %d at offset %d fileSize %d: %v", r.Name(), n.Id, needleOffset, sizeToRead, volumeOffset, fileSize, err)
+	}
+	return
+
+}
+
+// ReadNeedleMeta fills all metadata except the n.Data
+func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("panic occurred: %+v", r)
+		}
+	}()
+
+	bytes := make([]byte, NeedleHeaderSize+DataSizeSize)
+
+	count, err := r.ReadAt(bytes, offset)
+	if err == io.EOF && count == NeedleHeaderSize+DataSizeSize {
+		err = nil
+	}
+	if count != NeedleHeaderSize+DataSizeSize || err != nil {
+		return err
+	}
+	n.ParseNeedleHeader(bytes)
+	if n.Size != size {
+		if OffsetSize == 4 && offset < int64(MaxPossibleVolumeSize) {
+			return ErrorSizeMismatch
+		}
+		return fmt.Errorf("size mismatch for entry at offset %d: found id %x size %d, expected size %d", offset, n.Id, n.Size, size)
+	}
+	n.DataSize = util.BytesToUint32(bytes[NeedleHeaderSize : NeedleHeaderSize+DataSizeSize])
+	startOffset := offset + NeedleHeaderSize
+	if size.IsValid() {
+		startOffset = offset + NeedleHeaderSize + DataSizeSize + int64(n.DataSize)
+	}
+	dataSize := GetActualSize(size, version)
+	stopOffset := offset + dataSize
+	metaSize := stopOffset - startOffset
+	if metaSize < 0 || metaSize > 128*1024 {
+		return fmt.Errorf("invalid needle meta size %d: DataSize=%d, size=%d, offset=%d", metaSize, n.DataSize, size, offset)
+	}
+	metaSlice := make([]byte, int(metaSize))
+
+	count, err = r.ReadAt(metaSlice, startOffset)
+	if err != nil && int64(count) == metaSize {
+		err = nil
+	}
+	if err != nil {
+		return err
+	}
+
+	var index int
+	if size.IsValid() {
+		index, err = n.readNeedleDataVersion2NonData(metaSlice)
+	}
+
+	err = n.readNeedleTail(metaSlice[index:], version)
+	return err
+
+}
+
+func min(x, y int64) int64 {
+	if x < y {
+		return x
+	}
+	return y
+}

weed/storage/needle/needle_read_tail.go

@@ -0,0 +1,49 @@
+package needle
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func (n *Needle) readNeedleTail(needleBody []byte, version Version) error {
+	// for all versions, we need to read the checksum
+	if len(n.Data) > 0 {
+		expectedChecksum := CRC(util.BytesToUint32(needleBody[0:NeedleChecksumSize]))
+		dataChecksum := NewCRC(n.Data)
+		if expectedChecksum != dataChecksum && uint32(expectedChecksum) != dataChecksum.Value() {
+			// crc.Value() is the deprecated legacy transform used before commit 056c480eb
+			// (volume server <3.09). Accept either encoding so .dat files written by older
+			// versions still verify after an upgrade.
+			stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc()
+			return fmt.Errorf("invalid CRC for needle %v (got %08x, want %08x), data on disk corrupted: %w", n.Id, dataChecksum, expectedChecksum, ErrorCorrupted)
+		}
+		n.Checksum = dataChecksum
+	} else {
+		// when data is skipped from reading, just read the checksum
+		n.Checksum = CRC(util.BytesToUint32(needleBody[0:NeedleChecksumSize]))
+	}
+
+	if version == Version3 {
+		tsOffset := NeedleChecksumSize
+		n.AppendAtNs = util.BytesToUint64(needleBody[tsOffset : tsOffset+TimestampSize])
+	}
+	return nil
+}
+
+func PaddingLength(needleSize Size, version Version) Size {
+	if version == Version3 {
+		// this is same value as version2, but just listed here for clarity
+		return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize + TimestampSize) % NeedlePaddingSize)
+	}
+	return NeedlePaddingSize - ((NeedleHeaderSize + needleSize + NeedleChecksumSize) % NeedlePaddingSize)
+}
+
+func NeedleBodyLength(needleSize Size, version Version) int64 {
+	if version == Version3 {
+		return int64(needleSize) + NeedleChecksumSize + TimestampSize + int64(PaddingLength(needleSize, version))
+	}
+	return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
+}

weed/storage/needle/needle_read_tail_test.go

@@ -0,0 +1,53 @@
+package needle
+
+import (
+	"testing"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// .dat files written by volume server versions prior to 3.09 (before commit
+// 056c480eb) store the needle checksum using the legacy CRC.Value() transform.
+// Reads of such needles must still verify after an upgrade.
+func TestReadNeedleTailLegacyChecksum(t *testing.T) {
+	data := []byte("hello seaweed")
+	rawChecksum := NewCRC(data)
+	legacyChecksum := rawChecksum.Value()
+
+	tail := make([]byte, NeedleChecksumSize+TimestampSize)
+	util.Uint32toBytes(tail[0:NeedleChecksumSize], legacyChecksum)
+
+	n := &Needle{Data: data}
+	if err := n.readNeedleTail(tail, Version3); err != nil {
+		t.Fatalf("legacy checksum should verify, got %v", err)
+	}
+	if n.Checksum != rawChecksum {
+		t.Fatalf("expected raw checksum stored, got %x want %x", uint32(n.Checksum), uint32(rawChecksum))
+	}
+}
+
+func TestReadNeedleTailRawChecksum(t *testing.T) {
+	data := []byte("hello seaweed")
+	rawChecksum := NewCRC(data)
+
+	tail := make([]byte, NeedleChecksumSize+TimestampSize)
+	util.Uint32toBytes(tail[0:NeedleChecksumSize], uint32(rawChecksum))
+
+	n := &Needle{Data: data}
+	if err := n.readNeedleTail(tail, Version3); err != nil {
+		t.Fatalf("raw checksum should verify, got %v", err)
+	}
+}
+
+func TestReadNeedleTailCorrupted(t *testing.T) {
+	data := []byte("hello seaweed")
+
+	tail := make([]byte, NeedleChecksumSize+TimestampSize)
+	util.Uint32toBytes(tail[0:NeedleChecksumSize], 0xdeadbeef)
+
+	n := &Needle{Data: data}
+	if err := n.readNeedleTail(tail, Version3); err == nil {
+		t.Fatal("expected CRC mismatch error, got nil")
+	}
+}

weed/storage/needle/needle_test.go

@@ -0,0 +1,57 @@
+package needle
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestParseKeyHash(t *testing.T) {
+	testcases := []struct {
+		KeyHash string
+		ID      types.NeedleId
+		Cookie  types.Cookie
+		Err     bool
+	}{
+		// normal
+		{"4ed4c8116e41", 0x4ed4, 0xc8116e41, false},
+		// cookie with leading zeros
+		{"4ed401116e41", 0x4ed4, 0x01116e41, false},
+		// odd length
+		{"ed400116e41", 0xed4, 0x00116e41, false},
+		// uint
+		{"fed4c8114ed4c811f0116e41", 0xfed4c8114ed4c811, 0xf0116e41, false},
+		// err: too short
+		{"4ed4c811", 0, 0, true},
+		// err: too long
+		{"4ed4c8114ed4c8114ed4c8111", 0, 0, true},
+		// err: invalid character
+		{"helloworld", 0, 0, true},
+	}
+
+	for _, tc := range testcases {
+		if id, cookie, err := ParseNeedleIdCookie(tc.KeyHash); err != nil && !tc.Err {
+			t.Fatalf("Parse %s error: %v", tc.KeyHash, err)
+		} else if err == nil && tc.Err {
+			t.Fatalf("Parse %s expected error got nil", tc.KeyHash)
+		} else if id != tc.ID || cookie != tc.Cookie {
+			t.Fatalf("Parse %s wrong result. Expected: (%d, %d) got: (%d, %d)", tc.KeyHash, tc.ID, tc.Cookie, id, cookie)
+		}
+	}
+}
+
+func TestNeedleParsePathRejectsDeltaOverflow(t *testing.T) {
+	var n Needle
+	err := n.ParsePath("ffffffffffffffff00000000_1")
+	if err == nil {
+		t.Fatalf("ParsePath accepted overflowing delta with needle id %d", n.Id)
+	}
+}
+
+func BenchmarkParseKeyHash(b *testing.B) {
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		ParseNeedleIdCookie("4ed44ed44ed44ed4c8116e41")
+	}
+}

weed/storage/needle/needle_write.go

@@ -0,0 +1,84 @@
+package needle
+
+import (
+	"fmt"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
+)
+
+func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) {
+	end, _, e := w.GetStat()
+	if e != nil {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %w", e)
+		return
+	}
+	offset = uint64(end)
+	if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
+		err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
+		return
+	}
+	bytesBuffer := buffer_pool.SyncPoolGetBuffer()
+	defer func() {
+		if err != nil {
+			if te := w.Truncate(end); te != nil {
+				// handle error or log
+			}
+		}
+		buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+	}()
+
+	size, actualSize, err = writeNeedleByVersion(version, n, offset, bytesBuffer)
+	if err != nil {
+		return
+	}
+
+	_, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset))
+	if err != nil {
+		err = fmt.Errorf("failed to write %d bytes to %s at offset %d: %w", actualSize, w.Name(), offset, err)
+	}
+
+	return offset, size, actualSize, err
+}
+
+func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error) {
+
+	if end, _, e := w.GetStat(); e == nil {
+		defer func(w backend.BackendStorageFile, off int64) {
+			if err != nil {
+				if te := w.Truncate(end); te != nil {
+					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
+				}
+			}
+		}(w, end)
+		offset = uint64(end)
+	} else {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
+		return
+	}
+
+	if version == Version3 {
+		// compute byte offset as int to compare and slice correctly
+		tsOffset := int(NeedleHeaderSize) + int(size) + NeedleChecksumSize
+		// Ensure dataSlice has enough capacity for the timestamp
+		if tsOffset < 0 {
+			err = fmt.Errorf("invalid needle size %d results in negative timestamp offset %d", size, tsOffset)
+			return
+		}
+		if tsOffset+TimestampSize > len(dataSlice) {
+			err = fmt.Errorf("needle blob buffer too small: need %d bytes, have %d", tsOffset+TimestampSize, len(dataSlice))
+			return
+		}
+		util.Uint64toBytes(dataSlice[tsOffset:tsOffset+TimestampSize], appendAtNs)
+	}
+
+	if err == nil {
+		_, err = w.WriteAt(dataSlice, int64(offset))
+	}
+
+	return
+
+}

weed/storage/needle/needle_write_legacy.go

@@ -0,0 +1,142 @@
+package needle
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/buffer_pool"
+)
+
+func (n *Needle) LegacyPrepareWriteBuffer(version Version, writeBytes *bytes.Buffer) (Size, int64, error) {
+	writeBytes.Reset()
+	switch version {
+	case Version1:
+		header := make([]byte, NeedleHeaderSize)
+		CookieToBytes(header[0:CookieSize], n.Cookie)
+		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+		n.Size = Size(len(n.Data))
+		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+		size := n.Size
+		actualSize := NeedleHeaderSize + int64(n.Size)
+		writeBytes.Write(header)
+		writeBytes.Write(n.Data)
+		padding := PaddingLength(n.Size, version)
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		writeBytes.Write(header[0 : NeedleChecksumSize+padding])
+		return size, actualSize, nil
+	case Version2, Version3:
+		header := make([]byte, NeedleHeaderSize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
+		CookieToBytes(header[0:CookieSize], n.Cookie)
+		NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+		if len(n.Name) >= math.MaxUint8 {
+			n.NameSize = math.MaxUint8
+		} else {
+			n.NameSize = uint8(len(n.Name))
+		}
+		n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
+		if n.DataSize > 0 {
+			n.Size = 4 + Size(n.DataSize) + 1
+			if n.HasName() {
+				n.Size = n.Size + 1 + Size(n.NameSize)
+			}
+			if n.HasMime() {
+				n.Size = n.Size + 1 + Size(n.MimeSize)
+			}
+			if n.HasLastModifiedDate() {
+				n.Size = n.Size + LastModifiedBytesLength
+			}
+			if n.HasTtl() {
+				n.Size = n.Size + TtlBytesLength
+			}
+			if n.HasPairs() {
+				n.Size += 2 + Size(n.PairsSize)
+			}
+		} else {
+			n.Size = 0
+		}
+		SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+		writeBytes.Write(header[0:NeedleHeaderSize])
+		if n.DataSize > 0 {
+			util.Uint32toBytes(header[0:4], n.DataSize)
+			writeBytes.Write(header[0:4])
+			writeBytes.Write(n.Data)
+			util.Uint8toBytes(header[0:1], n.Flags)
+			writeBytes.Write(header[0:1])
+			if n.HasName() {
+				util.Uint8toBytes(header[0:1], n.NameSize)
+				writeBytes.Write(header[0:1])
+				writeBytes.Write(n.Name[:n.NameSize])
+			}
+			if n.HasMime() {
+				util.Uint8toBytes(header[0:1], n.MimeSize)
+				writeBytes.Write(header[0:1])
+				writeBytes.Write(n.Mime)
+			}
+			if n.HasLastModifiedDate() {
+				util.Uint64toBytes(header[0:8], n.LastModified)
+				writeBytes.Write(header[8-LastModifiedBytesLength : 8])
+			}
+			if n.HasTtl() && n.Ttl != nil {
+				n.Ttl.ToBytes(header[0:TtlBytesLength])
+				writeBytes.Write(header[0:TtlBytesLength])
+			}
+			if n.HasPairs() {
+				util.Uint16toBytes(header[0:2], n.PairsSize)
+				writeBytes.Write(header[0:2])
+				writeBytes.Write(n.Pairs)
+			}
+		}
+		padding := PaddingLength(n.Size, version)
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		if version == Version2 {
+			writeBytes.Write(header[0 : NeedleChecksumSize+padding])
+		} else {
+			// version3
+			util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
+			writeBytes.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
+		}
+
+		return Size(n.DataSize), GetActualSize(n.Size, version), nil
+	}
+
+	return 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
+}
+
+func (n *Needle) LegacyAppend(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error) {
+	if end, _, e := w.GetStat(); e == nil {
+		defer func(w backend.BackendStorageFile, off int64) {
+			if err != nil {
+				if te := w.Truncate(end); te != nil {
+					glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
+				}
+			}
+		}(w, end)
+		offset = uint64(end)
+	} else {
+		err = fmt.Errorf("Cannot Read Current Volume Position: %w", e)
+		return
+	}
+	if offset >= MaxPossibleVolumeSize && len(n.Data) != 0 {
+		err = fmt.Errorf("Volume Size %d Exceeded %d", offset, MaxPossibleVolumeSize)
+		return
+	}
+
+	bytesBuffer := buffer_pool.SyncPoolGetBuffer()
+	defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
+
+	size, actualSize, err = n.LegacyPrepareWriteBuffer(version, bytesBuffer)
+
+	if err == nil {
+		_, err = w.WriteAt(bytesBuffer.Bytes(), int64(offset))
+		if err != nil {
+			err = fmt.Errorf("failed to write %d bytes to %s at offset %d: %w", actualSize, w.Name(), offset, err)
+		}
+	}
+
+	return offset, size, actualSize, err
+}

weed/storage/needle/needle_write_test.go

@@ -0,0 +1,126 @@
+package needle
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func TestAppend(t *testing.T) {
+	n := &Needle{
+
+		Cookie:       types.Cookie(123),   // Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
+		Id:           types.NeedleId(123), // Id     NeedleId `comment:"needle id"`
+		Size:         8,                   // Size   uint32   `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
+		DataSize:     4,                   // DataSize     uint32 `comment:"Data size"` //version2
+		Data:         []byte("abcd"),      // Data         []byte `comment:"The actual file data"`
+		Flags:        0,                   // Flags        byte   `comment:"boolean flags"`          //version2
+		NameSize:     0,                   // NameSize     uint8                                     //version2
+		Name:         nil,                 // Name         []byte `comment:"maximum 256 characters"` //version2
+		MimeSize:     0,                   // MimeSize     uint8                                     //version2
+		Mime:         nil,                 // Mime         []byte `comment:"maximum 256 characters"` //version2
+		PairsSize:    0,                   // PairsSize    uint16                                    //version2
+		Pairs:        nil,                 // Pairs        []byte `comment:"additional name value pairs, json format, maximum 6
+		LastModified: 123,                 // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
+		Ttl:          nil,                 // Ttl          *TTL
+		Checksum:     123,                 // Checksum   CRC    `comment:"CRC32 to check integrity"`
+		AppendAtNs:   123,                 // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
+		Padding:      nil,                 // Padding    []byte `comment:"Aligned to 8 bytes"`
+	}
+
+	tempFile, err := os.CreateTemp("", ".dat")
+	if err != nil {
+		t.Errorf("Fail TempFile. %v", err)
+		return
+	}
+
+	/*
+		uint8  : 0 to 255
+		uint16 : 0 to 65535
+		uint32 : 0 to 4294967295
+		uint64 : 0 to 18446744073709551615
+		int8   : -128 to 127
+		int16  : -32768 to 32767
+		int32  : -2147483648 to 2147483647
+		int64  : -9223372036854775808 to 9223372036854775807
+	*/
+
+	fileSize := int64(4294967296) + 10000
+	tempFile.Truncate(fileSize)
+	defer func() {
+		tempFile.Close()
+		os.Remove(tempFile.Name())
+	}()
+
+	datBackend := backend.NewDiskFile(tempFile)
+	defer datBackend.Close()
+
+	offset, _, _, _ := n.Append(datBackend, GetCurrentVersion())
+	if offset != uint64(fileSize) {
+		t.Errorf("Fail to Append Needle.")
+	}
+}
+
+func versionString(v Version) string {
+	switch v {
+	case Version1:
+		return "Version1"
+	case Version2:
+		return "Version2"
+	case Version3:
+		return "Version3"
+	default:
+		return "UnknownVersion"
+	}
+}
+
+func TestWriteNeedle_CompatibilityWithLegacy(t *testing.T) {
+	versions := []Version{Version1, Version2, Version3}
+	for _, version := range versions {
+		t.Run(versionString(version), func(t *testing.T) {
+			n := &Needle{
+				Cookie:       0x12345678,
+				Id:           0x1122334455667788,
+				Data:         []byte("hello world"),
+				Flags:        0xFF,
+				Name:         []byte("filename.txt"),
+				Mime:         []byte("text/plain"),
+				LastModified: 0x1234567890,
+				Ttl:          nil, // Add TTL if needed
+				Pairs:        []byte("key=value"),
+				PairsSize:    9,
+				Checksum:     0xCAFEBABE,
+				AppendAtNs:   0xDEADBEEF,
+			}
+
+			// Legacy
+			legacyBuf := &bytes.Buffer{}
+			_, _, err := n.LegacyPrepareWriteBuffer(version, legacyBuf)
+			if err != nil {
+				t.Fatalf("LegacyPrepareWriteBuffer failed: %v", err)
+			}
+
+			// New
+			newBuf := &bytes.Buffer{}
+			offset := uint64(0)
+			switch version {
+			case Version1:
+				_, _, err = writeNeedleV1(n, offset, newBuf)
+			case Version2:
+				_, _, err = writeNeedleV2(n, offset, newBuf)
+			case Version3:
+				_, _, err = writeNeedleV3(n, offset, newBuf)
+			}
+			if err != nil {
+				t.Fatalf("writeNeedleV%d failed: %v", version, err)
+			}
+
+			if !bytes.Equal(legacyBuf.Bytes(), newBuf.Bytes()) {
+				t.Errorf("Data layout mismatch for version %d\nLegacy: %x\nNew:    %x", version, legacyBuf.Bytes(), newBuf.Bytes())
+			}
+		})
+	}
+}

weed/storage/needle/needle_write_v1.go

@@ -0,0 +1,26 @@
+package needle
+
+import (
+	"bytes"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV1(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	bytesBuffer.Reset()
+	header := make([]byte, NeedleHeaderSize)
+	CookieToBytes(header[0:CookieSize], n.Cookie)
+	NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+	n.Size = Size(len(n.Data))
+	SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+	size = n.Size
+	actualSize = NeedleHeaderSize + int64(n.Size)
+	bytesBuffer.Write(header)
+	bytesBuffer.Write(n.Data)
+	padding := PaddingLength(n.Size, Version1)
+	util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+	bytesBuffer.Write(header[0 : NeedleChecksumSize+padding])
+
+	return size, actualSize, nil
+}

weed/storage/needle/needle_write_v2.go

@@ -0,0 +1,86 @@
+package needle
+
+import (
+	"bytes"
+	"math"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV2(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	return writeNeedleCommon(n, offset, bytesBuffer, Version2, func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int) {
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		bytesBuffer.Write(header[0 : NeedleChecksumSize+padding])
+	})
+}
+
+func writeNeedleCommon(n *Needle, offset uint64, bytesBuffer *bytes.Buffer, version Version, writeFooter func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int)) (size Size, actualSize int64, err error) {
+	bytesBuffer.Reset()
+	header := make([]byte, NeedleHeaderSize+TimestampSize)
+	CookieToBytes(header[0:CookieSize], n.Cookie)
+	NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
+	if len(n.Name) >= math.MaxUint8 {
+		n.NameSize = math.MaxUint8
+	} else {
+		n.NameSize = uint8(len(n.Name))
+	}
+	n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
+	if n.DataSize > 0 {
+		n.Size = 4 + Size(n.DataSize) + 1
+		if n.HasName() {
+			n.Size = n.Size + 1 + Size(n.NameSize)
+		}
+		if n.HasMime() {
+			n.Size = n.Size + 1 + Size(n.MimeSize)
+		}
+		if n.HasLastModifiedDate() {
+			n.Size = n.Size + LastModifiedBytesLength
+		}
+		if n.HasTtl() {
+			n.Size = n.Size + TtlBytesLength
+		}
+		if n.HasPairs() {
+			n.Size += 2 + Size(n.PairsSize)
+		}
+	} else {
+		n.Size = 0
+	}
+	SizeToBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
+	bytesBuffer.Write(header[0:NeedleHeaderSize])
+	if n.DataSize > 0 {
+		util.Uint32toBytes(header[0:4], n.DataSize)
+		bytesBuffer.Write(header[0:4])
+		bytesBuffer.Write(n.Data)
+		util.Uint8toBytes(header[0:1], n.Flags)
+		bytesBuffer.Write(header[0:1])
+		if n.HasName() {
+			util.Uint8toBytes(header[0:1], n.NameSize)
+			bytesBuffer.Write(header[0:1])
+			bytesBuffer.Write(n.Name[:n.NameSize])
+		}
+		if n.HasMime() {
+			util.Uint8toBytes(header[0:1], n.MimeSize)
+			bytesBuffer.Write(header[0:1])
+			bytesBuffer.Write(n.Mime)
+		}
+		if n.HasLastModifiedDate() {
+			util.Uint64toBytes(header[0:8], n.LastModified)
+			bytesBuffer.Write(header[8-LastModifiedBytesLength : 8])
+		}
+		if n.HasTtl() && n.Ttl != nil {
+			n.Ttl.ToBytes(header[0:TtlBytesLength])
+			bytesBuffer.Write(header[0:TtlBytesLength])
+		}
+		if n.HasPairs() {
+			util.Uint16toBytes(header[0:2], n.PairsSize)
+			bytesBuffer.Write(header[0:2])
+			bytesBuffer.Write(n.Pairs)
+		}
+	}
+	padding := PaddingLength(n.Size, version)
+	writeFooter(n, header, bytesBuffer, int(padding))
+	size = Size(n.DataSize)
+	actualSize = GetActualSize(n.Size, version)
+	return size, actualSize, nil
+}

weed/storage/needle/needle_write_v3.go

@@ -0,0 +1,16 @@
+package needle
+
+import (
+	"bytes"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func writeNeedleV3(n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	return writeNeedleCommon(n, offset, bytesBuffer, Version3, func(n *Needle, header []byte, bytesBuffer *bytes.Buffer, padding int) {
+		util.Uint32toBytes(header[0:NeedleChecksumSize], uint32(n.Checksum))
+		util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
+		bytesBuffer.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
+	})
+}

weed/storage/needle/needle_write_version.go

@@ -0,0 +1,23 @@
+package needle
+
+import (
+	"bytes"
+	"fmt"
+
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+func writeNeedleByVersion(version Version, n *Needle, offset uint64, bytesBuffer *bytes.Buffer) (size Size, actualSize int64, err error) {
+	// Switch logic moved from needle_write.go
+	switch version {
+	case Version1:
+		size, actualSize, err = writeNeedleV1(n, offset, bytesBuffer)
+	case Version2:
+		size, actualSize, err = writeNeedleV2(n, offset, bytesBuffer)
+	case Version3:
+		size, actualSize, err = writeNeedleV3(n, offset, bytesBuffer)
+	default:
+		err = fmt.Errorf("unsupported version: %d", version)
+	}
+	return
+}

weed/storage/needle/volume_id.go

@@ -0,0 +1,18 @@
+package needle
+
+import (
+	"strconv"
+)
+
+type VolumeId uint32
+
+func NewVolumeId(vid string) (VolumeId, error) {
+	volumeId, err := strconv.ParseUint(vid, 10, 64)
+	return VolumeId(volumeId), err
+}
+func (vid VolumeId) String() string {
+	return strconv.FormatUint(uint64(vid), 10)
+}
+func (vid VolumeId) Next() VolumeId {
+	return VolumeId(uint32(vid) + 1)
+}

weed/storage/needle/volume_id_test.go

@@ -0,0 +1,45 @@
+package needle
+
+import "testing"
+
+func TestNewVolumeId(t *testing.T) {
+	if _, err := NewVolumeId("1"); err != nil {
+		t.Error(err)
+	}
+
+	if _, err := NewVolumeId("a"); err != nil {
+		t.Logf("a is not legal volume id, %v", err)
+	}
+}
+
+func TestVolumeId_String(t *testing.T) {
+	if str := VolumeId(10).String(); str != "10" {
+		t.Errorf("to string failed")
+	}
+
+	vid := VolumeId(11)
+	if str := vid.String(); str != "11" {
+		t.Errorf("to string failed")
+	}
+
+	pvid := &vid
+	if str := pvid.String(); str != "11" {
+		t.Errorf("to string failed")
+	}
+}
+
+func TestVolumeId_Next(t *testing.T) {
+	if vid := VolumeId(10).Next(); vid != VolumeId(11) {
+		t.Errorf("get next volume id failed")
+	}
+
+	vid := VolumeId(11)
+	if new := vid.Next(); new != 12 {
+		t.Errorf("get next volume id failed")
+	}
+
+	pvid := &vid
+	if new := pvid.Next(); new != 12 {
+		t.Errorf("get next volume id failed")
+	}
+}

weed/storage/needle/volume_ttl.go

@@ -0,0 +1,266 @@
+package needle
+
+import (
+	"fmt"
+	"strconv"
+)
+
+const (
+	// stored unit types
+	Empty byte = iota
+	Minute
+	Hour
+	Day
+	Week
+	Month
+	Year
+)
+
+type TTL struct {
+	Count byte
+	Unit  byte
+}
+
+var EMPTY_TTL = &TTL{}
+
+// translate a readable ttl to internal ttl
+// Supports format example:
+// 3m: 3 minutes
+// 4h: 4 hours
+// 5d: 5 days
+// 6w: 6 weeks
+// 7M: 7 months
+// 8y: 8 years
+func ReadTTL(ttlString string) (*TTL, error) {
+	if ttlString == "" {
+		return EMPTY_TTL, nil
+	}
+	ttlBytes := []byte(ttlString)
+	unitByte := ttlBytes[len(ttlBytes)-1]
+	countBytes := ttlBytes[0 : len(ttlBytes)-1]
+	if '0' <= unitByte && unitByte <= '9' {
+		countBytes = ttlBytes
+		unitByte = 'm'
+	}
+	count, err := strconv.Atoi(string(countBytes))
+	unit := toStoredByte(unitByte)
+	return fitTtlCount(count, unit), err
+}
+
+func fitTtlCount(count int, unit byte) *TTL {
+	seconds := ToSeconds(count, unit)
+	if seconds == 0 {
+		return EMPTY_TTL
+	}
+	if seconds%(3600*24*365) == 0 && seconds/(3600*24*365) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 365)), Unit: Year}
+	}
+	if seconds%(3600*24*30) == 0 && seconds/(3600*24*30) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 30)), Unit: Month}
+	}
+	if seconds%(3600*24*7) == 0 && seconds/(3600*24*7) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 7)), Unit: Week}
+	}
+	if seconds%(3600*24) == 0 && seconds/(3600*24) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24)), Unit: Day}
+	}
+	if seconds%(3600) == 0 && seconds/(3600) < 256 {
+		return &TTL{Count: byte(seconds / (3600)), Unit: Hour}
+	}
+	if seconds/60 < 256 {
+		return &TTL{Count: byte(seconds / 60), Unit: Minute}
+	}
+	if seconds/(3600) < 256 {
+		return &TTL{Count: byte(seconds / (3600)), Unit: Hour}
+	}
+	if seconds/(3600*24) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24)), Unit: Day}
+	}
+	if seconds/(3600*24*7) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 7)), Unit: Week}
+	}
+	if seconds/(3600*24*30) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 30)), Unit: Month}
+	}
+	if seconds/(3600*24*365) < 256 {
+		return &TTL{Count: byte(seconds / (3600 * 24 * 365)), Unit: Year}
+	}
+	return EMPTY_TTL
+}
+
+// storedTTLs interns every count paired with a defined unit. The master decodes
+// one per volume in every heartbeat and keeps it for the volume's lifetime, so
+// sharing spares a cluster with millions of volume replicas a distinct two-byte
+// object for each.
+//
+// Entries are immutable. Callers must not write through the returned pointer.
+var storedTTLs = func() (table [256][Year + 1]TTL) {
+	for count := range table {
+		for unit := range table[count] {
+			table[count][unit] = TTL{Count: byte(count), Unit: byte(unit)}
+		}
+	}
+	return table
+}()
+
+func loadTTL(count, unit byte) *TTL {
+	if count == 0 && unit == 0 {
+		return EMPTY_TTL
+	}
+	if unit <= Year {
+		return &storedTTLs[count][unit]
+	}
+	return &TTL{Count: count, Unit: unit}
+}
+
+// read stored bytes to a ttl
+func LoadTTLFromBytes(input []byte) (t *TTL) {
+	return loadTTL(input[0], input[1])
+}
+
+// read stored bytes to a ttl
+func LoadTTLFromUint32(ttl uint32) (t *TTL) {
+	return loadTTL(byte(ttl>>8), byte(ttl))
+}
+
+// save stored bytes to an output with 2 bytes
+func (t *TTL) ToBytes(output []byte) {
+	output[0] = t.Count
+	output[1] = t.Unit
+}
+
+func (t *TTL) ToUint32() (output uint32) {
+	if t == nil || t.Count == 0 {
+		return 0
+	}
+	output = uint32(t.Count) << 8
+	output += uint32(t.Unit)
+	return output
+}
+
+func (t *TTL) String() string {
+	if t == nil || t.Count == 0 {
+		return ""
+	}
+	if t.Unit == Empty {
+		return ""
+	}
+	countString := strconv.Itoa(int(t.Count))
+	switch t.Unit {
+	case Minute:
+		return countString + "m"
+	case Hour:
+		return countString + "h"
+	case Day:
+		return countString + "d"
+	case Week:
+		return countString + "w"
+	case Month:
+		return countString + "M"
+	case Year:
+		return countString + "y"
+	}
+	return ""
+}
+
+func (t *TTL) ToSeconds() uint64 {
+	return ToSeconds(int(t.Count), t.Unit)
+}
+
+func ToSeconds(count int, unit byte) uint64 {
+	switch unit {
+	case Empty:
+		return 0
+	case Minute:
+		return uint64(count) * 60
+	case Hour:
+		return uint64(count) * 60 * 60
+	case Day:
+		return uint64(count) * 60 * 24 * 60
+	case Week:
+		return uint64(count) * 60 * 24 * 7 * 60
+	case Month:
+		return uint64(count) * 60 * 24 * 30 * 60
+	case Year:
+		return uint64(count) * 60 * 24 * 365 * 60
+	}
+	return 0
+}
+
+func toStoredByte(readableUnitByte byte) byte {
+	switch readableUnitByte {
+	case 'm':
+		return Minute
+	case 'h':
+		return Hour
+	case 'd':
+		return Day
+	case 'w':
+		return Week
+	case 'M':
+		return Month
+	case 'y':
+		return Year
+	}
+	return 0
+}
+
+func (t TTL) Minutes() uint32 {
+	switch t.Unit {
+	case Empty:
+		return 0
+	case Minute:
+		return uint32(t.Count)
+	case Hour:
+		return uint32(t.Count) * 60
+	case Day:
+		return uint32(t.Count) * 60 * 24
+	case Week:
+		return uint32(t.Count) * 60 * 24 * 7
+	case Month:
+		return uint32(t.Count) * 60 * 24 * 30
+	case Year:
+		return uint32(t.Count) * 60 * 24 * 365
+	}
+	return 0
+}
+
+func SecondsToTTL(seconds int32) string {
+	if seconds == 0 {
+		return ""
+	}
+	if seconds%(3600*24*365) == 0 && seconds/(3600*24*365) < 256 {
+		return fmt.Sprintf("%dy", seconds/(3600*24*365))
+	}
+	if seconds%(3600*24*30) == 0 && seconds/(3600*24*30) < 256 {
+		return fmt.Sprintf("%dM", seconds/(3600*24*30))
+	}
+	if seconds%(3600*24*7) == 0 && seconds/(3600*24*7) < 256 {
+		return fmt.Sprintf("%dw", seconds/(3600*24*7))
+	}
+	if seconds%(3600*24) == 0 && seconds/(3600*24) < 256 {
+		return fmt.Sprintf("%dd", seconds/(3600*24))
+	}
+	if seconds%(3600) == 0 && seconds/(3600) < 256 {
+		return fmt.Sprintf("%dh", seconds/(3600))
+	}
+	if seconds/60 < 256 {
+		return fmt.Sprintf("%dm", seconds/60)
+	}
+	if seconds/(3600) < 256 {
+		return fmt.Sprintf("%dh", seconds/(3600))
+	}
+	if seconds/(3600*24) < 256 {
+		return fmt.Sprintf("%dd", seconds/(3600*24))
+	}
+	if seconds/(3600*24*7) < 256 {
+		return fmt.Sprintf("%dw", seconds/(3600*24*7))
+	}
+	if seconds/(3600*24*30) < 256 {
+		return fmt.Sprintf("%dM", seconds/(3600*24*30))
+	}
+	if seconds/(3600*24*365) < 256 {
+		return fmt.Sprintf("%dy", seconds/(3600*24*365))
+	}
+	return ""
+}

weed/storage/needle/volume_ttl_test.go

@@ -0,0 +1,98 @@
+package needle
+
+import (
+	"testing"
+)
+
+func TestTTLReadWrite(t *testing.T) {
+	ttl, _ := ReadTTL("")
+	if ttl.Minutes() != 0 {
+		t.Errorf("empty ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("9")
+	if ttl.Minutes() != 9 {
+		t.Errorf("9 ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("8m")
+	if ttl.Minutes() != 8 {
+		t.Errorf("8m ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("5h")
+	if ttl.Minutes() != 300 {
+		t.Errorf("5h ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("5d")
+	if ttl.Minutes() != 5*24*60 {
+		t.Errorf("5d ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("50d")
+	if ttl.Minutes() != 50*24*60 {
+		t.Errorf("50d ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("365d")
+	if ttl.Minutes() != 365*24*60 {
+		t.Errorf("365d ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("730d")
+	if ttl.Minutes() != 730*24*60 {
+		t.Errorf("730d ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("5w")
+	if ttl.Minutes() != 5*7*24*60 {
+		t.Errorf("5w ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("5M")
+	if ttl.Minutes() != 5*30*24*60 {
+		t.Errorf("5M ttl:%v", ttl)
+	}
+
+	ttl, _ = ReadTTL("5y")
+	if ttl.Minutes() != 5*365*24*60 {
+		t.Errorf("5y ttl:%v", ttl)
+	}
+
+	output := make([]byte, 2)
+	ttl.ToBytes(output)
+	ttl2 := LoadTTLFromBytes(output)
+	if ttl.Minutes() != ttl2.Minutes() {
+		t.Errorf("ttl:%v ttl2:%v", ttl, ttl2)
+	}
+
+	ttl3 := LoadTTLFromUint32(ttl.ToUint32())
+	if ttl.Minutes() != ttl3.Minutes() {
+		t.Errorf("ttl:%v ttl3:%v", ttl, ttl3)
+	}
+
+}
+
+func TestLoadTTLIsInterned(t *testing.T) {
+	for count := 0; count < 256; count++ {
+		for unit := 0; unit < 256; unit++ {
+			got := LoadTTLFromBytes([]byte{byte(count), byte(unit)})
+			if got.Count != byte(count) || got.Unit != byte(unit) {
+				if count == 0 && unit == 0 {
+					continue
+				}
+				t.Fatalf("count %d unit %d: got %+v", count, unit, got)
+			}
+			if fromUint32 := LoadTTLFromUint32(got.ToUint32()); count != 0 && *fromUint32 != *got {
+				t.Fatalf("count %d unit %d: uint32 round trip gave %+v", count, unit, fromUint32)
+			}
+		}
+	}
+	if LoadTTLFromBytes([]byte{3, Day}) != LoadTTLFromBytes([]byte{3, Day}) {
+		t.Error("expected repeated loads of the same stored ttl to share one value")
+	}
+	if LoadTTLFromBytes([]byte{0, 0}) != EMPTY_TTL {
+		t.Error("expected a zero ttl to stay EMPTY_TTL")
+	}
+}

weed/storage/needle/volume_version.go

@@ -0,0 +1,17 @@
+package needle
+
+type Version uint8
+
+const (
+	Version1 = Version(1)
+	Version2 = Version(2)
+	Version3 = Version(3)
+)
+
+func GetCurrentVersion() Version {
+	return Version3
+}
+
+func IsSupportedVersion(v Version) bool {
+	return v >= Version1 && v <= Version3
+}