snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/filer/entry_codec_attributes.go

@@ -0,0 +1,213 @@
+package filer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"unicode/utf8"
+
+	"google.golang.org/protobuf/encoding/protowire"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+)
+
+// Field numbers from filer.proto. Only the ones this decoder has to recognise
+// are named; every other Entry field is handed to the generated unmarshaller
+// as-is.
+const (
+	entryChunksField     = 3 // Entry.chunks
+	fileChunkOffsetField = 2 // FileChunk.offset
+	fileChunkSizeField   = 3 // FileChunk.size
+)
+
+// The chunk bytes are the one part of the blob the generated unmarshaller never
+// sees, so the checks it would have made are made here instead: a submessage
+// has to parse, and a proto3 string has to be valid UTF-8. Anything else in a
+// FileChunk is a scalar, which walking it already validates.
+// TestChunkValidationCoversEveryField fails if FileChunk gains a field of
+// either kind that is missing from these.
+var (
+	fileChunkMessageFields = map[protowire.Number]bool{
+		7: true, // fid, a FileId of scalars only, so walking it is a full check
+		8: true, // source_fid
+	}
+	fileChunkStringFields = map[protowire.Number]bool{
+		1: true, // file_id
+		5: true, // e_tag
+		6: true, // source_file_id
+	}
+)
+
+// attributesScratchPool holds the re-encoded entry, which is the blob minus its
+// chunks and so much smaller than what came in.
+var attributesScratchPool = sync.Pool{
+	New: func() any {
+		b := make([]byte, 0, 256)
+		return &b
+	},
+}
+
+// DecodeListedEntry decodes one listed entry, dropping the chunk list when the
+// listing asked for attributes only.
+func DecodeListedEntry(ctx context.Context, entry *Entry, blob []byte) error {
+	if filer_pb.ChunksOmitted(ctx) {
+		return entry.DecodeAttributesOnly(blob)
+	}
+	return entry.DecodeAttributesAndChunks(blob)
+}
+
+// DecodeAttributesOnly fills entry from blob without building its chunk list,
+// which is the bulk of the work for anything but a tiny file. Chunks are still
+// measured, because an entry whose stored FileSize is zero takes its size from
+// them, but no FileChunk is allocated.
+//
+// entry.Chunks is left nil. The one exception is a hard link, whose attributes
+// and chunks are replaced wholesale by a full decode of its own record in
+// FilerStoreWrapper.maybeReadHardLink straight after the store listing. Only a
+// caller that reads attributes and nothing else may use this.
+func (entry *Entry) DecodeAttributesOnly(blob []byte) error {
+	// The scratch buffer is only taken once a chunk is actually found, so an
+	// entry with none — every directory, for one — neither re-encodes nor
+	// touches the pool, and is unmarshalled where it lies.
+	var scratchPtr *[]byte
+	var scratch []byte
+	defer func() {
+		if scratchPtr != nil {
+			*scratchPtr = scratch
+			attributesScratchPool.Put(scratchPtr)
+		}
+	}()
+
+	var chunkExtent uint64
+	attributes := blob
+	for pos := 0; pos < len(blob); {
+		rest := blob[pos:]
+		num, typ, tagLen := protowire.ConsumeTag(rest)
+		if tagLen < 0 {
+			return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(tagLen))
+		}
+		var valLen int
+		if num == entryChunksField && typ == protowire.BytesType {
+			chunk, n := protowire.ConsumeBytes(rest[tagLen:])
+			if n < 0 {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(n))
+			}
+			valLen = n
+			end, err := chunkExtentEnd(chunk)
+			if err != nil {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, err)
+			}
+			if end > chunkExtent {
+				chunkExtent = end
+			}
+			if scratchPtr == nil {
+				scratchPtr = attributesScratchPool.Get().(*[]byte)
+				scratch = append((*scratchPtr)[:0], blob[:pos]...)
+			}
+		} else {
+			valLen = protowire.ConsumeFieldValue(num, typ, rest[tagLen:])
+			if valLen < 0 {
+				return fmt.Errorf("decoding value blob for %s: %w", entry.FullPath, protowire.ParseError(valLen))
+			}
+			if scratchPtr != nil {
+				scratch = append(scratch, rest[:tagLen+valLen]...)
+			}
+		}
+		pos += tagLen + valLen
+	}
+	if scratchPtr != nil {
+		attributes = scratch
+	}
+
+	message := pbEntryPool.Get().(*filer_pb.Entry)
+	defer func() {
+		resetPbEntry(message)
+		pbEntryPool.Put(message)
+	}()
+
+	if err := proto.Unmarshal(attributes, message); err != nil {
+		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
+	}
+
+	FromPbEntryToExistingEntry(message, entry)
+
+	// FromPbEntryToExistingEntry took the size over a chunk list that is not
+	// there, so fold in what the chunks actually reached. This is TotalSize.
+	if chunkExtent > entry.FileSize {
+		entry.FileSize = chunkExtent
+	}
+
+	return nil
+}
+
+// chunkExtentEnd reports where one encoded FileChunk ends, the offset plus size
+// that TotalSize maximises over, without building the chunk. A chunk this
+// rejects is one the full decoder rejects too, so a listing never reports a
+// size for an entry that cannot be opened.
+//
+// A manifest chunk needs no special handling: it carries the offset and size of
+// the whole range it stands for, and TotalSize does not resolve it either.
+func chunkExtentEnd(chunk []byte) (uint64, error) {
+	var offset int64
+	var size uint64
+	for len(chunk) > 0 {
+		num, typ, tagLen := protowire.ConsumeTag(chunk)
+		if tagLen < 0 {
+			return 0, protowire.ParseError(tagLen)
+		}
+		chunk = chunk[tagLen:]
+		if typ == protowire.VarintType && (num == fileChunkOffsetField || num == fileChunkSizeField) {
+			v, n := protowire.ConsumeVarint(chunk)
+			if n < 0 {
+				return 0, protowire.ParseError(n)
+			}
+			if num == fileChunkOffsetField {
+				offset = int64(v)
+			} else {
+				size = v
+			}
+			chunk = chunk[n:]
+			continue
+		}
+		if typ == protowire.BytesType {
+			v, n := protowire.ConsumeBytes(chunk)
+			if n < 0 {
+				return 0, protowire.ParseError(n)
+			}
+			if fileChunkMessageFields[num] {
+				if err := validateMessage(v); err != nil {
+					return 0, err
+				}
+			} else if fileChunkStringFields[num] && !utf8.Valid(v) {
+				return 0, errors.New("invalid UTF-8 in string field")
+			}
+			chunk = chunk[n:]
+			continue
+		}
+		n := protowire.ConsumeFieldValue(num, typ, chunk)
+		if n < 0 {
+			return 0, protowire.ParseError(n)
+		}
+		chunk = chunk[n:]
+	}
+	return uint64(offset + int64(size)), nil
+}
+
+// validateMessage walks an encoded message to check it parses, which is all the
+// generated unmarshaller would do for one whose fields are scalars.
+func validateMessage(b []byte) error {
+	for len(b) > 0 {
+		num, typ, tagLen := protowire.ConsumeTag(b)
+		if tagLen < 0 {
+			return protowire.ParseError(tagLen)
+		}
+		valLen := protowire.ConsumeFieldValue(num, typ, b[tagLen:])
+		if valLen < 0 {
+			return protowire.ParseError(valLen)
+		}
+		b = b[tagLen+valLen:]
+	}
+	return nil
+}