snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/worker/tasks/iceberg/filer_io.go

@@ -0,0 +1,484 @@
+package iceberg
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// filerFileEntry holds a non-directory entry with its full directory path.
+type filerFileEntry struct {
+	Dir   string
+	Entry *filer_pb.Entry
+}
+
+var initGlobalHTTPClientOnce sync.Once
+
+type singleFilerClient struct {
+	client filer_pb.SeaweedFilerClient
+}
+
+func (c singleFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
+	return fn(c.client)
+}
+
+func (c singleFilerClient) AdjustedUrl(location *filer_pb.Location) string {
+	if location == nil {
+		return ""
+	}
+	if location.PublicUrl != "" {
+		return location.PublicUrl
+	}
+	return location.Url
+}
+
+func (c singleFilerClient) GetDataCenter() string {
+	return ""
+}
+
+// listFilerEntries lists all entries in a directory.
+func listFilerEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, prefix string) ([]*filer_pb.Entry, error) {
+	var entries []*filer_pb.Entry
+	var lastFileName string
+	limit := uint32(10000)
+
+	for {
+		resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
+			Directory:          dir,
+			Prefix:             prefix,
+			StartFromFileName:  lastFileName,
+			InclusiveStartFrom: lastFileName == "",
+			Limit:              limit,
+		})
+		if err != nil {
+			// Treat not-found as empty directory; propagate other errors.
+			if status.Code(err) == codes.NotFound {
+				return entries, nil
+			}
+			return entries, fmt.Errorf("list entries in %s: %w", dir, err)
+		}
+
+		count := 0
+		for {
+			entry, recvErr := resp.Recv()
+			if recvErr != nil {
+				if recvErr == io.EOF {
+					break
+				}
+				return entries, fmt.Errorf("recv entry in %s: %w", dir, recvErr)
+			}
+			if entry.Entry != nil {
+				entries = append(entries, entry.Entry)
+				lastFileName = entry.Entry.Name
+				count++
+			}
+		}
+
+		if count < int(limit) {
+			break
+		}
+	}
+
+	return entries, nil
+}
+
+// walkFilerEntries recursively lists all non-directory entries under dir.
+func walkFilerEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string) ([]filerFileEntry, error) {
+	entries, err := listFilerEntries(ctx, client, dir, "")
+	if err != nil {
+		return nil, err
+	}
+
+	var result []filerFileEntry
+	for _, entry := range entries {
+		if entry.IsDirectory {
+			subDir := path.Join(dir, entry.Name)
+			subEntries, err := walkFilerEntries(ctx, client, subDir)
+			if err != nil {
+				glog.V(2).Infof("iceberg maintenance: cannot walk %s: %v", subDir, err)
+				continue
+			}
+			result = append(result, subEntries...)
+		} else {
+			result = append(result, filerFileEntry{Dir: dir, Entry: entry})
+		}
+	}
+	return result, nil
+}
+
+// loadCurrentMetadata loads and parses the current Iceberg metadata from the table entry's xattr.
+func loadCurrentMetadata(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, tablePath string) (*tableState, error) {
+	dir := path.Join(s3tables.TablesPath, bucketName, path.Dir(tablePath))
+	name := path.Base(tablePath)
+
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir,
+		Name:      name,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("lookup table entry %s/%s: %w", dir, name, err)
+	}
+	if resp == nil || resp.Entry == nil {
+		return nil, fmt.Errorf("table entry not found: %s/%s", dir, name)
+	}
+
+	metadataBytes, ok := resp.Entry.Extended[s3tables.ExtendedKeyMetadata]
+	if !ok || len(metadataBytes) == 0 {
+		return nil, fmt.Errorf("no metadata xattr on table entry %s/%s", dir, name)
+	}
+
+	state, err := parseTableMetadataEnvelope(metadataBytes, bucketName, tablePath)
+	if err != nil {
+		return nil, err
+	}
+	// Refuse to rewrite an entry that only looks like an Iceberg table. Every
+	// operation here rewrites or deletes files under the table location, and a
+	// foreign format's files are unreferenced by Iceberg metadata by definition.
+	if !isIcebergTableEntry(resp.Entry.Extended, state.Metadata) {
+		return nil, fmt.Errorf("entry %s/%s is not an iceberg table", dir, name)
+	}
+	return state, nil
+}
+
+// loadFileByIcebergPath loads a file from the filer given an Iceberg-style path.
+// dataPath is the bucket-relative directory the table's files live in.
+func loadFileByIcebergPath(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) ([]byte, error) {
+	fullPath, err := icebergFilerPath(bucketName, dataPath, icebergPath)
+	if err != nil {
+		return nil, err
+	}
+	dir, fileName := path.Dir(fullPath), path.Base(fullPath)
+
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: dir,
+		Name:      fileName,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("lookup %s/%s: %w", dir, fileName, err)
+	}
+	if resp == nil || resp.Entry == nil {
+		return nil, fmt.Errorf("file not found: %s/%s", dir, fileName)
+	}
+
+	if len(resp.Entry.Content) > 0 || len(resp.Entry.Chunks) == 0 {
+		return resp.Entry.Content, nil
+	}
+
+	initGlobalHTTPClientOnce.Do(util_http.InitGlobalHttpClient)
+	reader := filer.NewFileReader(singleFilerClient{client: client}, resp.Entry)
+	if closer, ok := reader.(io.Closer); ok {
+		defer closer.Close()
+	}
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("read chunked file %s/%s: %w", dir, fileName, err)
+	}
+	return data, nil
+}
+
+// normalizeIcebergPath converts an Iceberg path (an S3 URL, an absolute filer
+// path, or a path relative to the table root) into a path relative to the
+// bucket root, e.g. "ns/table/metadata/snap-1.avro". Absolute references are
+// resolved from the bucket root rather than from dataPath: an Iceberg table
+// records where its files actually are, and a client writing through the REST
+// catalog may place them outside the catalog path. The bucket-relative form is
+// the canonical key for comparing references that mix both spellings.
+func normalizeIcebergPath(icebergPath, bucketName, dataPath string) (string, error) {
+	p := icebergPath
+	absolute := false
+	if idx := strings.Index(p, "://"); idx >= 0 {
+		p = p[idx+3:]
+		absolute = true
+	} else if strings.HasPrefix(p, "/") {
+		absolute = true
+	}
+	p = strings.TrimPrefix(p, "/")
+
+	if absolute {
+		rest, ok := cutPathPrefix(p, bucketName)
+		if !ok {
+			// Filer-style references carry the /buckets prefix ahead of the bucket.
+			if withoutTablesPath, hasTablesPath := cutPathPrefix(p, strings.TrimPrefix(s3tables.TablesPath, "/")); hasTablesPath {
+				rest, ok = cutPathPrefix(withoutTablesPath, bucketName)
+			}
+		}
+		if !ok {
+			return "", fmt.Errorf("iceberg path %q is outside bucket %q", icebergPath, bucketName)
+		}
+		p = rest
+	} else {
+		if clean := path.Clean(p); clean == ".." || strings.HasPrefix(clean, "../") {
+			return "", fmt.Errorf("invalid iceberg path %q", icebergPath)
+		}
+		p = path.Join(dataPath, p)
+	}
+
+	p = path.Clean(p)
+	if p == "." || p == ".." || strings.HasPrefix(p, "../") {
+		return "", fmt.Errorf("invalid iceberg path %q", icebergPath)
+	}
+	return p, nil
+}
+
+// cutPathPrefix removes a leading path prefix at a path boundary.
+func cutPathPrefix(p, segment string) (string, bool) {
+	if segment == "" {
+		return p, false
+	}
+	rest, ok := strings.CutPrefix(p, segment+"/")
+	return rest, ok
+}
+
+// icebergFilerPath resolves an Iceberg path to the filer path holding the file.
+func icebergFilerPath(bucketName, dataPath, icebergPath string) (string, error) {
+	relPath, err := normalizeIcebergPath(icebergPath, bucketName, dataPath)
+	if err != nil {
+		return "", err
+	}
+	return path.Join(s3tables.TablesPath, bucketName, relPath), nil
+}
+
+// absoluteIcebergPath is the inverse of normalizeIcebergPath: it builds the
+// absolute s3:// URI for a bucket-relative path. The Iceberg spec requires
+// absolute locations in metadata — strict readers (Spark/Trino via S3FileIO)
+// reject paths with no scheme.
+func absoluteIcebergPath(bucketName string, elem ...string) string {
+	return "s3://" + path.Join(append([]string{bucketName}, elem...)...)
+}
+
+const (
+	// inlineContentLimit is the most saveFilerFile keeps in Entry.Content.
+	// Manifests and metadata JSON are small and stay inline; a compacted data
+	// file runs to hundreds of MB, and inlining that stores the parquet bytes
+	// verbatim in the filer store and ships them again through the metadata
+	// change log as one oversized event.
+	inlineContentLimit = 256 * 1024
+	// filerFileChunkSize is the upload chunk size for anything over the limit.
+	filerFileChunkSize = 8 * 1024 * 1024
+)
+
+// saveFilerFile saves a file to the filer, inline when it is small and as
+// volume chunks when it is not.
+func saveFilerFile(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, fileName string, content []byte) error {
+	entry := &filer_pb.Entry{
+		Name: fileName,
+		Attributes: &filer_pb.FuseAttributes{
+			Mtime:    time.Now().Unix(),
+			Crtime:   time.Now().Unix(),
+			FileMode: uint32(0644),
+			FileSize: uint64(len(content)),
+		},
+	}
+	if len(content) <= inlineContentLimit {
+		entry.Content = content
+	} else {
+		chunks, err := uploadFilerChunks(ctx, client, path.Join(dir, fileName), content)
+		if err != nil {
+			return fmt.Errorf("upload %s/%s: %w", dir, fileName, err)
+		}
+		entry.Chunks = chunks
+	}
+
+	resp, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
+		Directory: dir,
+		Entry:     entry,
+	})
+	if err != nil {
+		return fmt.Errorf("create entry %s/%s: %w", dir, fileName, err)
+	}
+	if resp.Error != "" {
+		return fmt.Errorf("create entry %s/%s: %s", dir, fileName, resp.Error)
+	}
+	return nil
+}
+
+// uploadFilerChunks writes content to volume servers, assigning through the
+// filer so the entry's storage rules apply.
+func uploadFilerChunks(ctx context.Context, client filer_pb.SeaweedFilerClient, fullPath string, content []byte) ([]*filer_pb.FileChunk, error) {
+	assignFn := func(ctx context.Context, count int, expectedDataSize uint64) (*operation.VolumeAssignRequest, *operation.AssignResult, error) {
+		resp, err := client.AssignVolume(ctx, &filer_pb.AssignVolumeRequest{
+			Count:            int32(count),
+			Path:             fullPath,
+			ExpectedDataSize: expectedDataSize,
+		})
+		if err != nil {
+			return nil, nil, err
+		}
+		if resp.Error != "" {
+			return nil, nil, errors.New(resp.Error)
+		}
+		if resp.Location == nil || resp.FileId == "" {
+			return nil, nil, fmt.Errorf("assign volume returned no location")
+		}
+		return nil, &operation.AssignResult{
+			Fid:       resp.FileId,
+			Url:       resp.Location.Url,
+			PublicUrl: resp.Location.PublicUrl,
+			Count:     uint64(count),
+			Auth:      security.EncodedJwt(resp.Auth),
+			Fsync:     resp.Fsync,
+		}, nil
+	}
+
+	initGlobalHTTPClientOnce.Do(util_http.InitGlobalHttpClient)
+	result, err := operation.UploadReaderInChunks(ctx, util.NewBytesReader(content), &operation.ChunkedUploadOption{
+		ChunkSize:  filerFileChunkSize,
+		AssignFunc: assignFn,
+	})
+	if err != nil {
+		// Chunks uploaded before the failure are orphaned: nothing references
+		// them, so name them for fsck rather than losing them silently.
+		if result != nil {
+			for _, chunk := range result.FileChunks {
+				glog.Warningf("iceberg: orphan chunk %s from failed upload of %s", chunk.GetFileIdString(), fullPath)
+			}
+		}
+		return nil, err
+	}
+	return result.FileChunks, nil
+}
+
+// deleteFilerFile deletes a file from the filer.
+func deleteFilerFile(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, fileName string) error {
+	return filer_pb.DoRemove(ctx, client, dir, fileName, true, false, true, false, nil)
+}
+
+// updateTableMetadataXattr updates the table entry's metadata xattr with
+// the new Iceberg metadata. It verifies the stored metadataVersion before
+// writing and passes the previous metadata xattr back to the filer as a
+// server-side precondition so concurrent writers fail with a retryable
+// metadata version conflict.
+// newMetadataLocation is the absolute location of the new metadata file
+// (e.g. "s3://bucket/ns/table/metadata/v3.metadata.json"), matching the form
+// the S3 Tables catalog stores.
+func updateTableMetadataXattr(ctx context.Context, client filer_pb.SeaweedFilerClient, tableDir string, expectedVersion int, newFullMetadata []byte, newMetadataLocation string) error {
+	tableName := path.Base(tableDir)
+	parentDir := path.Dir(tableDir)
+
+	resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+		Directory: parentDir,
+		Name:      tableName,
+	})
+	if err != nil {
+		return fmt.Errorf("lookup table entry: %w", err)
+	}
+	if resp == nil || resp.Entry == nil {
+		return fmt.Errorf("table entry not found")
+	}
+
+	existingXattr, ok := resp.Entry.Extended[s3tables.ExtendedKeyMetadata]
+	if !ok {
+		return fmt.Errorf("no metadata xattr on table entry")
+	}
+
+	// Parse existing xattr, update fullMetadata
+	var internalMeta map[string]json.RawMessage
+	if err := json.Unmarshal(existingXattr, &internalMeta); err != nil {
+		return fmt.Errorf("unmarshal existing xattr: %w", err)
+	}
+
+	// Verify the stored metadataVersion matches what we expect before issuing
+	// the conditional UpdateEntry request below.
+	versionRaw, ok := internalMeta["metadataVersion"]
+	if !ok {
+		return fmt.Errorf("%w: metadataVersion field missing from xattr", errMetadataVersionConflict)
+	}
+	var storedVersion int
+	if err := json.Unmarshal(versionRaw, &storedVersion); err != nil {
+		return fmt.Errorf("%w: cannot parse metadataVersion: %v", errMetadataVersionConflict, err)
+	}
+	if storedVersion != expectedVersion {
+		return fmt.Errorf("%w: expected version %d, found %d", errMetadataVersionConflict, expectedVersion, storedVersion)
+	}
+
+	// Update the metadata.fullMetadata field
+	var metadataObj map[string]json.RawMessage
+	if raw, ok := internalMeta["metadata"]; ok {
+		if err := json.Unmarshal(raw, &metadataObj); err != nil {
+			return fmt.Errorf("unmarshal metadata object: %w", err)
+		}
+	} else {
+		metadataObj = make(map[string]json.RawMessage)
+	}
+	metadataObj["fullMetadata"] = newFullMetadata
+	metadataJSON, err := json.Marshal(metadataObj)
+	if err != nil {
+		return fmt.Errorf("marshal metadata object: %w", err)
+	}
+	internalMeta["metadata"] = metadataJSON
+
+	// Increment version
+	newVersion := expectedVersion + 1
+	versionJSON, _ := json.Marshal(newVersion)
+	internalMeta["metadataVersion"] = versionJSON
+
+	// Update modifiedAt
+	modifiedAt, _ := json.Marshal(time.Now().Format(time.RFC3339Nano))
+	internalMeta["modifiedAt"] = modifiedAt
+
+	// Update metadataLocation to point to the new metadata file
+	metaLocJSON, _ := json.Marshal(newMetadataLocation)
+	internalMeta["metadataLocation"] = metaLocJSON
+
+	// Regenerate versionToken for consistency with the S3 Tables catalog
+	tokenJSON, _ := json.Marshal(generateIcebergVersionToken())
+	internalMeta["versionToken"] = tokenJSON
+
+	updatedXattr, err := json.Marshal(internalMeta)
+	if err != nil {
+		return fmt.Errorf("marshal updated xattr: %w", err)
+	}
+
+	// Assert the whole attribute set, not just the version: this rewrites the
+	// entry from the snapshot above, so a narrower precondition would delete an
+	// attribute a concurrent writer created, such as a maintenance configuration.
+	expectedExtended := s3tables.SnapshotExtended(resp.Entry.Extended)
+	resp.Entry.Extended[s3tables.ExtendedKeyMetadata] = updatedXattr
+	resp.Entry.Extended[s3tables.ExtendedKeyMetadataVersion] = metadataVersionXattr(newVersion)
+	_, err = client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
+		Directory:        parentDir,
+		Entry:            resp.Entry,
+		ExpectedExtended: expectedExtended,
+	})
+	if err != nil {
+		if status.Code(err) == codes.FailedPrecondition {
+			return fmt.Errorf("%w: table metadata changed during update", errMetadataVersionConflict)
+		}
+		return fmt.Errorf("update table entry: %w", err)
+	}
+	return nil
+}
+
+func metadataVersionXattr(version int) []byte {
+	return []byte(strconv.Itoa(version))
+}
+
+// generateIcebergVersionToken produces a random hex token, mirroring the
+// logic in s3tables.generateVersionToken (which is unexported).
+func generateIcebergVersionToken() string {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Sprintf("%x", time.Now().UnixNano())
+	}
+	return hex.EncodeToString(b)
+}