snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
685
weed/server/filer_server_tus_session.go
Normal file
685
weed/server/filer_server_tus_session.go
Normal file
@@ -0,0 +1,685 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/constants"
|
||||
)
|
||||
|
||||
const (
|
||||
TusVersion = "1.0.0"
|
||||
TusDefaultMaxSize = int64(5 * 1024 * 1024 * 1024) // 5GB
|
||||
TusDefaultSessionExpiry = 24 * time.Hour
|
||||
TusUploadsFolder = ".uploads.tus"
|
||||
TusInfoFileName = ".info"
|
||||
TusConsumedFileName = ".consumed"
|
||||
TusChunkExt = ".chunk"
|
||||
TusExtensions = "creation,creation-with-upload,termination,concatenation"
|
||||
TusConcatPartial = "partial"
|
||||
TusConcatFinalPrefix = "final;"
|
||||
)
|
||||
|
||||
// ErrWormEnforced marks a TUS completion rejected because the target entry is
|
||||
// WORM-protected. It shares the message the normal write path uses so it maps to
|
||||
// the same client-facing status.
|
||||
var ErrWormEnforced = errors.New(constants.ErrMsgOperationNotPermitted)
|
||||
|
||||
// TusSession represents an in-progress TUS upload session
|
||||
type TusSession struct {
|
||||
ID string `json:"id"`
|
||||
TargetPath string `json:"target_path"`
|
||||
Size int64 `json:"size"`
|
||||
Offset int64 `json:"offset"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ExpiresAt time.Time `json:"expires_at,omitempty"`
|
||||
Concat string `json:"concat,omitempty"`
|
||||
Chunks []*TusChunkInfo `json:"chunks,omitempty"`
|
||||
}
|
||||
|
||||
// isPartial reports whether the session is a concatenation partial upload: it
|
||||
// holds chunks for a later final upload instead of landing at its target path.
|
||||
func (session *TusSession) isPartial() bool {
|
||||
return session.Concat == TusConcatPartial
|
||||
}
|
||||
|
||||
func (session *TusSession) isFinal() bool {
|
||||
return strings.HasPrefix(session.Concat, TusConcatFinalPrefix)
|
||||
}
|
||||
|
||||
// TusChunkInfo tracks individual chunk uploads within a session
|
||||
type TusChunkInfo struct {
|
||||
Offset int64 `json:"offset"`
|
||||
Size int64 `json:"size"`
|
||||
FileId string `json:"file_id"`
|
||||
UploadAt int64 `json:"upload_at"`
|
||||
}
|
||||
|
||||
// tusSessionDir returns the directory path for storing TUS upload sessions
|
||||
func (fs *FilerServer) tusSessionDir() string {
|
||||
return "/" + TusUploadsFolder
|
||||
}
|
||||
|
||||
// tusSessionPath returns the path to a specific upload session directory
|
||||
func (fs *FilerServer) tusSessionPath(uploadID string) string {
|
||||
return fmt.Sprintf("/%s/%s", TusUploadsFolder, uploadID)
|
||||
}
|
||||
|
||||
// tusSessionInfoPath returns the path to the session info file
|
||||
func (fs *FilerServer) tusSessionInfoPath(uploadID string) string {
|
||||
return fmt.Sprintf("/%s/%s/%s", TusUploadsFolder, uploadID, TusInfoFileName)
|
||||
}
|
||||
|
||||
// tusSessionConsumedPath returns the path of the marker recording that a
|
||||
// session's chunks belong to a completed upload and must not be freed with it.
|
||||
func (fs *FilerServer) tusSessionConsumedPath(uploadID string) string {
|
||||
return fmt.Sprintf("/%s/%s/%s", TusUploadsFolder, uploadID, TusConsumedFileName)
|
||||
}
|
||||
|
||||
// markTusSessionConsumed claims a session's chunks for a completed upload. With
|
||||
// exclusive set, a session already claimed by a concurrent request fails with
|
||||
// filer_pb.ErrEntryAlreadyExists so one partial cannot be consumed twice.
|
||||
func (fs *FilerServer) markTusSessionConsumed(ctx context.Context, uploadID string, exclusive bool) error {
|
||||
return fs.filer.CreateEntry(ctx, &filer.Entry{
|
||||
FullPath: util.FullPath(fs.tusSessionConsumedPath(uploadID)),
|
||||
Attr: filer.Attr{
|
||||
Mode: 0644,
|
||||
Crtime: time.Now(),
|
||||
Mtime: time.Now(),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
},
|
||||
}, nil, exclusive, false, nil, true, fs.filer.MaxFilenameLength)
|
||||
}
|
||||
|
||||
// isTusSessionConsumed fails closed: when the marker cannot be looked up, the
|
||||
// caller must not treat the session's chunks as free.
|
||||
// claimTusPartial claims a partial for one final upload, serialized per session
|
||||
// on this filer, and re-verifies the pinned session under the claim. A failed
|
||||
// verification releases the claim before returning.
|
||||
func (fs *FilerServer) claimTusPartial(ctx context.Context, partial *TusSession) error {
|
||||
sessionPath := util.FullPath(fs.tusSessionPath(partial.ID))
|
||||
pathLock := fs.entryLockTable.AcquireLock("tusClaim", sessionPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(sessionPath, pathLock)
|
||||
|
||||
if err := fs.markTusSessionConsumed(ctx, partial.ID, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fs.refreshTusSessionChunks(ctx, partial); err != nil {
|
||||
fs.rollbackTusSessionConsumed(ctx, partial.ID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollbackTusSessionConsumed releases a consumed marker after a failed claim or
|
||||
// completion. A failed rollback wedges the session as consumed: DELETE and
|
||||
// expiry then preserve its chunks, which leak until removed by fsck.
|
||||
func (fs *FilerServer) rollbackTusSessionConsumed(ctx context.Context, uploadID string) {
|
||||
if err := fs.filer.DeleteEntryMetaAndData(ctx, util.FullPath(fs.tusSessionConsumedPath(uploadID)), false, false, false, false, nil, 0); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
glog.Errorf("TUS session %s wedged as consumed, marker rollback failed: %v", uploadID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *FilerServer) isTusSessionConsumed(ctx context.Context, uploadID string) (bool, error) {
|
||||
_, err := fs.filer.FindEntry(ctx, util.FullPath(fs.tusSessionConsumedPath(uploadID)))
|
||||
if err == nil {
|
||||
return true, nil
|
||||
}
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
// tusChunkPath returns the path to store a chunk info file
|
||||
// Format: /{TusUploadsFolder}/{uploadID}/chunk_{offset}_{size}_{encodedFileId}
|
||||
func (fs *FilerServer) tusChunkPath(uploadID string, offset, size int64, fileId string) string {
|
||||
// Use URL-safe base64 encoding to safely encode fileId (handles both / and _ in fileId)
|
||||
encodedFileId := base64.RawURLEncoding.EncodeToString([]byte(fileId))
|
||||
return fmt.Sprintf("/%s/%s/chunk_%016d_%016d_%s", TusUploadsFolder, uploadID, offset, size, encodedFileId)
|
||||
}
|
||||
|
||||
// parseTusChunkPath parses chunk info from a chunk entry
|
||||
// The entry's Crtime is used for the UploadAt timestamp to preserve the actual upload time
|
||||
func parseTusChunkPath(entry *filer.Entry) (*TusChunkInfo, error) {
|
||||
name := entry.Name()
|
||||
if !strings.HasPrefix(name, "chunk_") {
|
||||
return nil, fmt.Errorf("not a chunk file: %s", name)
|
||||
}
|
||||
// Use strings.Cut to correctly handle base64-encoded fileId which may contain underscores
|
||||
s := name[6:] // Skip "chunk_" prefix
|
||||
offsetStr, rest, found := strings.Cut(s, "_")
|
||||
if !found {
|
||||
return nil, fmt.Errorf("invalid chunk file name format (missing offset): %s", name)
|
||||
}
|
||||
sizeStr, encodedFileId, found := strings.Cut(rest, "_")
|
||||
if !found {
|
||||
return nil, fmt.Errorf("invalid chunk file name format (missing size): %s", name)
|
||||
}
|
||||
|
||||
offset, err := strconv.ParseInt(offsetStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid offset in chunk file %q: %w", name, err)
|
||||
}
|
||||
size, err := strconv.ParseInt(sizeStr, 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid size in chunk file %q: %w", name, err)
|
||||
}
|
||||
// Decode fileId from URL-safe base64
|
||||
fileIdBytes, err := base64.RawURLEncoding.DecodeString(encodedFileId)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid fileId encoding in chunk file %q: %w", name, err)
|
||||
}
|
||||
return &TusChunkInfo{
|
||||
Offset: offset,
|
||||
Size: size,
|
||||
FileId: string(fileIdBytes),
|
||||
UploadAt: entry.Crtime.UnixNano(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// createTusSession creates a new TUS upload session
|
||||
func (fs *FilerServer) createTusSession(ctx context.Context, uploadID, targetPath string, size int64, metadata map[string]string, concat string) (*TusSession, error) {
|
||||
session := &TusSession{
|
||||
ID: uploadID,
|
||||
TargetPath: targetPath,
|
||||
Size: size,
|
||||
Offset: 0,
|
||||
Metadata: metadata,
|
||||
CreatedAt: time.Now(),
|
||||
ExpiresAt: time.Now().Add(fs.option.TusSessionExpiry),
|
||||
Concat: concat,
|
||||
Chunks: []*TusChunkInfo{},
|
||||
}
|
||||
|
||||
// Create session directory
|
||||
sessionDirPath := util.FullPath(fs.tusSessionPath(uploadID))
|
||||
if err := fs.filer.CreateEntry(ctx, &filer.Entry{
|
||||
FullPath: sessionDirPath,
|
||||
Attr: filer.Attr{
|
||||
Mode: os.ModeDir | 0755,
|
||||
Crtime: time.Now(),
|
||||
Mtime: time.Now(),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
},
|
||||
}, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {
|
||||
return nil, fmt.Errorf("create session directory: %w", err)
|
||||
}
|
||||
|
||||
// Save session info
|
||||
if err := fs.saveTusSession(ctx, session); err != nil {
|
||||
// Cleanup the directory on failure
|
||||
fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, true, false, false, nil, 0)
|
||||
return nil, fmt.Errorf("save session info: %w", err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Created TUS session %s for %s, size=%d", uploadID, targetPath, size)
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// saveTusSession saves the session info to the filer
|
||||
func (fs *FilerServer) saveTusSession(ctx context.Context, session *TusSession) error {
|
||||
sessionData, err := json.Marshal(session)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal session: %w", err)
|
||||
}
|
||||
|
||||
infoPath := util.FullPath(fs.tusSessionInfoPath(session.ID))
|
||||
entry := &filer.Entry{
|
||||
FullPath: infoPath,
|
||||
Attr: filer.Attr{
|
||||
Mode: 0644,
|
||||
Crtime: session.CreatedAt,
|
||||
Mtime: time.Now(),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
},
|
||||
Content: sessionData,
|
||||
}
|
||||
|
||||
if err := fs.filer.CreateEntry(ctx, entry, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {
|
||||
return fmt.Errorf("save session info entry: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readTusSessionInfo reads and validates a session's immutable .info metadata
|
||||
// without listing its chunks, the cheap lookup the authorization check needs for
|
||||
// the stored TargetPath. It rejects a metadata file whose id, target or size is
|
||||
// unusable so a corrupt or replaced session cannot be authorized or completed.
|
||||
func (fs *FilerServer) readTusSessionInfo(ctx context.Context, uploadID string) (*TusSession, error) {
|
||||
if !isCanonicalTusUploadID(uploadID) {
|
||||
return nil, fmt.Errorf("invalid TUS upload id: %q", uploadID)
|
||||
}
|
||||
infoPath := util.FullPath(fs.tusSessionInfoPath(uploadID))
|
||||
entry, err := fs.filer.FindEntry(ctx, infoPath)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
return nil, fmt.Errorf("TUS upload session not found: %s: %w", uploadID, filer_pb.ErrNotFound)
|
||||
}
|
||||
return nil, fmt.Errorf("find session: %w", err)
|
||||
}
|
||||
|
||||
var session TusSession
|
||||
if err := json.Unmarshal(entry.Content, &session); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal session: %w", err)
|
||||
}
|
||||
if session.ID != uploadID {
|
||||
return nil, fmt.Errorf("TUS session id mismatch: got %q, want %q", session.ID, uploadID)
|
||||
}
|
||||
target := canonicalTusTargetPath(session.TargetPath)
|
||||
if target == "" || target == "/" {
|
||||
return nil, fmt.Errorf("invalid TUS target path: %q", session.TargetPath)
|
||||
}
|
||||
if session.Size < 0 || session.Size > fs.option.TusMaxSize {
|
||||
return nil, fmt.Errorf("invalid TUS upload size: %d", session.Size)
|
||||
}
|
||||
// Pin authorization and every later operation to the same canonical path.
|
||||
session.TargetPath = target
|
||||
return &session, nil
|
||||
}
|
||||
|
||||
// getTusSession retrieves a validated TUS session by upload ID, including its
|
||||
// chunks and current offset.
|
||||
func (fs *FilerServer) getTusSession(ctx context.Context, uploadID string) (*TusSession, error) {
|
||||
session, err := fs.readTusSessionInfo(ctx, uploadID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := fs.loadTusSessionChunks(ctx, session); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return session, nil
|
||||
}
|
||||
|
||||
// loadTusSessionChunks refreshes a session's chunks and offset from its session
|
||||
// directory, leaving the immutable .info metadata untouched.
|
||||
func (fs *FilerServer) loadTusSessionChunks(ctx context.Context, session *TusSession) error {
|
||||
// Load chunks from directory listing with pagination (atomic read, no race condition)
|
||||
sessionDirPath := util.FullPath(fs.tusSessionPath(session.ID))
|
||||
session.Chunks = nil
|
||||
session.Offset = 0
|
||||
|
||||
lastFileName := ""
|
||||
pageSize := 1000
|
||||
for {
|
||||
entries, hasMore, err := fs.filer.ListDirectoryEntries(ctx, sessionDirPath, lastFileName, false, int64(pageSize), "", "", "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("list session directory: %w", err)
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if strings.HasPrefix(e.Name(), "chunk_") {
|
||||
chunk, parseErr := parseTusChunkPath(e)
|
||||
if parseErr != nil {
|
||||
glog.V(1).Infof("Skipping invalid chunk file %s: %v", e.Name(), parseErr)
|
||||
continue
|
||||
}
|
||||
session.Chunks = append(session.Chunks, chunk)
|
||||
}
|
||||
lastFileName = e.Name()
|
||||
}
|
||||
|
||||
if !hasMore || len(entries) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Sort chunks by offset and compute current offset as maximum contiguous range from 0
|
||||
if len(session.Chunks) > 0 {
|
||||
sort.Slice(session.Chunks, func(i, j int) bool {
|
||||
return session.Chunks[i].Offset < session.Chunks[j].Offset
|
||||
})
|
||||
// Compute the maximum contiguous offset from 0
|
||||
// This correctly handles gaps in the upload sequence
|
||||
contiguousEnd := int64(0)
|
||||
for _, chunk := range session.Chunks {
|
||||
if chunk.Offset > contiguousEnd {
|
||||
// Gap detected, stop at the first gap
|
||||
break
|
||||
}
|
||||
chunkEnd := chunk.Offset + chunk.Size
|
||||
if chunkEnd > contiguousEnd {
|
||||
contiguousEnd = chunkEnd
|
||||
}
|
||||
}
|
||||
session.Offset = contiguousEnd
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyTusSessionUnchanged confirms the stored .info still identifies the same
|
||||
// pinned session.
|
||||
func (fs *FilerServer) verifyTusSessionUnchanged(ctx context.Context, session *TusSession) error {
|
||||
stored, err := fs.readTusSessionInfo(ctx, session.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stored.TargetPath != session.TargetPath || stored.Size != session.Size || !stored.CreatedAt.Equal(session.CreatedAt) {
|
||||
return fmt.Errorf("TUS session identity changed: %s", session.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// refreshTusSessionChunks verifies the pinned session still exists and still
|
||||
// identifies the same upload before refreshing its chunk state, so a PATCH
|
||||
// cannot complete after a concurrent DELETE or metadata replacement and land at
|
||||
// a TargetPath other than the one that was authorized.
|
||||
func (fs *FilerServer) refreshTusSessionChunks(ctx context.Context, session *TusSession) error {
|
||||
if err := fs.verifyTusSessionUnchanged(ctx, session); err != nil {
|
||||
return err
|
||||
}
|
||||
return fs.loadTusSessionChunks(ctx, session)
|
||||
}
|
||||
|
||||
// saveTusChunk stores the chunk info as a separate file entry
|
||||
// This avoids read-modify-write race conditions across multiple filer instances
|
||||
// The chunk metadata is encoded in the filename; the entry's Crtime preserves upload time
|
||||
func (fs *FilerServer) saveTusChunk(ctx context.Context, uploadID string, chunk *TusChunkInfo) error {
|
||||
if chunk == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Store chunk info as a separate file entry (atomic operation)
|
||||
// Chunk metadata is encoded in the filename; Crtime is used for UploadAt when reading back
|
||||
chunkPath := util.FullPath(fs.tusChunkPath(uploadID, chunk.Offset, chunk.Size, chunk.FileId))
|
||||
|
||||
if err := fs.filer.CreateEntry(ctx, &filer.Entry{
|
||||
FullPath: chunkPath,
|
||||
Attr: filer.Attr{
|
||||
Mode: 0644,
|
||||
Crtime: time.Now(),
|
||||
Mtime: time.Now(),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
},
|
||||
}, nil, false, false, nil, false, fs.filer.MaxFilenameLength); err != nil {
|
||||
return fmt.Errorf("save chunk info: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteTusChunk drops a chunk's record before freeing its data, so a save that
|
||||
// failed after the record landed cannot leave the session pointing at a needle
|
||||
// that is about to be deleted. Data outliving a lost record only leaks.
|
||||
func (fs *FilerServer) deleteTusChunk(ctx context.Context, session *TusSession, chunk *TusChunkInfo) {
|
||||
chunkPath := util.FullPath(fs.tusChunkPath(session.ID, chunk.Offset, chunk.Size, chunk.FileId))
|
||||
if err := fs.filer.DeleteEntryMetaAndData(ctx, chunkPath, false, false, false, false, nil, 0); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
glog.Errorf("TUS chunk %s record kept, its data leaks: %v", chunkPath, err)
|
||||
return
|
||||
}
|
||||
fs.filer.DeleteChunks(ctx, util.FullPath(session.TargetPath), []*filer_pb.FileChunk{{FileId: chunk.FileId}})
|
||||
}
|
||||
|
||||
// deleteTusSession removes a TUS upload session and all its data
|
||||
func (fs *FilerServer) deleteTusSession(ctx context.Context, uploadID string) error {
|
||||
sessionPath := util.FullPath(fs.tusSessionPath(uploadID))
|
||||
pathLock := fs.entryLockTable.AcquireLock("tusDelete", sessionPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(sessionPath, pathLock)
|
||||
|
||||
session, err := fs.getTusSession(ctx, uploadID)
|
||||
if err != nil {
|
||||
// Session might already be deleted or never existed
|
||||
glog.V(1).Infof("TUS session %s not found for deletion: %v", uploadID, err)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove the .info before deciding about chunk data: a final request claims
|
||||
// its consumed marker before re-verifying the .info, so once the .info is
|
||||
// gone no new claim can pass verification, and a claim that did pass was
|
||||
// created earlier and is visible at the check below.
|
||||
if err := fs.filer.DeleteEntryMetaAndData(ctx, util.FullPath(fs.tusSessionInfoPath(uploadID)), false, false, false, false, nil, 0); err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return fmt.Errorf("delete session info: %w", err)
|
||||
}
|
||||
|
||||
// Batch delete all uploaded chunks from volume servers, unless the session
|
||||
// was consumed: then the chunks belong to a completed upload's entry.
|
||||
consumed, err := fs.isTusSessionConsumed(ctx, uploadID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("check consumed marker: %w", err)
|
||||
}
|
||||
if len(session.Chunks) > 0 && !consumed {
|
||||
var chunksToDelete []*filer_pb.FileChunk
|
||||
for _, chunk := range session.Chunks {
|
||||
if chunk.FileId != "" {
|
||||
chunksToDelete = append(chunksToDelete, &filer_pb.FileChunk{FileId: chunk.FileId})
|
||||
}
|
||||
}
|
||||
if len(chunksToDelete) > 0 {
|
||||
fs.filer.DeleteChunks(ctx, util.FullPath(session.TargetPath), chunksToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the session directory
|
||||
sessionDirPath := util.FullPath(fs.tusSessionPath(uploadID))
|
||||
if err := fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, true, false, false, nil, 0); err != nil {
|
||||
return fmt.Errorf("delete session directory: %w", err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Deleted TUS session %s", uploadID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// completeTusUpload assembles all chunks and creates the final file
|
||||
func (fs *FilerServer) completeTusUpload(ctx context.Context, session *TusSession) error {
|
||||
if session.Offset != session.Size {
|
||||
return fmt.Errorf("upload incomplete: offset=%d, expected=%d", session.Offset, session.Size)
|
||||
}
|
||||
|
||||
// Serialize the ownership transition with deleteTusSession on this filer;
|
||||
// the marker/.info handshake below covers a delete served by another filer.
|
||||
sessionPath := util.FullPath(fs.tusSessionPath(session.ID))
|
||||
pathLock := fs.entryLockTable.AcquireLock("tusComplete", sessionPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(sessionPath, pathLock)
|
||||
|
||||
// Sort by offset, widest record first among equals, so a duplicate sorts
|
||||
// behind the record that covers it
|
||||
sort.Slice(session.Chunks, func(i, j int) bool {
|
||||
if session.Chunks[i].Offset != session.Chunks[j].Offset {
|
||||
return session.Chunks[i].Offset < session.Chunks[j].Offset
|
||||
}
|
||||
return session.Chunks[i].Size > session.Chunks[j].Size
|
||||
})
|
||||
|
||||
// A PATCH retried while its predecessor was still storing a sub-chunk can
|
||||
// record one range twice. The records must cover the full size with no gap,
|
||||
// the same watermark HEAD reports the offset from; a record extending
|
||||
// coverage joins the entry (the read path resolves a partial overlap by
|
||||
// ModifiedTsNs, and the raced copies carry identical bytes), while a fully
|
||||
// covered duplicate is freed once the entry lands.
|
||||
var fileChunks []*filer_pb.FileChunk
|
||||
var duplicateChunks []*filer_pb.FileChunk
|
||||
covered := int64(0)
|
||||
for _, chunk := range session.Chunks {
|
||||
if chunk.Offset > covered {
|
||||
return fmt.Errorf("chunk gap detected: covered up to %d, next chunk at %d", covered, chunk.Offset)
|
||||
}
|
||||
if chunk.Offset+chunk.Size <= covered {
|
||||
duplicateChunks = append(duplicateChunks, &filer_pb.FileChunk{FileId: chunk.FileId})
|
||||
continue
|
||||
}
|
||||
covered = chunk.Offset + chunk.Size
|
||||
|
||||
fid, fidErr := filer_pb.ToFileIdObject(chunk.FileId)
|
||||
if fidErr != nil {
|
||||
return fmt.Errorf("invalid file ID %s at offset %d: %w", chunk.FileId, chunk.Offset, fidErr)
|
||||
}
|
||||
|
||||
fileChunks = append(fileChunks, &filer_pb.FileChunk{
|
||||
FileId: chunk.FileId,
|
||||
Offset: chunk.Offset,
|
||||
Size: uint64(chunk.Size),
|
||||
ModifiedTsNs: chunk.UploadAt,
|
||||
Fid: fid,
|
||||
})
|
||||
}
|
||||
if covered != session.Size {
|
||||
return fmt.Errorf("chunks do not cover full file: chunks end at %d, expected %d", covered, session.Size)
|
||||
}
|
||||
|
||||
// Determine content type from metadata
|
||||
contentType := ""
|
||||
if session.Metadata != nil {
|
||||
if ct, ok := session.Metadata["content-type"]; ok {
|
||||
contentType = ct
|
||||
}
|
||||
}
|
||||
|
||||
// Create the final file entry
|
||||
targetPath := util.FullPath(session.TargetPath)
|
||||
entry := &filer.Entry{
|
||||
FullPath: targetPath,
|
||||
Attr: filer.Attr{
|
||||
Mode: 0644,
|
||||
Crtime: session.CreatedAt,
|
||||
Mtime: time.Now(),
|
||||
Uid: OS_UID,
|
||||
Gid: OS_GID,
|
||||
Mime: contentType,
|
||||
},
|
||||
Chunks: fileChunks,
|
||||
}
|
||||
|
||||
// Apply the same storage rule (read-only prefixes, TTL) and WORM protections
|
||||
// the normal write path enforces before landing the entry at the
|
||||
// client-chosen target path.
|
||||
so, err := fs.applyStorageDefaultsToEntry(ctx, entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if wormEnforced, err := fs.wormEnforcedForEntry(ctx, string(targetPath)); err != nil {
|
||||
return fmt.Errorf("check worm: %w", err)
|
||||
} else if wormEnforced {
|
||||
return ErrWormEnforced
|
||||
}
|
||||
|
||||
// Claim the chunks for the entry before creating it: once the marker is
|
||||
// durable, a failed cleanup below cannot lead DELETE or expiry to free the
|
||||
// entry's chunks. If the entry creation then fails, the client retry re-runs
|
||||
// completion; an abandoned marked session leaks its chunks instead of
|
||||
// corrupting a live entry.
|
||||
if err := fs.markTusSessionConsumed(ctx, session.ID, false); err != nil {
|
||||
return fmt.Errorf("mark session consumed: %w", err)
|
||||
}
|
||||
|
||||
// Handshake with deleteTusSession, which removes the .info before checking
|
||||
// the marker: a session whose .info is still present here cannot have its
|
||||
// chunks freed by a delete that missed the marker just made durable.
|
||||
if err := fs.verifyTusSessionUnchanged(ctx, session); err != nil {
|
||||
fs.rollbackTusSessionConsumed(ctx, session.ID)
|
||||
return fmt.Errorf("session deleted before completion: %w", err)
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
if err := fs.filer.CreateEntry(ctx, entry, nil, false, false, nil, false, so.MaxFileNameLength); err != nil {
|
||||
return fmt.Errorf("create final file entry: %w", err)
|
||||
}
|
||||
|
||||
// Free the duplicates' data; their records go with the session directory
|
||||
// below. A file id the entry still references is never freed: coverage is
|
||||
// computed from ranges, so a malformed record could name one.
|
||||
if len(duplicateChunks) > 0 {
|
||||
referenced := make(map[string]bool, len(fileChunks))
|
||||
for _, chunk := range fileChunks {
|
||||
referenced[chunk.FileId] = true
|
||||
}
|
||||
var chunksToDelete []*filer_pb.FileChunk
|
||||
for _, chunk := range duplicateChunks {
|
||||
if !referenced[chunk.FileId] {
|
||||
chunksToDelete = append(chunksToDelete, chunk)
|
||||
}
|
||||
}
|
||||
if len(chunksToDelete) > 0 {
|
||||
fs.filer.DeleteChunks(ctx, targetPath, chunksToDelete)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the session (but keep the chunks since they're now part of the final file)
|
||||
sessionDirPath := util.FullPath(fs.tusSessionPath(session.ID))
|
||||
if err := fs.filer.DeleteEntryMetaAndData(ctx, sessionDirPath, true, false, false, false, nil, 0); err != nil {
|
||||
glog.V(1).Infof("Failed to cleanup TUS session directory %s: %v", session.ID, err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("Completed TUS upload %s -> %s, size=%d, chunks=%d",
|
||||
session.ID, session.TargetPath, session.Size, len(fileChunks))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartTusSessionCleanup starts a background goroutine that periodically cleans up expired TUS sessions
|
||||
func (fs *FilerServer) StartTusSessionCleanup(interval time.Duration) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
fs.cleanupExpiredTusSessions()
|
||||
}
|
||||
}()
|
||||
glog.V(0).Infof("TUS session cleanup started with interval %v", interval)
|
||||
}
|
||||
|
||||
// cleanupExpiredTusSessions scans for and removes expired TUS upload sessions
|
||||
func (fs *FilerServer) cleanupExpiredTusSessions() {
|
||||
ctx := context.Background()
|
||||
uploadsDir := util.FullPath(fs.tusSessionDir())
|
||||
|
||||
// List all session directories under the TUS uploads folder
|
||||
var lastFileName string
|
||||
const pageSize = 100
|
||||
|
||||
for {
|
||||
entries, hasMore, err := fs.filer.ListDirectoryEntries(ctx, uploadsDir, lastFileName, false, int64(pageSize), "", "", "")
|
||||
if err != nil {
|
||||
glog.V(1).Infof("TUS cleanup: failed to list sessions: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDirectory() {
|
||||
lastFileName = entry.Name()
|
||||
continue
|
||||
}
|
||||
|
||||
uploadID := entry.Name()
|
||||
session, err := fs.getTusSession(ctx, uploadID)
|
||||
if err != nil {
|
||||
glog.V(2).Infof("TUS cleanup: skipping session %s: %v", uploadID, err)
|
||||
lastFileName = uploadID
|
||||
continue
|
||||
}
|
||||
|
||||
if !session.ExpiresAt.IsZero() && now.After(session.ExpiresAt) {
|
||||
glog.V(1).Infof("TUS cleanup: removing expired session %s (expired at %v)", uploadID, session.ExpiresAt)
|
||||
if err := fs.deleteTusSession(ctx, uploadID); err != nil {
|
||||
glog.V(1).Infof("TUS cleanup: failed to delete session %s: %v", uploadID, err)
|
||||
}
|
||||
}
|
||||
|
||||
lastFileName = uploadID
|
||||
}
|
||||
|
||||
if !hasMore || len(entries) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user