snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
663
weed/command/filer.go
Normal file
663
weed/command/filer.go
Normal file
@@ -0,0 +1,663 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
"google.golang.org/grpc/reflection"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/credential/filer_etc"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/credential/memory"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/credential/postgres"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
||||
)
|
||||
|
||||
var (
|
||||
f FilerOptions
|
||||
filerStartS3 *bool
|
||||
filerS3Options S3Options
|
||||
filerStartWebDav *bool
|
||||
filerWebDavOptions WebDavOption
|
||||
filerStartIam *bool
|
||||
filerIamOptions IamOptions
|
||||
filerStartSftp *bool
|
||||
filerSftpOptions SftpOptions
|
||||
)
|
||||
|
||||
// allowUntrustedRemoteEndpointsUsage documents the flag shared by the filer and
|
||||
// S3 gateway, whose remote-mount read paths dial the mounted endpoint directly.
|
||||
const allowUntrustedRemoteEndpointsUsage = "if true, a read of a remote-only entry accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints."
|
||||
|
||||
type FilerOptions struct {
|
||||
masters *pb.ServerDiscovery
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
port *int
|
||||
portGrpc *int
|
||||
publicPort *int
|
||||
filerGroup *string
|
||||
collection *string
|
||||
defaultReplicaPlacement *string
|
||||
disableDirListing *bool
|
||||
maxMB *int
|
||||
dirListingLimit *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
enableNotification *bool
|
||||
disableHttp *bool
|
||||
cipher *bool
|
||||
metricsHttpPort *int
|
||||
metricsHttpIp *string
|
||||
saveToFilerLimit *int
|
||||
defaultLevelDbDirectory *string
|
||||
concurrentUploadLimitMB *int
|
||||
concurrentFileUploadLimit *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
allowedOrigins *string
|
||||
exposeDirectoryData *bool
|
||||
tusBasePath *string
|
||||
tusMaxSizeMB *int
|
||||
tusSessionExpiry *time.Duration
|
||||
s3ConfigFile *string // optional path to static S3 identity config
|
||||
|
||||
allowUntrustedRemoteEndpoints *bool
|
||||
// shutdownCtx, when non-nil, tells startFiler to gracefully shut down its
|
||||
// HTTP/gRPC servers once the ctx is cancelled. Used by integration tests
|
||||
// and by weed mini; nil for standalone weed filer.
|
||||
shutdownCtx context.Context
|
||||
// gracefulStopTimeout caps how long startFiler waits for gRPC graceful
|
||||
// stop before forcing the server to stop. Zero means the default of 10s.
|
||||
gracefulStopTimeout time.Duration
|
||||
}
|
||||
|
||||
func init() {
|
||||
cmdFiler.Run = runFiler // break init cycle
|
||||
f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers or a single DNS SRV record of at least 1 master server, prepended with dnssrv+")
|
||||
f.filerGroup = cmdFiler.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
|
||||
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
|
||||
f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
|
||||
f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
|
||||
f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
|
||||
f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
|
||||
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
|
||||
f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
|
||||
f.maxMB = cmdFiler.Flag.Int("maxMB", 4, "split files larger than the limit")
|
||||
f.dirListingLimit = cmdFiler.Flag.Int("dirListLimit", 100000, "limit sub dir listing size")
|
||||
f.dataCenter = cmdFiler.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
f.rack = cmdFiler.Flag.String("rack", "", "prefer to write to volumes in this rack")
|
||||
f.disableHttp = cmdFiler.Flag.Bool("disableHttp", false, "disable http request, only gRpc operations are allowed")
|
||||
f.cipher = cmdFiler.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
|
||||
f.metricsHttpPort = cmdFiler.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
f.metricsHttpIp = cmdFiler.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
|
||||
f.saveToFilerLimit = cmdFiler.Flag.Int("saveToFilerLimit", 0, "files smaller than this limit will be saved in filer store")
|
||||
f.defaultLevelDbDirectory = cmdFiler.Flag.String("defaultStoreDir", ".", "if filer.toml is empty, use an embedded filer store in the directory")
|
||||
f.concurrentUploadLimitMB = cmdFiler.Flag.Int("concurrentUploadLimitMB", 0, "limit total concurrent upload size, 0 means unlimited")
|
||||
f.concurrentFileUploadLimit = cmdFiler.Flag.Int("concurrentFileUploadLimit", 0, "limit number of concurrent file uploads, 0 means unlimited")
|
||||
f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
|
||||
f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
|
||||
f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
|
||||
f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")
|
||||
f.tusMaxSizeMB = cmdFiler.Flag.Int("tusMaxSizeMB", 5*1024, "maximum TUS upload size in MB")
|
||||
f.tusSessionExpiry = cmdFiler.Flag.Duration("tusSessionExpiry", 24*time.Hour, "incomplete TUS upload sessions are cleaned up after this duration, e.g. \"48h\", \"7h30m\"")
|
||||
f.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
|
||||
filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
|
||||
filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
||||
filerS3Options.allowedOrigins = cmdFiler.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
|
||||
filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
|
||||
filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
|
||||
filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
|
||||
filerS3Options.iamConfig = cmdFiler.Flag.String("s3.iam.config", "", "path to the advanced IAM config file")
|
||||
filerS3Options.auditLogConfig = cmdFiler.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
|
||||
filerS3Options.metricsHttpPort = cmdFiler.Flag.Int("s3.metricsPort", 0, "Prometheus metrics listen port")
|
||||
filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.")
|
||||
cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
|
||||
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
|
||||
filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
|
||||
filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
|
||||
filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
|
||||
filerS3Options.bindIp = cmdFiler.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
|
||||
filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
|
||||
filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited")
|
||||
filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
|
||||
filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")
|
||||
filerS3Options.cipher = cmdFiler.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
|
||||
filerS3Options.iamReadOnly = cmdFiler.Flag.Bool("s3.iam.readOnly", true, "disable IAM write operations on this server")
|
||||
filerS3Options.portIceberg = cmdFiler.Flag.Int("s3.port.iceberg", 8181, "Iceberg REST Catalog server listen port (0 to disable)")
|
||||
filerS3Options.portLance = cmdFiler.Flag.Int("s3.port.lance", 9101, "Lance Namespace server listen port (0 to disable)")
|
||||
filerS3Options.externalUrl = cmdFiler.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Advertised to Iceberg and Lance clients, and tried first when verifying S3 signatures behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
|
||||
filerS3Options.defaultFileMode = cmdFiler.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
|
||||
filerS3Options.cacheSizeMB = cmdFiler.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
|
||||
filerS3Options.allowUntrustedRemoteEndpoints = cmdFiler.Flag.Bool("s3.allowUntrustedRemoteEndpoints", false, allowUntrustedRemoteEndpointsUsage)
|
||||
|
||||
// start webdav on filer
|
||||
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
|
||||
filerWebDavOptions.port = cmdFiler.Flag.Int("webdav.port", 7333, "webdav server http listen port")
|
||||
filerWebDavOptions.collection = cmdFiler.Flag.String("webdav.collection", "", "collection to create the files")
|
||||
filerWebDavOptions.replication = cmdFiler.Flag.String("webdav.replication", "", "replication to create the files")
|
||||
filerWebDavOptions.disk = cmdFiler.Flag.String("webdav.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
filerWebDavOptions.tlsPrivateKey = cmdFiler.Flag.String("webdav.key.file", "", "path to the TLS private key file")
|
||||
filerWebDavOptions.tlsCertificate = cmdFiler.Flag.String("webdav.cert.file", "", "path to the TLS certificate file")
|
||||
filerWebDavOptions.cacheDir = cmdFiler.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
|
||||
filerWebDavOptions.cacheSizeMB = cmdFiler.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
|
||||
filerWebDavOptions.maxMB = cmdFiler.Flag.Int("webdav.maxMB", 4, "split files larger than the limit")
|
||||
filerWebDavOptions.filerRootPath = cmdFiler.Flag.String("webdav.filer.path", "/", "use this remote path from filer server")
|
||||
|
||||
// start iam on filer
|
||||
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
|
||||
filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
|
||||
filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
|
||||
|
||||
filerStartSftp = cmdFiler.Flag.Bool("sftp", false, "whether to start the SFTP server")
|
||||
filerSftpOptions.port = cmdFiler.Flag.Int("sftp.port", 2022, "SFTP server listen port")
|
||||
filerSftpOptions.sshPrivateKey = cmdFiler.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")
|
||||
filerSftpOptions.hostKeysFolder = cmdFiler.Flag.String("sftp.hostKeysFolder", "", "path to folder containing SSH private key files for host authentication")
|
||||
filerSftpOptions.authMethods = cmdFiler.Flag.String("sftp.authMethods", "password,publickey", "comma-separated list of allowed auth methods: password, publickey, certificate")
|
||||
filerSftpOptions.maxAuthTries = cmdFiler.Flag.Int("sftp.maxAuthTries", 6, "maximum number of authentication attempts per connection")
|
||||
filerSftpOptions.bannerMessage = cmdFiler.Flag.String("sftp.bannerMessage", "SeaweedFS SFTP Server - Unauthorized access is prohibited", "message displayed before authentication")
|
||||
filerSftpOptions.loginGraceTime = cmdFiler.Flag.Duration("sftp.loginGraceTime", 2*time.Minute, "timeout for authentication")
|
||||
filerSftpOptions.clientAliveInterval = cmdFiler.Flag.Duration("sftp.clientAliveInterval", 5*time.Second, "interval for sending keep-alive messages")
|
||||
filerSftpOptions.clientAliveCountMax = cmdFiler.Flag.Int("sftp.clientAliveCountMax", 3, "maximum number of missed keep-alive messages before disconnecting")
|
||||
filerSftpOptions.userStoreFile = cmdFiler.Flag.String("sftp.userStoreFile", "", "path to JSON file containing user credentials and permissions")
|
||||
filerSftpOptions.trustedUserCAKeysFile = cmdFiler.Flag.String("sftp.trustedUserCAKeysFile", "", "path to a file with trusted user CA public keys (OpenSSH authorized_keys format); required when 'certificate' is in -sftp.authMethods")
|
||||
filerSftpOptions.dataCenter = cmdFiler.Flag.String("sftp.dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
filerSftpOptions.bindIp = cmdFiler.Flag.String("sftp.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
|
||||
filerSftpOptions.localSocket = cmdFiler.Flag.String("sftp.localSocket", "", "default to /tmp/seaweedfs-sftp-<port>.sock")
|
||||
}
|
||||
|
||||
func filerLongDesc() string {
|
||||
desc := `start a file server which accepts REST operation for any files.
|
||||
|
||||
//create or overwrite the file, the directories /path/to will be automatically created
|
||||
POST /path/to/file
|
||||
//get the file content
|
||||
GET /path/to/file
|
||||
//create or overwrite the file, the filename in the multipart request will be used
|
||||
POST /path/to/
|
||||
//return a json format subdirectory and files listing
|
||||
GET /path/to/
|
||||
|
||||
The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", "/usr/local/etc/seaweedfs/", or "/etc/seaweedfs/", in that order.
|
||||
If the "filer.toml" is not found, an embedded filer store will be created under "-defaultStoreDir".
|
||||
|
||||
The example filer.toml configuration file can be generated by "weed scaffold -config=filer"
|
||||
|
||||
Supported Filer Stores:
|
||||
`
|
||||
|
||||
storeNames := make([]string, len(filer.Stores))
|
||||
for i, store := range filer.Stores {
|
||||
storeNames[i] = "\t" + store.GetName()
|
||||
}
|
||||
sort.Strings(storeNames)
|
||||
storeList := strings.Join(storeNames, "\n")
|
||||
return desc + storeList
|
||||
}
|
||||
|
||||
var cmdFiler = &Command{
|
||||
UsageLine: "filer -port=8888 -master=<ip:port>[,<ip:port>]*",
|
||||
Short: "start a file server that points to a master server, or a list of master servers",
|
||||
Long: filerLongDesc(),
|
||||
}
|
||||
|
||||
func runFiler(cmd *Command, args []string) bool {
|
||||
if *f.debug {
|
||||
go http.ListenAndServe(fmt.Sprintf(":%d", *f.debugPort), nil)
|
||||
}
|
||||
|
||||
*f.defaultLevelDbDirectory = util.ResolvePath(*f.defaultLevelDbDirectory)
|
||||
filerS3Options.resolvePaths()
|
||||
filerWebDavOptions.resolvePaths()
|
||||
filerSftpOptions.resolvePaths()
|
||||
util.LoadSecurityConfiguration()
|
||||
|
||||
// Share the S3 static identity config file with the filer regardless of
|
||||
// whether the embedded S3 gateway runs on this node: the IAM gRPC service
|
||||
// the admin UI and weed shell talk to is wired up unconditionally, and it
|
||||
// needs the same identities the S3 server would load from -s3.config.
|
||||
f.s3ConfigFile = filerS3Options.config
|
||||
|
||||
switch {
|
||||
case *f.metricsHttpIp != "":
|
||||
// noting to do, use f.metricsHttpIp
|
||||
case *f.bindIp != "":
|
||||
*f.metricsHttpIp = *f.bindIp
|
||||
case *f.ip != "":
|
||||
*f.metricsHttpIp = *f.ip
|
||||
}
|
||||
go stats_collect.StartMetricsServer(*f.metricsHttpIp, *f.metricsHttpPort)
|
||||
|
||||
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
filerS3Options.ip = f.ip
|
||||
if *filerS3Options.bindIp == "" {
|
||||
filerS3Options.bindIp = f.bindIp
|
||||
}
|
||||
filerS3Options.localFilerSocket = f.localSocket
|
||||
if *f.dataCenter != "" && *filerS3Options.dataCenter == "" {
|
||||
filerS3Options.dataCenter = f.dataCenter
|
||||
}
|
||||
// Set S3 metrics IP based on bind IP if not explicitly set
|
||||
if *filerS3Options.metricsHttpIp == "" {
|
||||
*filerS3Options.metricsHttpIp = *filerS3Options.bindIp
|
||||
}
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerS3Options.startS3Server()
|
||||
}(startDelay)
|
||||
startDelay++
|
||||
}
|
||||
|
||||
if *filerStartWebDav {
|
||||
filerWebDavOptions.filer = &filerAddress
|
||||
filerWebDavOptions.ipBind = f.bindIp
|
||||
|
||||
if *filerWebDavOptions.disk == "" {
|
||||
filerWebDavOptions.disk = f.diskType
|
||||
}
|
||||
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerWebDavOptions.startWebDav()
|
||||
}(startDelay)
|
||||
startDelay++
|
||||
}
|
||||
|
||||
if *filerStartIam {
|
||||
filerIamOptions.filer = &filerAddress
|
||||
filerIamOptions.masters = f.mastersString
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerIamOptions.startIamServer()
|
||||
}(startDelay)
|
||||
startDelay++
|
||||
}
|
||||
|
||||
if *filerStartSftp {
|
||||
filerSftpOptions.filer = &filerAddress
|
||||
if *filerSftpOptions.bindIp == "" {
|
||||
filerSftpOptions.bindIp = f.bindIp
|
||||
}
|
||||
if *f.dataCenter != "" && *filerSftpOptions.dataCenter == "" {
|
||||
filerSftpOptions.dataCenter = f.dataCenter
|
||||
}
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerSftpOptions.startSftpServer()
|
||||
}(startDelay)
|
||||
}
|
||||
|
||||
f.masters = pb.ServerAddresses(*f.mastersString).ToServiceDiscovery()
|
||||
|
||||
f.startFiler()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (fo *FilerOptions) startFiler() {
|
||||
|
||||
defaultMux := http.NewServeMux()
|
||||
publicVolumeMux := defaultMux
|
||||
|
||||
if *fo.publicPort != 0 {
|
||||
publicVolumeMux = http.NewServeMux()
|
||||
}
|
||||
if *fo.portGrpc == 0 {
|
||||
*fo.portGrpc = 10000 + *fo.port
|
||||
}
|
||||
if *fo.bindIp == "" {
|
||||
*fo.bindIp = *fo.ip
|
||||
}
|
||||
util.SetOutboundLocalIP(*fo.bindIp)
|
||||
if *fo.allowedOrigins == "" {
|
||||
*fo.allowedOrigins = "*"
|
||||
}
|
||||
|
||||
defaultLevelDbDirectory := *fo.defaultLevelDbDirectory + "/filerldb2"
|
||||
|
||||
filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)
|
||||
|
||||
// Initialize credential manager for IAM gRPC service
|
||||
var credentialManager *credential.CredentialManager
|
||||
var err error
|
||||
credentialManager, err = credential.NewCredentialManagerWithDefaults("")
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to initialize credential manager: %v", err)
|
||||
} else {
|
||||
glog.V(0).Infof("Initialized credential manager: %s", credentialManager.GetStoreName())
|
||||
}
|
||||
|
||||
// Load static S3 identities from config file if specified
|
||||
if fo.s3ConfigFile != nil && *fo.s3ConfigFile != "" {
|
||||
if credentialManager != nil {
|
||||
if err := credentialManager.LoadS3ConfigFile(*fo.s3ConfigFile); err != nil {
|
||||
glog.Warningf("Failed to load S3 config file for static identities: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
|
||||
Masters: fo.masters,
|
||||
FilerGroup: *fo.filerGroup,
|
||||
Collection: *fo.collection,
|
||||
DefaultReplication: *fo.defaultReplicaPlacement,
|
||||
DisableDirListing: *fo.disableDirListing,
|
||||
MaxMB: *fo.maxMB,
|
||||
DirListingLimit: *fo.dirListingLimit,
|
||||
DataCenter: *fo.dataCenter,
|
||||
Rack: *fo.rack,
|
||||
DefaultLevelDbDir: defaultLevelDbDirectory,
|
||||
DisableHttp: *fo.disableHttp,
|
||||
Host: filerAddress,
|
||||
Cipher: *fo.cipher,
|
||||
SaveToFilerLimit: int64(*fo.saveToFilerLimit),
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ConcurrentFileUploadLimit: int64(*fo.concurrentFileUploadLimit),
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
DiskType: *fo.diskType,
|
||||
AllowedOrigins: strings.Split(*fo.allowedOrigins, ","),
|
||||
ExposeDirectoryData: *fo.exposeDirectoryData,
|
||||
TusBasePath: *fo.tusBasePath,
|
||||
TusMaxSize: int64(*fo.tusMaxSizeMB) * 1024 * 1024,
|
||||
TusSessionExpiry: *fo.tusSessionExpiry,
|
||||
CredentialManager: credentialManager,
|
||||
|
||||
AllowUntrustedRemoteEndpoints: *fo.allowUntrustedRemoteEndpoints,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
}
|
||||
|
||||
// Serve "//" and ".." paths at their cleaned form instead of letting the mux
|
||||
// redirect: its Location is double-escaped, turning non-ASCII names into
|
||||
// percent-encoded directory names when a client follows it (#11125).
|
||||
defaultHandler := weed_server.CleanPathHandler(defaultMux)
|
||||
|
||||
// Ensure fs.Shutdown() runs exactly once, whether triggered by a signal hook
|
||||
// or by the main goroutine after Serve() returns (e.g., MiniCluster tests).
|
||||
var shutdownOnce sync.Once
|
||||
shutdownFiler := func() {
|
||||
shutdownOnce.Do(func() {
|
||||
fs.Shutdown()
|
||||
})
|
||||
}
|
||||
|
||||
if *fo.publicPort != 0 {
|
||||
publicListeningAddress := util.JoinHostPort(*fo.bindIp, *fo.publicPort)
|
||||
glog.V(0).Infoln("Start Seaweed filer server", version.Version(), "public at", publicListeningAddress)
|
||||
publicListener, localPublicListener, e := util.NewIpAndLocalListeners(*fo.bindIp, *fo.publicPort, 0)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer server public listener error on port %d:%v", *fo.publicPort, e)
|
||||
}
|
||||
publicHandler := weed_server.CleanPathHandler(publicVolumeMux)
|
||||
go func() {
|
||||
if e := http.Serve(publicListener, publicHandler); e != nil {
|
||||
glog.Fatalf("Volume server fail to serve public: %v", e)
|
||||
}
|
||||
}()
|
||||
if localPublicListener != nil {
|
||||
go func() {
|
||||
if e := http.Serve(localPublicListener, publicHandler); e != nil {
|
||||
glog.Errorf("Volume server fail to serve public: %v", e)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
glog.V(0).Infof("Start Seaweed Filer %s at %s:%d", version.Version(), *fo.ip, *fo.port)
|
||||
filerListener, filerLocalListener, e := util.NewIpAndLocalListeners(
|
||||
*fo.bindIp, *fo.port,
|
||||
time.Duration(10)*time.Second,
|
||||
)
|
||||
if e != nil {
|
||||
glog.Fatalf("Filer listener error: %v", e)
|
||||
}
|
||||
|
||||
// starting grpc server
|
||||
grpcPort := *fo.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*fo.bindIp, grpcPort, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.filer"))
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcS, fs)
|
||||
|
||||
// Register the IAM gRPC service. Auth is opt-in: when
|
||||
// jwt.filer_signing.key is configured the service requires a Bearer token
|
||||
// signed with that key; otherwise it runs unauthenticated, matching the
|
||||
// rest of the filer's gRPC surface. Operators who expose the filer gRPC
|
||||
// port beyond a trusted network should set jwt.filer_signing.key on both
|
||||
// the filer and the admin server.
|
||||
if credentialManager != nil {
|
||||
adminSigningKey := security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key"))
|
||||
iamGrpcServer := weed_server.NewIamGrpcServer(credentialManager, adminSigningKey)
|
||||
iam_pb.RegisterSeaweedIdentityAccessManagementServer(grpcS, iamGrpcServer)
|
||||
if len(adminSigningKey) == 0 {
|
||||
glog.V(0).Info("Registered IAM gRPC service on filer (unauthenticated; set jwt.filer_signing.key in security.toml to require admin Bearer token)")
|
||||
} else {
|
||||
glog.V(0).Info("Registered IAM gRPC service on filer (admin Bearer token required)")
|
||||
}
|
||||
}
|
||||
|
||||
reflection.Register(grpcS)
|
||||
if grpcLocalL != nil {
|
||||
go grpcS.Serve(grpcLocalL)
|
||||
}
|
||||
go grpcS.Serve(grpcL)
|
||||
pb.ServeGrpcOnLocalSocket(grpcS, grpcPort)
|
||||
|
||||
// Helper to gracefully stop the gRPC server, waiting for active RPCs.
|
||||
gracefulTimeout := fo.gracefulStopTimeout
|
||||
if gracefulTimeout <= 0 {
|
||||
gracefulTimeout = 10 * time.Second
|
||||
}
|
||||
stopGrpcServer := func() {
|
||||
glog.V(0).Infof("Gracefully stopping gRPC server")
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
grpcS.GracefulStop()
|
||||
close(stopped)
|
||||
}()
|
||||
select {
|
||||
case <-stopped:
|
||||
glog.V(0).Infof("gRPC server stopped gracefully")
|
||||
case <-time.After(gracefulTimeout):
|
||||
glog.V(0).Infof("gRPC server graceful stop timed out after %s, forcing stop", gracefulTimeout)
|
||||
grpcS.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
var socketServer *http.Server
|
||||
if runtime.GOOS != "windows" {
|
||||
localSocket := *fo.localSocket
|
||||
if localSocket == "" {
|
||||
localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port)
|
||||
}
|
||||
if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) {
|
||||
glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error())
|
||||
}
|
||||
filerSocketListener, err := net.Listen("unix", localSocket)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to listen on %s: %v", localSocket, err)
|
||||
}
|
||||
socketServer = newHttpServer(defaultHandler, nil)
|
||||
go socketServer.Serve(filerSocketListener)
|
||||
}
|
||||
|
||||
if viper.GetString("https.filer.key") != "" {
|
||||
certFile := viper.GetString("https.filer.cert")
|
||||
keyFile := viper.GetString("https.filer.key")
|
||||
caCertFile := viper.GetString("https.filer.ca")
|
||||
disbaleTlsVerifyClientCert := viper.GetBool("https.filer.disable_tls_verify_client_cert")
|
||||
|
||||
getCert, certProvider, err := security.NewReloadingServerCertificate(certFile, keyFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("Filer failed to load HTTPS certificate: %v", err)
|
||||
}
|
||||
defer certProvider.Close()
|
||||
|
||||
caCertPool := x509.NewCertPool()
|
||||
if caCertFile != "" {
|
||||
caCertFile, err := os.ReadFile(caCertFile)
|
||||
if err != nil {
|
||||
glog.Fatalf("error reading CA certificate: %v", err)
|
||||
}
|
||||
caCertPool.AppendCertsFromPEM(caCertFile)
|
||||
}
|
||||
|
||||
clientAuth := tls.NoClientCert
|
||||
if !disbaleTlsVerifyClientCert {
|
||||
clientAuth = tls.RequireAndVerifyClientCert
|
||||
}
|
||||
|
||||
tlsConfig := &tls.Config{
|
||||
GetCertificate: getCert,
|
||||
ClientAuth: clientAuth,
|
||||
ClientCAs: caCertPool,
|
||||
}
|
||||
|
||||
err = security.FixTlsConfig(util.GetViper(), tlsConfig)
|
||||
if err != nil {
|
||||
glog.Fatalf("Filer failed to fix TLS config: %v", err)
|
||||
}
|
||||
|
||||
var localTLSServer *http.Server
|
||||
if filerLocalListener != nil {
|
||||
localTLSServer = newHttpServer(defaultHandler, tlsConfig)
|
||||
go func() {
|
||||
if err := localTLSServer.ServeTLS(filerLocalListener, "", ""); err != nil {
|
||||
glog.Errorf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
httpS := newHttpServer(defaultHandler, tlsConfig)
|
||||
|
||||
// Register a single shutdown hook that runs the steps in the correct order:
|
||||
// stop accepting new gRPC/HTTP requests, then close the filer database.
|
||||
// Combining them into one hook keeps ordering intact regardless of how
|
||||
// grace fires interrupt hooks (FIFO vs LIFO).
|
||||
grace.OnInterrupt(func() {
|
||||
stopGrpcServer()
|
||||
glog.V(0).Infof("Gracefully stopping all HTTP servers")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if socketServer != nil {
|
||||
err = socketServer.Shutdown(shutdownCtx)
|
||||
if err != nil {
|
||||
glog.Warningf("socket server shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
if localTLSServer != nil {
|
||||
err = localTLSServer.Shutdown(shutdownCtx)
|
||||
if err != nil {
|
||||
glog.Warningf("local TLS server shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
if err := httpS.Shutdown(shutdownCtx); err != nil {
|
||||
glog.Warningf("HTTPS server shutdown: %v", err)
|
||||
}
|
||||
shutdownFiler()
|
||||
})
|
||||
|
||||
if fo.shutdownCtx != nil {
|
||||
go func() {
|
||||
<-fo.shutdownCtx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err := httpS.ServeTLS(filerListener, "", ""); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
// Close database after servers have stopped to prevent data corruption
|
||||
shutdownFiler()
|
||||
} else {
|
||||
var localHTTPServer *http.Server
|
||||
if filerLocalListener != nil {
|
||||
localHTTPServer = newHttpServer(defaultHandler, nil)
|
||||
go func() {
|
||||
if err := localHTTPServer.Serve(filerLocalListener); err != nil {
|
||||
glog.Errorf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
httpS := newHttpServer(defaultHandler, nil)
|
||||
|
||||
// Register a single shutdown hook that runs the steps in the correct order:
|
||||
// stop accepting new gRPC/HTTP requests, then close the filer database.
|
||||
// Combining them into one hook keeps ordering intact regardless of how
|
||||
// grace fires interrupt hooks (FIFO vs LIFO).
|
||||
grace.OnInterrupt(func() {
|
||||
stopGrpcServer()
|
||||
glog.V(0).Infof("Gracefully stopping all HTTP servers")
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if socketServer != nil {
|
||||
socketServer.Shutdown(shutdownCtx)
|
||||
}
|
||||
if localHTTPServer != nil {
|
||||
localHTTPServer.Shutdown(shutdownCtx)
|
||||
}
|
||||
if err := httpS.Shutdown(shutdownCtx); err != nil {
|
||||
glog.Warningf("HTTP server shutdown: %v", err)
|
||||
}
|
||||
shutdownFiler()
|
||||
})
|
||||
|
||||
if fo.shutdownCtx != nil {
|
||||
go func() {
|
||||
<-fo.shutdownCtx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err := httpS.Serve(filerListener); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
// Close database after servers have stopped to prevent data corruption
|
||||
shutdownFiler()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user