snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,781 @@
package weed_server
import (
"context"
"errors"
"net"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
s3remote "github.com/seaweedfs/seaweedfs/weed/remote_storage/s3"
)
// stubLookup returns a resolver func that maps the supplied hostnames to
// the supplied IP addresses, and errors for any host that is not in the map.
func stubLookup(t *testing.T, mapping map[string][]net.IP) func(ctx context.Context, host string) ([]net.IPAddr, error) {
t.Helper()
return func(_ context.Context, host string) ([]net.IPAddr, error) {
ips, ok := mapping[host]
if !ok {
return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true}
}
out := make([]net.IPAddr, 0, len(ips))
for _, ip := range ips {
out = append(out, net.IPAddr{IP: ip})
}
return out, nil
}
}
func TestValidateRemoteEndpoint(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
lookupIPAddrFunc = stubLookup(t, map[string][]net.IP{
"s3.us-east-1.amazonaws.com": {net.ParseIP("52.216.10.10")},
"internal.example.com": {net.ParseIP("127.0.0.1")},
"linklocal.example.com": {net.ParseIP("169.254.10.20")},
"private.example.com": {net.ParseIP("10.1.2.3")},
"private172.example.com": {net.ParseIP("172.20.0.5")},
"private192.example.com": {net.ParseIP("192.168.1.1")},
"cgnat.example.com": {net.ParseIP("100.64.0.42")},
})
cases := []struct {
name string
endpoint string
wantErr bool
wantSub string
}{
{
name: "empty",
endpoint: "",
wantErr: true,
wantSub: "empty",
},
{
name: "loopback literal",
endpoint: "http://127.0.0.1:8080",
wantErr: true,
wantSub: "loopback",
},
{
name: "ipv6 loopback",
endpoint: "http://[::1]:8080",
wantErr: true,
wantSub: "loopback",
},
{
name: "imds ipv4",
endpoint: "http://169.254.169.254/",
wantErr: true,
wantSub: "metadata",
},
{
name: "unspecified ipv4",
endpoint: "http://0.0.0.0/",
wantErr: true,
wantSub: "unspecified",
},
{
name: "link-local ipv6",
endpoint: "http://[fe80::1]/",
wantErr: true,
wantSub: "link-local",
},
{
name: "ftp scheme",
endpoint: "ftp://example.com/",
wantErr: true,
wantSub: "http or https",
},
{
name: "missing scheme",
endpoint: "example.com/",
wantErr: true,
wantSub: "http or https",
},
{
name: "imds hostname",
endpoint: "http://metadata.google.internal/",
wantErr: true,
wantSub: "metadata service",
},
{
name: "imds short hostname",
endpoint: "http://metadata/",
wantErr: true,
wantSub: "metadata service",
},
{
name: "host resolves to loopback",
endpoint: "https://internal.example.com/",
wantErr: true,
wantSub: "loopback",
},
{
name: "host resolves to link-local",
endpoint: "https://linklocal.example.com/",
wantErr: true,
wantSub: "link-local",
},
{
name: "rfc1918 10/8 literal",
endpoint: "http://10.0.0.1/",
wantErr: true,
wantSub: "private",
},
{
name: "rfc1918 172.16/12 literal",
endpoint: "http://172.16.5.5/",
wantErr: true,
wantSub: "private",
},
{
name: "rfc1918 192.168/16 literal",
endpoint: "http://192.168.0.1/",
wantErr: true,
wantSub: "private",
},
{
name: "cgnat literal",
endpoint: "http://100.64.0.1/",
wantErr: true,
wantSub: "CGNAT",
},
{
name: "host resolves to rfc1918 10/8",
endpoint: "https://private.example.com/",
wantErr: true,
wantSub: "private",
},
{
name: "host resolves to rfc1918 172/12",
endpoint: "https://private172.example.com/",
wantErr: true,
wantSub: "private",
},
{
name: "host resolves to rfc1918 192.168/16",
endpoint: "https://private192.example.com/",
wantErr: true,
wantSub: "private",
},
{
name: "host resolves to cgnat",
endpoint: "https://cgnat.example.com/",
wantErr: true,
wantSub: "CGNAT",
},
{
name: "nat64 imds",
endpoint: "http://[64:ff9b::a9fe:a9fe]/",
wantErr: true,
wantSub: "metadata",
},
{
name: "nat64 loopback",
endpoint: "http://[64:ff9b::7f00:1]/",
wantErr: true,
wantSub: "loopback",
},
{
name: "6to4 private",
endpoint: "http://[2002:a00:1::]/",
wantErr: true,
wantSub: "private",
},
{
name: "teredo loopback",
endpoint: "http://[2001:0:4136:e378:8000:63bf:80ff:fffe]/",
wantErr: true,
wantSub: "loopback",
},
{
name: "ipv4-compatible loopback",
endpoint: "http://[::7f00:1]/",
wantErr: true,
wantSub: "loopback",
},
{
name: "nat64 public passes",
endpoint: "http://[64:ff9b::808:808]/",
wantErr: false,
},
{
name: "6to4 public passes",
endpoint: "http://[2002:808:808::]/",
wantErr: false,
},
{
name: "teredo public passes",
endpoint: "http://[2001::f7f7:f7f7]/",
wantErr: false,
},
{
name: "ipv4-compatible public passes",
endpoint: "http://[::808:808]/",
wantErr: false,
},
{
name: "nat64 non-wellknown-prefix not decoded",
endpoint: "http://[64:ff9b:1::a9fe:a9fe]/",
wantErr: false,
},
{
name: "public s3",
endpoint: "https://s3.us-east-1.amazonaws.com/",
wantErr: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := validateRemoteEndpoint(context.Background(), tc.endpoint)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error for %q, got nil", tc.endpoint)
}
if tc.wantSub != "" && !strings.Contains(err.Error(), tc.wantSub) {
t.Fatalf("expected error to contain %q, got %v", tc.wantSub, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error for %q: %v", tc.endpoint, err)
}
})
}
}
func TestValidateRemoteEndpointResolverFailure(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
resolveErr := errors.New("simulated DNS failure")
lookupIPAddrFunc = func(_ context.Context, _ string) ([]net.IPAddr, error) {
return nil, resolveErr
}
err := validateRemoteEndpoint(context.Background(), "https://does-not-resolve.example.com/")
if err == nil {
t.Fatal("expected error when resolver fails")
}
if !strings.Contains(err.Error(), "resolve remote endpoint host") {
t.Fatalf("expected resolver error wrapping, got %v", err)
}
}
// TestGuardedDialerRebind simulates a DNS rebinding attack: the host first
// resolves to a public address (passing validateRemoteEndpoint) and then
// flips to 127.0.0.1 on the very next lookup (what the AWS SDK would do at
// dial time). The dial path must refuse the loopback answer instead of
// connecting to it.
func TestGuardedDialerRebind(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
const host = "rebind.example.com"
endpoint := "https://" + host + "/"
var calls atomic.Int32
lookupIPAddrFunc = func(_ context.Context, name string) ([]net.IPAddr, error) {
if name != host {
return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
}
if calls.Add(1) == 1 {
return []net.IPAddr{{IP: net.ParseIP("52.216.10.10")}}, nil
}
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}}, nil
}
if err := validateRemoteEndpoint(context.Background(), endpoint); err != nil {
t.Fatalf("first-pass validation should accept public IP, got %v", err)
}
dial := guardedDialer(endpoint)
conn, err := dial(context.Background(), "tcp", host+":443")
if conn != nil {
conn.Close()
t.Fatalf("guarded dialer must refuse loopback rebind, got conn")
}
if err == nil || !strings.Contains(err.Error(), "loopback") {
t.Fatalf("guarded dialer should fail with loopback error, got %v", err)
}
}
// TestRemoteEndpointGuardCoversS3CompatibleSiblings confirms the SSRF guard
// reaches every S3-SDK-backed provider, not just type "s3". It replays the two
// steps FetchAndWriteNeedle performs before building the client: resolve the
// endpoint the type would dial, then validate it against the deny-list. A
// sibling type pointed at an internal address must be rejected.
func TestRemoteEndpointGuardCoversS3CompatibleSiblings(t *testing.T) {
cases := []struct {
conf *remote_pb.RemoteConf
wantSub string
}{
{&remote_pb.RemoteConf{Type: "wasabi", WasabiEndpoint: "http://169.254.169.254/"}, "metadata"},
{&remote_pb.RemoteConf{Type: "b2", BackblazeEndpoint: "http://127.0.0.1/"}, "loopback"},
{&remote_pb.RemoteConf{Type: "aliyun", AliyunEndpoint: "http://192.168.0.1/"}, "private"},
{&remote_pb.RemoteConf{Type: "tencent", TencentEndpoint: "http://100.64.0.1/"}, "CGNAT"},
{&remote_pb.RemoteConf{Type: "baidu", BaiduEndpoint: "http://169.254.169.254/"}, "metadata"},
{&remote_pb.RemoteConf{Type: "filebase", FilebaseEndpoint: "http://172.16.0.1/"}, "private"},
{&remote_pb.RemoteConf{Type: "storj", StorjEndpoint: "http://10.0.0.5/"}, "private"},
{&remote_pb.RemoteConf{Type: "contabo", ContaboEndpoint: "http://[::1]/"}, "loopback"},
}
for _, tc := range cases {
endpoint, ok := s3remote.S3CompatibleEndpoint(tc.conf)
if !ok {
t.Errorf("type %q: not recognized as S3-compatible, guard would be skipped", tc.conf.Type)
continue
}
err := validateRemoteEndpoint(context.Background(), endpoint)
if err == nil {
t.Errorf("type %q: expected endpoint %q to be rejected", tc.conf.Type, endpoint)
continue
}
if !strings.Contains(err.Error(), tc.wantSub) {
t.Errorf("type %q: error %q missing %q", tc.conf.Type, err, tc.wantSub)
}
}
}
// TestRemoteEndpointGuardCoversAzure confirms the SSRF guard reaches the azure
// backend, which dials a caller-supplied AzureEndpoint. It replays the two
// steps FetchAndWriteNeedle performs before building the client: resolve the
// endpoint the type would dial via guardedRemoteClient, then validate it. An
// azure conf pointed at an internal address must be rejected.
func TestRemoteEndpointGuardCoversAzure(t *testing.T) {
cases := []struct {
name string
conf *remote_pb.RemoteConf
wantSub string
}{
{"imds", &remote_pb.RemoteConf{Type: "azure", AzureEndpoint: "https://169.254.169.254/"}, "metadata"},
{"loopback", &remote_pb.RemoteConf{Type: "azure", AzureEndpoint: "https://127.0.0.1/"}, "loopback"},
{"private", &remote_pb.RemoteConf{Type: "azure", AzureEndpoint: "https://10.0.0.5/"}, "private"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
endpoint, _, ok := guardedRemoteClient(tc.conf)
if !ok {
t.Fatalf("azure endpoint %q not guarded, the SSRF check would be skipped", tc.conf.AzureEndpoint)
}
err := validateRemoteEndpoint(context.Background(), endpoint)
if err == nil {
t.Fatalf("expected endpoint %q to be rejected", endpoint)
}
if !strings.Contains(err.Error(), tc.wantSub) {
t.Fatalf("error %q missing %q", err, tc.wantSub)
}
})
}
}
// TestValidateReplicaTarget covers the replica upload leg of
// FetchAndWriteNeedle. Replica targets are peer volume servers, so unlike the
// remote endpoint they may sit on a private network; the guard still rejects
// loopback / link-local / unspecified hosts and any target that is not a bare
// host:port, since a scheme, path or query would move the upload to a different
// URL through the format string.
func TestValidateReplicaTarget(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
lookupIPAddrFunc = stubLookup(t, map[string][]net.IP{
"peer.example.com": {net.ParseIP("10.0.0.7")},
"loop.example.com": {net.ParseIP("127.0.0.1")},
"linklocal.example.com": {net.ParseIP("169.254.169.254")},
})
cases := []struct {
name string
target string
wantErr bool
wantSub string
}{
// A path plus a trailing "?a=" would otherwise swallow ?type=replicate.
{"embedded path and query", "127.0.0.1:7000/status/x/?a=", true, "bare host:port"},
{"loopback literal", "127.0.0.1:8080", true, "loopback"},
{"ipv6 loopback", "[::1]:8080", true, "loopback"},
{"metadata literal", "169.254.169.254:80", true, "metadata"},
{"unspecified", "0.0.0.0:8080", true, "unspecified"},
{"metadata hostname", "metadata:80", true, "metadata"},
{"scheme rejected", "http://10.0.0.7:8080", true, "bare host:port"},
{"path rejected", "10.0.0.7:8080/x", true, "bare host:port"},
{"query rejected", "10.0.0.7:8080?a=b", true, "bare host:port"},
{"userinfo rejected", "user@10.0.0.7:8080", true, "bare host:port"},
{"missing port literal", "10.0.0.7", true, "bare host:port"},
{"missing port hostname", "peer.example.com", true, "bare host:port"},
{"empty", "", true, "empty"},
{"resolves to loopback", "loop.example.com:8080", true, "loopback"},
{"resolves to link-local", "linklocal.example.com:8080", true, "metadata"},
// Legitimate peer volume servers on private networks must pass.
{"private peer literal", "10.0.0.7:8080", false, ""},
{"private 192 peer", "192.168.1.5:8080", false, ""},
{"private peer hostname", "peer.example.com:8080", false, ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
err := validateReplicaTarget(context.Background(), tc.target)
if tc.wantErr {
if err == nil {
t.Fatalf("expected error for %q, got nil", tc.target)
}
if tc.wantSub != "" && !strings.Contains(err.Error(), tc.wantSub) {
t.Fatalf("expected error to contain %q, got %v", tc.wantSub, err)
}
return
}
if err != nil {
t.Fatalf("unexpected error for %q: %v", tc.target, err)
}
})
}
}
// TestGuardedRemoteClientSkipsFixedHostBackends confirms backends that only
// reach a fixed provider host bypass the endpoint guard: azure with no explicit
// endpoint (public cloud, host derived from the account) and unrelated types.
func TestGuardedRemoteClientSkipsFixedHostBackends(t *testing.T) {
for _, conf := range []*remote_pb.RemoteConf{
{Type: "azure", AzureAccountName: "acct"},
{Type: "gcs"},
nil,
} {
if _, _, ok := guardedRemoteClient(conf); ok {
t.Errorf("conf %+v should not be guarded", conf)
}
}
}
// TestGuardedRemoteClientAzureBuildsGuardedClient exercises the whole azure
// path: a public endpoint passes validation and the constructor builds a client
// through the guarded HTTP transport.
func TestGuardedRemoteClientAzureBuildsGuardedClient(t *testing.T) {
conf := &remote_pb.RemoteConf{
Type: "azure",
AzureAccountName: "testaccount",
AzureAccountKey: "aW52YWxpZGtleQ==",
AzureEndpoint: "https://testaccount.blob.core.usgovcloudapi.net/",
}
endpoint, makeClient, ok := guardedRemoteClient(conf)
if !ok {
t.Fatal("azure with an endpoint should be guarded")
}
client, err := makeClient(newGuardedHTTPClient(endpoint))
if err != nil {
t.Fatalf("build guarded azure client: %v", err)
}
if client == nil {
t.Fatal("expected a client")
}
}
// TestGcsCredentialsArePath confirms a caller-supplied gcs credentials value is
// only accepted as inline JSON. A filesystem path would otherwise be read from
// disk by the SDK when handling the request.
func TestGcsCredentialsArePath(t *testing.T) {
paths := []string{
"/etc/hostname",
"/etc/shadow",
"/nope/nothere",
"~/creds.json",
"relative/creds.json",
}
for _, p := range paths {
if !gcsCredentialsArePath(p) {
t.Errorf("expected %q to be treated as a path", p)
}
}
inlineOrEmpty := []string{
"",
`{"type":"service_account"}`,
`{}`,
}
for _, c := range inlineOrEmpty {
if gcsCredentialsArePath(c) {
t.Errorf("expected %q to be accepted (inline or empty)", c)
}
}
}
// TestGuardedRemoteClientGuardsGcsTokenURL confirms the token endpoint named by
// inline gcs credentials is the endpoint the guard validates, so a loopback
// token_uri is refused while the Google default passes.
func TestGuardedRemoteClientGuardsGcsTokenURL(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
lookupIPAddrFunc = stubLookup(t, map[string][]net.IP{
"oauth2.googleapis.com": {net.ParseIP("142.250.72.10")},
})
endpoint, makeClient, ok := guardedRemoteClient(&remote_pb.RemoteConf{
Type: "gcs",
GcsGoogleApplicationCredentials: `{"type":"service_account","token_uri":"http://127.0.0.1:9/token"}`,
})
if !ok {
t.Fatal("gcs conf with inline credentials should be guarded")
}
if endpoint != "http://127.0.0.1:9/token" {
t.Errorf("endpoint = %q, want the credential token_uri", endpoint)
}
if err := validateRemoteEndpoint(context.Background(), endpoint); err == nil {
t.Error("expected the loopback token endpoint to be rejected")
}
if makeClient == nil {
t.Error("expected a constructor")
}
endpoint, _, ok = guardedRemoteClient(&remote_pb.RemoteConf{
Type: "gcs",
GcsGoogleApplicationCredentials: `{"type":"service_account"}`,
})
if !ok {
t.Fatal("gcs conf with inline credentials should be guarded")
}
if err := validateRemoteEndpoint(context.Background(), endpoint); err != nil {
t.Errorf("default token endpoint %q should pass: %v", endpoint, err)
}
}
// TestCheckGcsCredentials confirms only inline credentials that carry their own
// key material are accepted. The federated types name a url, file or executable
// that the SDK reads the token from, none of which the endpoint guard sees.
func TestCheckGcsCredentials(t *testing.T) {
rejected := []string{
"/etc/hostname",
"~/creds.json",
`{`,
`{}`,
`{"type":"external_account","token_url":"http://127.0.0.1:9/v1/token","credential_source":{"url":"http://169.254.169.254/latest/meta-data/"}}`,
`{"type":"external_account","token_url":"http://127.0.0.1:9/v1/token","credential_source":{"file":"/etc/shadow"}}`,
`{"type":"external_account","credential_source":{"executable":{"command":"/bin/sh"}}}`,
`{"type":"external_account_authorized_user","token_url":"http://127.0.0.1:9/v1/token"}`,
`{"type":"impersonated_service_account","service_account_impersonation_url":"http://127.0.0.1:9/x"}`,
}
for _, creds := range rejected {
if err := checkGcsCredentials(creds); err == nil {
t.Errorf("expected %q to be rejected", creds)
}
}
accepted := []string{
"",
`{"type":"service_account","client_email":"a@b.com","private_key":"k"}`,
`{"type":"service_account","token_uri":"https://oauth2.googleapis.com/token"}`,
`{"type":"authorized_user","refresh_token":"r"}`,
}
for _, creds := range accepted {
if err := checkGcsCredentials(creds); err != nil {
t.Errorf("expected %q to be accepted, got %v", creds, err)
}
}
}
// TestGuardedDialerLiteralBlocked confirms that a literal blocked IP target
// is refused without any DNS lookup.
func TestGuardedDialerLiteralBlocked(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
lookupIPAddrFunc = func(_ context.Context, name string) ([]net.IPAddr, error) {
t.Fatalf("resolver should not be called for IP literal target, got lookup of %q", name)
return nil, nil
}
dial := guardedDialer("http://10.0.0.5:80")
conn, err := dial(context.Background(), "tcp", "10.0.0.5:80")
if conn != nil {
conn.Close()
t.Fatalf("guarded dialer must refuse rfc1918 literal, got conn")
}
if err == nil || !strings.Contains(err.Error(), "private") {
t.Fatalf("guarded dialer should fail with private-address error, got %v", err)
}
}
// TestGuardedReplicaDialerRebind confirms the replica upload's dial-time guard
// refuses a hostname that rebinds to loopback after validateReplicaTarget, yet
// keeps letting private peers through (allowPrivate).
func TestGuardedReplicaDialerRebind(t *testing.T) {
originalLookup := lookupIPAddrFunc
t.Cleanup(func() { lookupIPAddrFunc = originalLookup })
const host = "replica.example.com"
var calls atomic.Int32
lookupIPAddrFunc = func(_ context.Context, name string) ([]net.IPAddr, error) {
if name != host {
return nil, &net.DNSError{Err: "no such host", Name: name, IsNotFound: true}
}
if calls.Add(1) == 1 {
return []net.IPAddr{{IP: net.ParseIP("52.216.10.10")}}, nil
}
return []net.IPAddr{{IP: net.ParseIP("127.0.0.1")}}, nil
}
// Up-front validation sees the public answer and accepts the target.
if err := validateReplicaTarget(context.Background(), host+":8080"); err != nil {
t.Fatalf("public replica target should validate, got %v", err)
}
// The dial then re-resolves to loopback and must refuse it.
dial := guardedDialerPolicy(host+":8080", true)
conn, err := dial(context.Background(), "tcp", host+":8080")
if conn != nil {
conn.Close()
t.Fatalf("guarded replica dialer must refuse loopback rebind, got conn")
}
if err == nil || !strings.Contains(err.Error(), "loopback") {
t.Fatalf("expected loopback refusal, got %v", err)
}
// A private literal peer is allowed through: the dial is attempted (and here
// fails on the already-cancelled context) rather than blocked as private.
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, perr := guardedDialerPolicy("10.0.0.5:80", true)(ctx, "tcp", "10.0.0.5:80"); perr != nil && strings.Contains(perr.Error(), "private") {
t.Fatalf("private peer must be allowed by the replica dialer, got %v", perr)
}
}
// TestBuildGuardedRemoteStorageClient confirms the shared builder refuses a
// caller-influenced endpoint that resolves to a blocked address, and a gcs
// credentials path, while allowUntrusted falls back to the plain builder.
func TestBuildGuardedRemoteStorageClient(t *testing.T) {
loopbackS3 := &remote_pb.RemoteConf{
Name: "poc",
Type: "s3",
S3Endpoint: "http://127.0.0.1:8000",
S3AccessKey: "k",
S3SecretKey: "s",
S3Region: "us-east-1",
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), loopbackS3, false); err == nil {
t.Error("expected a loopback s3 endpoint to be rejected")
} else if !strings.Contains(err.Error(), "reject remote endpoint") {
t.Errorf("error = %v, want reject remote endpoint", err)
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), loopbackS3, true); err != nil {
t.Errorf("allowUntrusted should build the client: %v", err)
}
gcsPathCreds := &remote_pb.RemoteConf{
Name: "poc",
Type: "gcs",
GcsGoogleApplicationCredentials: "/etc/hostname",
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), gcsPathCreds, false); err == nil {
t.Error("expected a non-credentials file path to be rejected")
} else if !strings.Contains(err.Error(), "reject remote credentials") {
t.Errorf("error = %v, want reject remote credentials", err)
} else if strings.Contains(err.Error(), "/etc/hostname") {
t.Errorf("error must not leak the file path: %v", err)
}
// A file path that points to valid GCS credentials should be accepted.
credsFile := filepath.Join(t.TempDir(), "service-account.json")
validCreds := `{"type":"service_account","token_uri":"https://oauth2.googleapis.com/token","client_email":"sa@example.iam.gserviceaccount.com","private_key":"-----BEGIN PRIVATE KEY-----\nMIIBVwIBADANBgkqhkiG9w0BAQEFAASCAUEwggE9AgEAAkEAxY\n-----END PRIVATE KEY-----\n","private_key_id":"key1"}`
if err := os.WriteFile(credsFile, []byte(validCreds), 0600); err != nil {
t.Fatalf("write creds file: %v", err)
}
gcsFileCreds := &remote_pb.RemoteConf{
Name: "good",
Type: "gcs",
GcsGoogleApplicationCredentials: credsFile,
}
if err := checkGcsCredentials(credsFile); err != nil {
t.Errorf("valid gcs credentials file should pass: %v", err)
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), gcsFileCreds, false); err != nil {
t.Errorf("valid gcs credentials file should build: %v", err)
}
// A nonexistent path must be rejected without leaking the path in the error.
gcsMissingCreds := &remote_pb.RemoteConf{
Name: "missing",
Type: "gcs",
GcsGoogleApplicationCredentials: filepath.Join(t.TempDir(), "does-not-exist.json"),
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), gcsMissingCreds, false); err == nil {
t.Error("expected a nonexistent credentials file to be rejected")
} else if strings.Contains(err.Error(), "does-not-exist") {
t.Errorf("error must not leak the file path: %v", err)
}
}
// TestValidateRemoteConfForLoad confirms the load-time validator (injected into
// the filer's FilerRemoteStorage) rejects a RemoteConf whose endpoint resolves
// to a blocked address, while allowUntrusted skips the check. A conf whose type
// dials a fixed provider host (no caller-supplied endpoint) passes.
func TestValidateRemoteConfForLoad(t *testing.T) {
loopbackS3 := &remote_pb.RemoteConf{
Name: "poc",
Type: "s3",
S3Endpoint: "http://127.0.0.1:8000",
S3Region: "us-east-1",
}
if err := ValidateRemoteConfForLoad(context.Background(), loopbackS3, false); err == nil {
t.Error("expected a loopback s3 endpoint to be rejected at load")
} else if !strings.Contains(err.Error(), "reject remote endpoint") {
t.Errorf("error = %v, want reject remote endpoint", err)
}
// allowUntrusted mirrors the volume server opt-out.
if err := ValidateRemoteConfForLoad(context.Background(), loopbackS3, true); err != nil {
t.Errorf("allowUntrusted should accept the conf: %v", err)
}
// A non-S3-compatible type with no caller-supplied endpoint dials a fixed
// provider host, so there is nothing caller-influenced to deny.
fixedHost := &remote_pb.RemoteConf{Name: "fixed", Type: "gcs"}
if err := ValidateRemoteConfForLoad(context.Background(), fixedHost, false); err != nil {
t.Errorf("fixed-host provider should pass: %v", err)
}
// nil conf is a no-op.
if err := ValidateRemoteConfForLoad(context.Background(), nil, false); err != nil {
t.Errorf("nil conf should be a no-op: %v", err)
}
// A hostname endpoint is not resolved at load time (DNS is left to the
// build-time guard at dial), so it must pass even if it would resolve to a
// blocked address. This prevents transient DNS failures from disabling
// working mounts during /etc/remote reload.
hostnameS3 := &remote_pb.RemoteConf{
Name: "host",
Type: "s3",
S3Endpoint: "http://internal.example.com",
S3Region: "us-east-1",
}
if err := ValidateRemoteConfForLoad(context.Background(), hostnameS3, false); err != nil {
t.Errorf("hostname endpoint should pass at load (DNS deferred to dial): %v", err)
}
// A standard AWS S3 config with no custom endpoint (empty S3Endpoint) has
// no caller-supplied endpoint to guard — the AWS SDK derives the regional
// endpoint. Both the load-time validator and the build-time guard must
// accept it so standard AWS S3 mounts keep working.
standardS3 := &remote_pb.RemoteConf{
Name: "aws",
Type: "s3",
S3Region: "us-east-1",
}
if err := ValidateRemoteConfForLoad(context.Background(), standardS3, false); err != nil {
t.Errorf("standard AWS S3 (empty endpoint) should pass: %v", err)
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), standardS3, false); err != nil {
t.Errorf("standard AWS S3 (empty endpoint) should build: %v", err)
}
// A non-s3 S3-compatible type with an empty endpoint is a misconfiguration
// (the AWS SDK would derive an AWS endpoint). The guard must reject it
// rather than fall through to the unguarded cache.
aliyunNoEndpoint := &remote_pb.RemoteConf{
Name: "aliyun",
Type: "aliyun",
AliyunRegion: "cn-hangzhou",
}
if err := ValidateRemoteConfForLoad(context.Background(), aliyunNoEndpoint, false); err == nil {
t.Error("aliyun with empty endpoint should be rejected at load")
}
if _, err := BuildGuardedRemoteStorageClient(context.Background(), aliyunNoEndpoint, false); err == nil {
t.Error("aliyun with empty endpoint should be rejected by the guard")
}
}