snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled
This commit is contained in:
111
weed/remote_storage/azure/azure_credentials.go
Normal file
111
weed/remote_storage/azure/azure_credentials.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package azure
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
// Azure storage account names are 3 to 24 lowercase letters and digits. The
|
||||
// name lands in the service URL, where a stray "/", "?" or "@" would move the
|
||||
// authority somewhere else and send an authenticated request to that host.
|
||||
var validAzureAccountName = regexp.MustCompile(`^[a-z0-9]{3,24}$`)
|
||||
|
||||
// NewAzBlobClient builds a blob service client for accountName.
|
||||
//
|
||||
// An empty accountKey selects Entra ID instead of a shared key: azidentity
|
||||
// resolves a workload identity, a managed identity, or a developer login from
|
||||
// the environment, and access is granted through RBAC. Fleets that cannot
|
||||
// distribute and rotate storage account keys authenticate that way. clientID
|
||||
// pins a user-assigned identity when the environment offers more than one.
|
||||
//
|
||||
// endpoint is the blob service URL, for accounts outside the public cloud. An
|
||||
// empty endpoint derives the public one from accountName.
|
||||
//
|
||||
// A non-nil httpClient overrides the SDK transport; the volume server passes one
|
||||
// whose dialer pins the validated endpoint against DNS rebinding.
|
||||
func NewAzBlobClient(accountName, accountKey, clientID, endpoint string, httpClient *http.Client) (*azblob.Client, error) {
|
||||
|
||||
if accountName == "" {
|
||||
return nil, fmt.Errorf("azure account name is required")
|
||||
}
|
||||
if !validAzureAccountName.MatchString(accountName) {
|
||||
return nil, fmt.Errorf("invalid azure account name %q: expecting 3 to 24 lowercase letters and digits", accountName)
|
||||
}
|
||||
|
||||
serviceURL, err := azureServiceURL(accountName, endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
options := DefaultAzBlobClientOptions()
|
||||
if httpClient != nil {
|
||||
options.Transport = httpClient
|
||||
}
|
||||
|
||||
if accountKey == "" {
|
||||
credential, err := newAzureTokenCredential(clientID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure Entra ID credential for account %s: %w", accountName, err)
|
||||
}
|
||||
glog.V(1).Infof("azure %s: authenticating with Entra ID", accountName)
|
||||
client, err := azblob.NewClient(serviceURL, credential, options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure client: %w", err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure credential with account name:%s: %w", accountName, err)
|
||||
}
|
||||
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Azure client: %w", err)
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// azureServiceURL locates the blob service. Sovereign clouds and private
|
||||
// endpoints do not live under blob.core.windows.net, so they name the service
|
||||
// URL outright rather than having it derived from the account.
|
||||
func azureServiceURL(accountName, endpoint string) (string, error) {
|
||||
if endpoint == "" {
|
||||
return fmt.Sprintf("https://%s.blob.core.windows.net/", accountName), nil
|
||||
}
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid azure endpoint %q: %w", endpoint, err)
|
||||
}
|
||||
// plain http would carry the account key or the bearer token in the clear
|
||||
if parsed.Scheme != "https" || parsed.Hostname() == "" {
|
||||
return "", fmt.Errorf("invalid azure endpoint %q: expecting an https service url, such as https://%s.blob.core.usgovcloudapi.net/", endpoint, accountName)
|
||||
}
|
||||
return endpoint, nil
|
||||
}
|
||||
|
||||
// newAzureTokenCredential resolves an Entra ID credential. Without a pinned
|
||||
// clientID the default chain discovers whatever the host offers. With one, the
|
||||
// federated token file projected by the Azure workload identity webhook tells
|
||||
// the two identity flavors apart.
|
||||
func newAzureTokenCredential(clientID string) (azcore.TokenCredential, error) {
|
||||
if clientID == "" {
|
||||
return azidentity.NewDefaultAzureCredential(nil)
|
||||
}
|
||||
if os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" {
|
||||
return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
|
||||
ClientID: clientID,
|
||||
})
|
||||
}
|
||||
return azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
|
||||
ID: azidentity.ClientID(clientID),
|
||||
})
|
||||
}
|
||||
98
weed/remote_storage/azure/azure_credentials_test.go
Normal file
98
weed/remote_storage/azure/azure_credentials_test.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package azure
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewAzBlobClientRequiresAccountName(t *testing.T) {
|
||||
if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", "", "", nil); err == nil {
|
||||
t.Error("expected an error without an account name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAzBlobClientRejectsMalformedAccountName(t *testing.T) {
|
||||
for _, accountName := range []string{"ab", "TestAccount", "test-account", "evil.com/x", "evil@host.com", "account?x=1"} {
|
||||
if _, err := NewAzBlobClient(accountName, "", "", "", nil); err == nil {
|
||||
t.Errorf("expected an error for account name %q", accountName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAzBlobClientSharedKey(t *testing.T) {
|
||||
client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create a shared key client: %v", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Error("expected a client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAzBlobClientSharedKeyRejectsMalformedKey(t *testing.T) {
|
||||
if _, err := NewAzBlobClient("testaccount", "not base64", "", "", nil); err == nil {
|
||||
t.Error("expected an error with a malformed account key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureServiceURL(t *testing.T) {
|
||||
serviceURL, err := azureServiceURL("testaccount", "")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to derive the public service url: %v", err)
|
||||
}
|
||||
if serviceURL != "https://testaccount.blob.core.windows.net/" {
|
||||
t.Errorf("unexpected public service url %q", serviceURL)
|
||||
}
|
||||
|
||||
government := "https://testaccount.blob.core.usgovcloudapi.net/"
|
||||
serviceURL, err = azureServiceURL("testaccount", government)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to keep the configured service url: %v", err)
|
||||
}
|
||||
if serviceURL != government {
|
||||
t.Errorf("expected %q, got %q", government, serviceURL)
|
||||
}
|
||||
|
||||
for _, endpoint := range []string{"core.usgovcloudapi.net", "http://testaccount.blob.core.windows.net/", "https://", "https://:443/", "://x"} {
|
||||
if _, err := azureServiceURL("testaccount", endpoint); err == nil {
|
||||
t.Errorf("expected an error for endpoint %q", endpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// no account key: authenticate through the Entra ID chain instead
|
||||
func TestNewAzBlobClientEntraID(t *testing.T) {
|
||||
client, err := NewAzBlobClient("testaccount", "", "", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create an Entra ID client: %v", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Error("expected a client")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewAzBlobClientWorkloadIdentity(t *testing.T) {
|
||||
t.Setenv("AZURE_FEDERATED_TOKEN_FILE", filepath.Join(t.TempDir(), "token"))
|
||||
t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000")
|
||||
|
||||
client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111", "", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create a workload identity client: %v", err)
|
||||
}
|
||||
if client == nil {
|
||||
t.Error("expected a client")
|
||||
}
|
||||
}
|
||||
|
||||
// a pinned client id without a projected token file is a plain managed identity
|
||||
func TestNewAzureTokenCredentialManagedIdentity(t *testing.T) {
|
||||
t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "")
|
||||
|
||||
credential, err := newAzureTokenCredential("11111111-1111-1111-1111-111111111111")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create a managed identity credential: %v", err)
|
||||
}
|
||||
if credential == nil {
|
||||
t.Error("expected a credential")
|
||||
}
|
||||
}
|
||||
589
weed/remote_storage/azure/azure_storage_client.go
Normal file
589
weed/remote_storage/azure/azure_storage_client.go
Normal file
@@ -0,0 +1,589 @@
|
||||
package azure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultBlockSize = 4 * 1024 * 1024
|
||||
defaultConcurrency = 16
|
||||
defaultReadConcurrency = 16
|
||||
|
||||
// DefaultAzureOpTimeout is the timeout for short metadata operations
|
||||
// (GetProperties, Delete, conditional updates, etc.). Blob body transfers
|
||||
// (ReadFile / WriteFile) are NOT bounded by this timeout: they use
|
||||
// parallel 4 MiB block transfers and rely on the SDK's per-try TryTimeout
|
||||
// to bound each individual block.
|
||||
DefaultAzureOpTimeout = 60 * time.Second
|
||||
)
|
||||
|
||||
// DefaultAzBlobClientOptions returns the default Azure blob client options
|
||||
// with consistent retry configuration across the application.
|
||||
// This centralizes the retry policy to ensure uniform behavior between
|
||||
// remote storage and replication sink implementations.
|
||||
//
|
||||
// TryTimeout bounds a single HTTP attempt (headers + body) in the Azure SDK.
|
||||
// With parallel 4 MiB block transfers (DownloadBuffer / UploadStream) a 60s
|
||||
// TryTimeout is comfortable. A previous regression set this to 10s "to fail
|
||||
// faster on auth issues" — that silently broke large-blob reads: a single
|
||||
// non-parallelized Download of more than ~40 MiB over a typical WAN link
|
||||
// cannot finish in 10s, so re-caching a large remote object (e.g. a 2 GB
|
||||
// blob mounted via `remote.mount` and fetched through the S3 gateway) failed
|
||||
// every attempt with a generic context-deadline error.
|
||||
func DefaultAzBlobClientOptions() *azblob.ClientOptions {
|
||||
return &azblob.ClientOptions{
|
||||
ClientOptions: azcore.ClientOptions{
|
||||
Retry: policy.RetryOptions{
|
||||
MaxRetries: 3,
|
||||
TryTimeout: 60 * time.Second,
|
||||
RetryDelay: 1 * time.Second,
|
||||
MaxRetryDelay: 10 * time.Second,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// invalidMetadataChars matches any character that is not valid in Azure metadata keys.
|
||||
// Azure metadata keys must be valid C# identifiers: letters, digits, and underscores only.
|
||||
var invalidMetadataChars = regexp.MustCompile(`[^a-zA-Z0-9_]`)
|
||||
|
||||
// sanitizeMetadataKey converts an S3 metadata key to a valid Azure metadata key.
|
||||
// Azure metadata keys must be valid C# identifiers (letters, digits, underscores only, cannot start with digit).
|
||||
// To prevent collisions, invalid characters are replaced with their hex representation (_XX_).
|
||||
// Examples:
|
||||
// - "my-key" -> "my_2d_key"
|
||||
// - "my.key" -> "my_2e_key"
|
||||
// - "key@value" -> "key_40_value"
|
||||
func sanitizeMetadataKey(key string) string {
|
||||
// Replace each invalid character with _XX_ where XX is the hex code
|
||||
result := invalidMetadataChars.ReplaceAllStringFunc(key, func(s string) string {
|
||||
return fmt.Sprintf("_%02x_", s[0])
|
||||
})
|
||||
|
||||
// Azure metadata keys cannot start with a digit
|
||||
if len(result) > 0 && result[0] >= '0' && result[0] <= '9' {
|
||||
result = "_" + result
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func init() {
|
||||
remote_storage.RemoteStorageClientMakers["azure"] = new(azureRemoteStorageMaker)
|
||||
}
|
||||
|
||||
// resolveAzureAccount completes the configured account from the environment. A
|
||||
// configured client id asks for Entra ID, so a key left over in the environment
|
||||
// must not quietly take the request back to shared key auth.
|
||||
func resolveAzureAccount(conf *remote_pb.RemoteConf) (accountName, accountKey string) {
|
||||
accountName, accountKey = conf.AzureAccountName, conf.AzureAccountKey
|
||||
if len(accountName) == 0 {
|
||||
accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
|
||||
}
|
||||
if len(accountKey) == 0 && len(conf.AzureClientId) == 0 {
|
||||
accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
type azureRemoteStorageMaker struct{}
|
||||
|
||||
func (s azureRemoteStorageMaker) HasBucket() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
|
||||
return MakeWithHTTPClient(conf, nil)
|
||||
}
|
||||
|
||||
// MakeWithHTTPClient builds an azure client using the supplied *http.Client for
|
||||
// its transport (or the SDK default when nil). Callers that need to pin the dial
|
||||
// path against DNS rebinding pass a client whose transport has a guarded
|
||||
// DialContext, mirroring the S3 backend.
|
||||
func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
|
||||
|
||||
client := &azureRemoteStorageClient{
|
||||
conf: conf,
|
||||
}
|
||||
|
||||
accountName, accountKey := resolveAzureAccount(conf)
|
||||
if len(accountName) == 0 {
|
||||
return nil, fmt.Errorf("neither azure_account_name nor the AZURE_STORAGE_ACCOUNT environment variable is set")
|
||||
}
|
||||
|
||||
azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId, conf.AzureEndpoint, httpClient)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client.client = azClient
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
type azureRemoteStorageClient struct {
|
||||
conf *remote_pb.RemoteConf
|
||||
client *azblob.Client
|
||||
}
|
||||
|
||||
var _ = remote_storage.RemoteStorageClient(&azureRemoteStorageClient{})
|
||||
var _ = remote_storage.RemoteStorageConcurrentReader(&azureRemoteStorageClient{})
|
||||
|
||||
func (az *azureRemoteStorageClient) uploadConcurrency() int {
|
||||
if n := int(az.conf.GetUploadConcurrency()); n > 0 {
|
||||
return n
|
||||
}
|
||||
return defaultConcurrency
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) downloadConcurrency() int {
|
||||
if n := int(az.conf.GetDownloadConcurrency()); n > 0 {
|
||||
return n
|
||||
}
|
||||
return defaultReadConcurrency
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
|
||||
pathKey := loc.Path[1:]
|
||||
if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
|
||||
pathKey += "/"
|
||||
}
|
||||
|
||||
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
|
||||
pager := containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
|
||||
Prefix: &pathKey,
|
||||
})
|
||||
|
||||
for pager.More() {
|
||||
resp, pageErr := pager.NextPage(ctx)
|
||||
if pageErr != nil {
|
||||
return fmt.Errorf("azure list directory %s%s: %w", loc.Bucket, loc.Path, pageErr)
|
||||
}
|
||||
|
||||
for _, prefix := range resp.Segment.BlobPrefixes {
|
||||
if prefix.Name == nil {
|
||||
continue
|
||||
}
|
||||
dirKey := "/" + strings.TrimSuffix(*prefix.Name, "/")
|
||||
dir, name := util.FullPath(dirKey).DirAndName()
|
||||
if err = visitFn(dir, name, true, nil); err != nil {
|
||||
return fmt.Errorf("azure processing directory prefix %s: %w", *prefix.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, blobItem := range resp.Segment.BlobItems {
|
||||
if blobItem.Name == nil {
|
||||
continue
|
||||
}
|
||||
key := "/" + *blobItem.Name
|
||||
if strings.HasSuffix(key, "/") {
|
||||
continue // skip directory markers
|
||||
}
|
||||
dir, name := util.FullPath(key).DirAndName()
|
||||
|
||||
remoteEntry := &filer_pb.RemoteEntry{
|
||||
StorageName: az.conf.Name,
|
||||
}
|
||||
if blobItem.Properties != nil {
|
||||
if blobItem.Properties.LastModified != nil {
|
||||
remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
|
||||
}
|
||||
if blobItem.Properties.ContentLength != nil {
|
||||
remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
|
||||
}
|
||||
if blobItem.Properties.ETag != nil {
|
||||
remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
|
||||
}
|
||||
remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
|
||||
}
|
||||
|
||||
if err = visitFn(dir, name, false, remoteEntry); err != nil {
|
||||
return fmt.Errorf("azure processing blob %s: %w", *blobItem.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
|
||||
key := loc.Path[1:]
|
||||
ctx, cancel := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
|
||||
defer cancel()
|
||||
resp, err := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key).GetProperties(ctx, nil)
|
||||
if err != nil {
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("stat azure %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
remoteEntry = &filer_pb.RemoteEntry{
|
||||
StorageName: az.conf.Name,
|
||||
}
|
||||
if resp.ContentLength != nil {
|
||||
remoteEntry.RemoteSize = *resp.ContentLength
|
||||
}
|
||||
if resp.LastModified != nil {
|
||||
remoteEntry.RemoteMtime = resp.LastModified.Unix()
|
||||
}
|
||||
if resp.ETag != nil {
|
||||
remoteEntry.RemoteETag = string(*resp.ETag)
|
||||
}
|
||||
remoteEntry.RemoteContentEncoding = remoteContentEncoding(resp.ContentEncoding)
|
||||
return remoteEntry, nil
|
||||
}
|
||||
|
||||
// blob properties report no Content-Encoding as nil; the remote entry records
|
||||
// that authoritatively as empty
|
||||
func remoteContentEncoding(encoding *string) *string {
|
||||
if encoding == nil {
|
||||
empty := ""
|
||||
return &empty
|
||||
}
|
||||
return encoding
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
|
||||
|
||||
pathKey := loc.Path[1:]
|
||||
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
|
||||
|
||||
// List blobs with pager
|
||||
pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
|
||||
Prefix: &pathKey,
|
||||
})
|
||||
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(context.Background())
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure traverse %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
|
||||
for _, blobItem := range resp.Segment.BlobItems {
|
||||
if blobItem.Name == nil {
|
||||
continue
|
||||
}
|
||||
key := "/" + *blobItem.Name
|
||||
dir, name := util.FullPath(key).DirAndName()
|
||||
|
||||
remoteEntry := &filer_pb.RemoteEntry{
|
||||
StorageName: az.conf.Name,
|
||||
}
|
||||
if blobItem.Properties != nil {
|
||||
if blobItem.Properties.LastModified != nil {
|
||||
remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
|
||||
}
|
||||
if blobItem.Properties.ContentLength != nil {
|
||||
remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
|
||||
}
|
||||
if blobItem.Properties.ETag != nil {
|
||||
remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
|
||||
}
|
||||
remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
|
||||
}
|
||||
|
||||
err = visitFn(dir, name, false, remoteEntry)
|
||||
if err != nil {
|
||||
return fmt.Errorf("azure processing %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
|
||||
return az.ReadFileWithConcurrency(loc, offset, size, 0)
|
||||
}
|
||||
|
||||
// ReadFileWithConcurrency fetches a byte range of a blob using the Azure SDK's
|
||||
// parallel block downloader. The blob is split into `defaultBlockSize` (4 MiB)
|
||||
// blocks that are fetched in parallel up to `concurrency` at a time. This
|
||||
// keeps each individual HTTP GET small enough to complete well within the
|
||||
// SDK's per-try TryTimeout, so large-blob reads remain reliable even on
|
||||
// slower WAN links.
|
||||
func (az *azureRemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error) {
|
||||
|
||||
if offset < 0 {
|
||||
return nil, fmt.Errorf("invalid offset %d for %s%s", offset, loc.Bucket, loc.Path)
|
||||
}
|
||||
if size < 0 {
|
||||
return nil, fmt.Errorf("invalid size %d for %s%s", size, loc.Bucket, loc.Path)
|
||||
}
|
||||
|
||||
if concurrency <= 0 {
|
||||
concurrency = az.downloadConcurrency()
|
||||
}
|
||||
if concurrency > math.MaxUint16 {
|
||||
concurrency = math.MaxUint16
|
||||
}
|
||||
|
||||
key := loc.Path[1:]
|
||||
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
|
||||
|
||||
// DownloadBuffer requires a pre-sized destination. If the caller asked for
|
||||
// "read to end" (size == 0), discover the blob length via GetProperties
|
||||
// first so we can allocate accurately.
|
||||
if size == 0 {
|
||||
propsCtx, cancelProps := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
|
||||
props, propsErr := blobClient.GetProperties(propsCtx, nil)
|
||||
cancelProps()
|
||||
if propsErr != nil {
|
||||
return nil, fmt.Errorf("get properties %s%s: %w", loc.Bucket, loc.Path, propsErr)
|
||||
}
|
||||
if props.ContentLength == nil {
|
||||
return nil, fmt.Errorf("azure %s%s: missing ContentLength", loc.Bucket, loc.Path)
|
||||
}
|
||||
size = *props.ContentLength - offset
|
||||
if size <= 0 {
|
||||
return []byte{}, nil
|
||||
}
|
||||
}
|
||||
|
||||
data = make([]byte, size)
|
||||
n, err := blobClient.DownloadBuffer(context.Background(), data, &blob.DownloadBufferOptions{
|
||||
Range: blob.HTTPRange{
|
||||
Offset: offset,
|
||||
Count: size,
|
||||
},
|
||||
BlockSize: defaultBlockSize,
|
||||
Concurrency: uint16(concurrency),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to download file %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
// Pre-sized buffer: a short read stays zero-padded. Reject it rather than
|
||||
// cache corrupt content.
|
||||
if n != size {
|
||||
return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
|
||||
key := loc.Path[1:]
|
||||
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
|
||||
|
||||
downloadResponse, err := blobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
|
||||
Range: blob.HTTPRange{
|
||||
Offset: offset,
|
||||
Count: size,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("failed to open stream for %s%s: %v", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
return downloadResponse.Body, nil
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
|
||||
// the trailing slash keeps sibling prefixes that share the name intact
|
||||
prefix := loc.Path[1:]
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
if prefix == "" {
|
||||
// the mount root maps to the whole container; wiping every blob from a
|
||||
// single namespace event is too destructive, so keep them
|
||||
glog.Warningf("azure %s: skip removing directory mapped to the container root", loc.Bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
|
||||
pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
for pager.More() {
|
||||
resp, pageErr := pager.NextPage(context.Background())
|
||||
if pageErr != nil {
|
||||
return fmt.Errorf("azure list %s/%s: %w", loc.Bucket, prefix, pageErr)
|
||||
}
|
||||
for _, blobItem := range resp.Segment.BlobItems {
|
||||
if blobItem.Name == nil {
|
||||
continue
|
||||
}
|
||||
_, delErr := containerClient.NewBlobClient(*blobItem.Name).Delete(context.Background(), &blob.DeleteOptions{
|
||||
DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
|
||||
})
|
||||
if delErr != nil && !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
|
||||
return fmt.Errorf("azure delete %s/%s: %w", loc.Bucket, *blobItem.Name, delErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
|
||||
|
||||
key := loc.Path[1:]
|
||||
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
|
||||
|
||||
// Upload from reader
|
||||
metadata := toMetadata(entry.Extended)
|
||||
httpHeaders := &blob.HTTPHeaders{}
|
||||
if entry.Attributes != nil && entry.Attributes.Mime != "" {
|
||||
httpHeaders.BlobContentType = &entry.Attributes.Mime
|
||||
}
|
||||
if contentEncoding := remote_storage.EntryContentEncoding(entry); contentEncoding != "" {
|
||||
httpHeaders.BlobContentEncoding = &contentEncoding
|
||||
}
|
||||
|
||||
_, err = blobClient.UploadStream(context.Background(), reader, &blockblob.UploadStreamOptions{
|
||||
BlockSize: defaultBlockSize,
|
||||
Concurrency: az.uploadConcurrency(),
|
||||
HTTPHeaders: httpHeaders,
|
||||
Metadata: metadata,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("azure upload to %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
|
||||
// read back the remote entry
|
||||
return az.readFileRemoteEntry(loc)
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
|
||||
return az.StatFile(loc)
|
||||
}
|
||||
|
||||
func toMetadata(attributes map[string][]byte) map[string]*string {
|
||||
metadata := make(map[string]*string)
|
||||
for k, v := range attributes {
|
||||
if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) {
|
||||
// S3 stores metadata keys in lowercase; normalize for consistency.
|
||||
key := strings.ToLower(k[len(s3_constants.AmzUserMetaPrefix):])
|
||||
|
||||
// Sanitize key to prevent collisions and ensure Azure compliance
|
||||
key = sanitizeMetadataKey(key)
|
||||
|
||||
val := string(v)
|
||||
metadata[key] = &val
|
||||
}
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
|
||||
if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
|
||||
return nil
|
||||
}
|
||||
metadata := toMetadata(newEntry.Extended)
|
||||
|
||||
key := loc.Path[1:]
|
||||
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
|
||||
|
||||
if !reflect.DeepEqual(toMetadata(oldEntry.Extended), metadata) {
|
||||
if _, err = blobClient.SetMetadata(context.Background(), metadata, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
|
||||
// SetHTTPHeaders replaces the whole header set, so carry the rest over
|
||||
props, getErr := blobClient.GetProperties(context.Background(), nil)
|
||||
if getErr != nil {
|
||||
return fmt.Errorf("azure get properties %s%s: %w", loc.Bucket, loc.Path, getErr)
|
||||
}
|
||||
httpHeaders := blob.ParseHTTPHeaders(props)
|
||||
httpHeaders.BlobContentEncoding = nil
|
||||
if encoding != "" {
|
||||
httpHeaders.BlobContentEncoding = &encoding
|
||||
}
|
||||
if _, err = blobClient.SetHTTPHeaders(context.Background(), httpHeaders, nil); err != nil {
|
||||
return fmt.Errorf("azure set http headers %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
|
||||
key := loc.Path[1:]
|
||||
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
|
||||
|
||||
_, err = blobClient.Delete(context.Background(), &blob.DeleteOptions{
|
||||
DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
|
||||
})
|
||||
if err != nil {
|
||||
// Make delete idempotent - don't return error if blob doesn't exist
|
||||
if bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("azure delete %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
|
||||
pager := az.client.NewListContainersPager(nil)
|
||||
|
||||
for pager.More() {
|
||||
resp, err := pager.NextPage(context.Background())
|
||||
if err != nil {
|
||||
return buckets, err
|
||||
}
|
||||
|
||||
for _, containerItem := range resp.ContainerItems {
|
||||
if containerItem.Name != nil {
|
||||
bucket := &remote_storage.Bucket{
|
||||
Name: *containerItem.Name,
|
||||
}
|
||||
if containerItem.Properties != nil && containerItem.Properties.LastModified != nil {
|
||||
bucket.CreatedAt = *containerItem.Properties.LastModified
|
||||
}
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) {
|
||||
containerClient := az.client.ServiceClient().NewContainerClient(name)
|
||||
_, err = containerClient.Create(context.Background(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create bucket %s: %w", name, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) {
|
||||
containerClient := az.client.ServiceClient().NewContainerClient(name)
|
||||
_, err = containerClient.Delete(context.Background(), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete bucket %s: %w", name, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
432
weed/remote_storage/azure/azure_storage_client_test.go
Normal file
432
weed/remote_storage/azure/azure_storage_client_test.go
Normal file
@@ -0,0 +1,432 @@
|
||||
package azure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestAzureStorageClientBasic tests basic Azure storage client operations
|
||||
func TestAzureStorageClientBasic(t *testing.T) {
|
||||
// Skip if credentials not available
|
||||
accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
|
||||
accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
|
||||
testContainer := os.Getenv("AZURE_TEST_CONTAINER")
|
||||
|
||||
if accountName == "" || accountKey == "" {
|
||||
t.Skip("Skipping Azure storage test: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
|
||||
}
|
||||
if testContainer == "" {
|
||||
testContainer = "seaweedfs-test"
|
||||
}
|
||||
|
||||
// Create client
|
||||
maker := azureRemoteStorageMaker{}
|
||||
conf := &remote_pb.RemoteConf{
|
||||
Name: "test-azure",
|
||||
AzureAccountName: accountName,
|
||||
AzureAccountKey: accountKey,
|
||||
}
|
||||
|
||||
client, err := maker.Make(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create Azure client: %v", err)
|
||||
}
|
||||
|
||||
azClient := client.(*azureRemoteStorageClient)
|
||||
|
||||
// Test 1: Create bucket/container
|
||||
t.Run("CreateBucket", func(t *testing.T) {
|
||||
err := azClient.CreateBucket(testContainer)
|
||||
// Ignore error if bucket already exists
|
||||
if err != nil && !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
|
||||
t.Fatalf("Failed to create bucket: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 2: List buckets
|
||||
t.Run("ListBuckets", func(t *testing.T) {
|
||||
buckets, err := azClient.ListBuckets()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to list buckets: %v", err)
|
||||
}
|
||||
if len(buckets) == 0 {
|
||||
t.Log("No buckets found (might be expected)")
|
||||
} else {
|
||||
t.Logf("Found %d buckets", len(buckets))
|
||||
}
|
||||
})
|
||||
|
||||
// Test 3: Write file
|
||||
testContent := []byte("Hello from SeaweedFS Azure SDK migration test!")
|
||||
testKey := fmt.Sprintf("/test-file-%d.txt", time.Now().Unix())
|
||||
loc := &remote_pb.RemoteStorageLocation{
|
||||
Name: "test-azure",
|
||||
Bucket: testContainer,
|
||||
Path: testKey,
|
||||
}
|
||||
|
||||
t.Run("WriteFile", func(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: time.Now().Unix(),
|
||||
Mime: "text/plain",
|
||||
},
|
||||
Extended: map[string][]byte{
|
||||
"x-amz-meta-test-key": []byte("test-value"),
|
||||
},
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(testContent)
|
||||
remoteEntry, err := azClient.WriteFile(loc, entry, reader)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to write file: %v", err)
|
||||
}
|
||||
if remoteEntry == nil {
|
||||
t.Fatal("Remote entry is nil")
|
||||
}
|
||||
if remoteEntry.RemoteSize != int64(len(testContent)) {
|
||||
t.Errorf("Expected size %d, got %d", len(testContent), remoteEntry.RemoteSize)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 4: Read file
|
||||
t.Run("ReadFile", func(t *testing.T) {
|
||||
data, err := azClient.ReadFile(loc, 0, int64(len(testContent)))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data, testContent) {
|
||||
t.Errorf("Content mismatch. Expected: %s, Got: %s", testContent, data)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 5: Read partial file
|
||||
t.Run("ReadPartialFile", func(t *testing.T) {
|
||||
data, err := azClient.ReadFile(loc, 0, 5)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read partial file: %v", err)
|
||||
}
|
||||
expected := testContent[:5]
|
||||
if !bytes.Equal(data, expected) {
|
||||
t.Errorf("Content mismatch. Expected: %s, Got: %s", expected, data)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 6: Update metadata
|
||||
t.Run("UpdateMetadata", func(t *testing.T) {
|
||||
oldEntry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
"x-amz-meta-test-key": []byte("test-value"),
|
||||
},
|
||||
}
|
||||
newEntry := &filer_pb.Entry{
|
||||
Extended: map[string][]byte{
|
||||
"x-amz-meta-test-key": []byte("test-value"),
|
||||
"x-amz-meta-new-key": []byte("new-value"),
|
||||
},
|
||||
}
|
||||
err := azClient.UpdateFileMetadata(loc, oldEntry, newEntry)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update metadata: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 7: Traverse (list objects)
|
||||
t.Run("Traverse", func(t *testing.T) {
|
||||
foundFile := false
|
||||
err := azClient.Traverse(loc, func(dir string, name string, isDir bool, remoteEntry *filer_pb.RemoteEntry) error {
|
||||
if !isDir && name == testKey[1:] { // Remove leading slash
|
||||
foundFile = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to traverse: %v", err)
|
||||
}
|
||||
if !foundFile {
|
||||
t.Log("Test file not found in traverse (might be expected due to path matching)")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 8: Delete file
|
||||
t.Run("DeleteFile", func(t *testing.T) {
|
||||
err := azClient.DeleteFile(loc)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to delete file: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Test 9: Verify file deleted (should fail)
|
||||
t.Run("VerifyDeleted", func(t *testing.T) {
|
||||
_, err := azClient.ReadFile(loc, 0, 10)
|
||||
if !bloberror.HasCode(err, bloberror.BlobNotFound) {
|
||||
t.Errorf("Expected BlobNotFound error, but got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Clean up: Try to delete the test container
|
||||
// Comment out if you want to keep the container
|
||||
/*
|
||||
t.Run("DeleteBucket", func(t *testing.T) {
|
||||
err := azClient.DeleteBucket(testContainer)
|
||||
if err != nil {
|
||||
t.Logf("Warning: Failed to delete bucket: %v", err)
|
||||
}
|
||||
})
|
||||
*/
|
||||
}
|
||||
|
||||
// TestToMetadata tests the metadata conversion function
|
||||
func TestToMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input map[string][]byte
|
||||
expected map[string]*string
|
||||
}{
|
||||
{
|
||||
name: "basic metadata",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "key1": []byte("value1"),
|
||||
s3_constants.AmzUserMetaPrefix + "key2": []byte("value2"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"key1": stringPtr("value1"),
|
||||
"key2": stringPtr("value2"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "metadata with dashes",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "content-type": []byte("text/plain"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"content_2d_type": stringPtr("text/plain"), // dash (0x2d) -> _2d_
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "non-metadata keys ignored",
|
||||
input: map[string][]byte{
|
||||
"some-other-key": []byte("ignored"),
|
||||
s3_constants.AmzUserMetaPrefix + "included": []byte("included"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"included": stringPtr("included"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keys starting with digits",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "123key": []byte("value1"),
|
||||
s3_constants.AmzUserMetaPrefix + "456-test": []byte("value2"),
|
||||
s3_constants.AmzUserMetaPrefix + "789": []byte("value3"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"_123key": stringPtr("value1"), // starts with digit -> prefix _
|
||||
"_456_2d_test": stringPtr("value2"), // starts with digit AND has dash
|
||||
"_789": stringPtr("value3"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "uppercase and mixed case keys",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "My-Key": []byte("value1"),
|
||||
s3_constants.AmzUserMetaPrefix + "UPPERCASE": []byte("value2"),
|
||||
s3_constants.AmzUserMetaPrefix + "MiXeD-CaSe": []byte("value3"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"my_2d_key": stringPtr("value1"), // lowercase + dash -> _2d_
|
||||
"uppercase": stringPtr("value2"),
|
||||
"mixed_2d_case": stringPtr("value3"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "keys with invalid characters",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "my.key": []byte("value1"),
|
||||
s3_constants.AmzUserMetaPrefix + "key+plus": []byte("value2"),
|
||||
s3_constants.AmzUserMetaPrefix + "key@symbol": []byte("value3"),
|
||||
s3_constants.AmzUserMetaPrefix + "key-with.": []byte("value4"),
|
||||
s3_constants.AmzUserMetaPrefix + "key/slash": []byte("value5"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"my_2e_key": stringPtr("value1"), // dot (0x2e) -> _2e_
|
||||
"key_2b_plus": stringPtr("value2"), // plus (0x2b) -> _2b_
|
||||
"key_40_symbol": stringPtr("value3"), // @ (0x40) -> _40_
|
||||
"key_2d_with_2e_": stringPtr("value4"), // dash and dot
|
||||
"key_2f_slash": stringPtr("value5"), // slash (0x2f) -> _2f_
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "collision prevention",
|
||||
input: map[string][]byte{
|
||||
s3_constants.AmzUserMetaPrefix + "my-key": []byte("value1"),
|
||||
s3_constants.AmzUserMetaPrefix + "my.key": []byte("value2"),
|
||||
s3_constants.AmzUserMetaPrefix + "my_key": []byte("value3"),
|
||||
},
|
||||
expected: map[string]*string{
|
||||
"my_2d_key": stringPtr("value1"), // dash (0x2d)
|
||||
"my_2e_key": stringPtr("value2"), // dot (0x2e)
|
||||
"my_key": stringPtr("value3"), // underscore is valid, no encoding
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
input: map[string][]byte{},
|
||||
expected: map[string]*string{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := toMetadata(tt.input)
|
||||
if len(result) != len(tt.expected) {
|
||||
t.Errorf("Expected %d keys, got %d", len(tt.expected), len(result))
|
||||
}
|
||||
for key, expectedVal := range tt.expected {
|
||||
if resultVal, ok := result[key]; !ok {
|
||||
t.Errorf("Expected key %s not found", key)
|
||||
} else if resultVal == nil || expectedVal == nil {
|
||||
if resultVal != expectedVal {
|
||||
t.Errorf("For key %s: expected %v, got %v", key, expectedVal, resultVal)
|
||||
}
|
||||
} else if *resultVal != *expectedVal {
|
||||
t.Errorf("For key %s: expected %s, got %s", key, *expectedVal, *resultVal)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
return bytes.Contains([]byte(s), []byte(substr))
|
||||
}
|
||||
|
||||
func stringPtr(s string) *string {
|
||||
return &s
|
||||
}
|
||||
|
||||
// Benchmark tests
|
||||
func BenchmarkToMetadata(b *testing.B) {
|
||||
input := map[string][]byte{
|
||||
"x-amz-meta-key1": []byte("value1"),
|
||||
"x-amz-meta-key2": []byte("value2"),
|
||||
"x-amz-meta-content-type": []byte("text/plain"),
|
||||
"other-key": []byte("ignored"),
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
toMetadata(input)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that the maker implements the interface
|
||||
func TestAzureRemoteStorageMaker(t *testing.T) {
|
||||
maker := azureRemoteStorageMaker{}
|
||||
|
||||
if !maker.HasBucket() {
|
||||
t.Error("Expected HasBucket() to return true")
|
||||
}
|
||||
|
||||
// Test with missing credentials - unset env vars (auto-restored by t.Setenv)
|
||||
t.Setenv("AZURE_STORAGE_ACCOUNT", "")
|
||||
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
|
||||
|
||||
conf := &remote_pb.RemoteConf{
|
||||
Name: "test",
|
||||
}
|
||||
_, err := maker.Make(conf)
|
||||
if err == nil {
|
||||
t.Error("Expected error with missing credentials")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAzureAccount(t *testing.T) {
|
||||
t.Setenv("AZURE_STORAGE_ACCOUNT", "envaccount")
|
||||
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "ZW52a2V5")
|
||||
|
||||
accountName, accountKey := resolveAzureAccount(&remote_pb.RemoteConf{})
|
||||
if accountName != "envaccount" || accountKey != "ZW52a2V5" {
|
||||
t.Errorf("expected the environment account, got %q/%q", accountName, accountKey)
|
||||
}
|
||||
|
||||
// a configured identity keeps a leftover environment key out of the way
|
||||
accountName, accountKey = resolveAzureAccount(&remote_pb.RemoteConf{
|
||||
AzureAccountName: "testaccount",
|
||||
AzureClientId: "11111111-1111-1111-1111-111111111111",
|
||||
})
|
||||
if accountName != "testaccount" {
|
||||
t.Errorf("expected the configured account name, got %q", accountName)
|
||||
}
|
||||
if accountKey != "" {
|
||||
t.Errorf("expected no account key alongside a client id, got %q", accountKey)
|
||||
}
|
||||
}
|
||||
|
||||
// An account name without a key authenticates with Entra ID
|
||||
func TestAzureRemoteStorageMakerWithoutAccountKey(t *testing.T) {
|
||||
t.Setenv("AZURE_STORAGE_ACCOUNT", "")
|
||||
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
|
||||
|
||||
maker := azureRemoteStorageMaker{}
|
||||
conf := &remote_pb.RemoteConf{
|
||||
Name: "test",
|
||||
AzureAccountName: "testaccount",
|
||||
}
|
||||
client, err := maker.Make(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create client without an account key: %v", err)
|
||||
}
|
||||
if client.(*azureRemoteStorageClient).client == nil {
|
||||
t.Error("Expected a blob client")
|
||||
}
|
||||
}
|
||||
|
||||
// Test error cases
|
||||
func TestAzureStorageClientErrors(t *testing.T) {
|
||||
// Test with invalid credentials
|
||||
maker := azureRemoteStorageMaker{}
|
||||
conf := &remote_pb.RemoteConf{
|
||||
Name: "test",
|
||||
AzureAccountName: "invalid",
|
||||
AzureAccountKey: "aW52YWxpZGtleQ==", // base64 encoded "invalidkey"
|
||||
}
|
||||
|
||||
client, err := maker.Make(conf)
|
||||
if err != nil {
|
||||
t.Skip("Invalid credentials correctly rejected at client creation")
|
||||
}
|
||||
|
||||
// If client creation succeeded, operations should fail
|
||||
azClient := client.(*azureRemoteStorageClient)
|
||||
loc := &remote_pb.RemoteStorageLocation{
|
||||
Name: "test",
|
||||
Bucket: "nonexistent",
|
||||
Path: "/test.txt",
|
||||
}
|
||||
|
||||
// These operations should fail with invalid credentials
|
||||
_, err = azClient.ReadFile(loc, 0, 10)
|
||||
if err == nil {
|
||||
t.Log("Expected error with invalid credentials on ReadFile, but got none (might be cached)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureRemoteStorageClientImplementsInterface(t *testing.T) {
|
||||
var _ remote_storage.RemoteStorageClient = (*azureRemoteStorageClient)(nil)
|
||||
}
|
||||
|
||||
func TestAzureErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
|
||||
require.Error(t, remote_storage.ErrRemoteObjectNotFound)
|
||||
require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
|
||||
}
|
||||
Reference in New Issue
Block a user