snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
go: build dev binaries / cleanup (push) Successful in 4s
Codespell / Check for spelling errors (push) Failing after 3s
docker: build dev containers / build-rust-binaries (amd64, x86_64-unknown-linux-musl) (push) Failing after 3s
docker: build dev containers / build-dev-containers (push) Has been skipped
docker: build dev containers / build-rust-binaries (arm64, true, aarch64-unknown-linux-musl) (push) Failing after 3s
End to End / FUSE Mount (push) Failing after 2s
EC Integration Tests / EC Integration Tests (push) Failing after 30s
FUSE DLM Integration Tests / FUSE DLM Integration Tests (push) Failing after 3s
FUSE Volume Server Failover Tests / FUSE Volume Server Failover (push) Failing after 3s
FUSE Integration Tests / FUSE Integration Testing (push) Failing after 2s
FUSE P2P Peer Chunk Sharing Integration Tests / FUSE P2P Peer Chunk Sharing (push) Failing after 3s
go: build binary / Go Vet (push) Failing after 2s
go: build binary / Go Vet 32-bit (push) Failing after 3s
go: build binary / Build (push) Failing after 4s
go: build binary / Build cross-platform (freebsd/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/amd64) (push) Failing after 3s
go: build binary / Build cross-platform (darwin/arm64) (push) Failing after 3s
go: build binary / Build cross-platform (windows/arm64) (push) Failing after 3s
go: build binary / Test (push) Failing after 2s
go: build binary / Test 32-bit (push) Failing after 3s
helm: lint and test charts / lint-test (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (11) (push) Failing after 3s
Java Client Integration Tests / Java Integration Tests (17) (push) Failing after 3s
go: build dev binaries / build_dev_linux_windows (amd64, linux) (push) Failing after 2s
go: build dev binaries / build_dev_linux_windows (amd64, windows) (push) Failing after 3s
go: build dev binaries / build_dev_darwin (amd64, darwin) (push) Has been skipped
go: build dev binaries / build_dev_darwin (arm64, darwin) (push) Has been skipped
Java Client Unit Tests / Java Unit Tests (11) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (17) (push) Failing after 2s
Java Client Unit Tests / Java Unit Tests (21) (push) Failing after 3s
Java Client Unit Tests / Java Unit Tests (8) (push) Failing after 3s
Kafka Quick Test (Load Test with Schema Registry) / Kafka Client Load Test (Quick) (push) Failing after 3s
Kafka Gateway Tests / Kafka Unit Tests (unit-tests-1) (push) Failing after 11s
Kafka Gateway Tests / Kafka Integration Tests (Critical) (integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka End-to-End Tests (with SMQ) (e2e-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Consumer Group Tests (Highly Isolated) (consumer-group-1) (push) Failing after 2s
Kafka Gateway Tests / Kafka Client Compatibility (with SMQ) (client-compat-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka SMQ Integration Tests (Full Stack) (smq-integration-1) (push) Failing after 3s
Kafka Gateway Tests / Kafka Protocol Tests (Isolated) (protocol-1) (push) Failing after 3s
KMS Tests / KMS Provider Integration Tests (push) Failing after 2s
KMS Tests / S3 KMS End-to-End Tests (push) Failing after 3s
Master Cold Start Tests / Master Cold Start Tests (push) Failing after 4s
Metadata Subscribe Integration Tests / Metadata Subscribe Integration Tests (push) Failing after 3s
Multi-Master Tests / Multi-Master Failover Tests (push) Failing after 3s
Performance / CPU and Heap Profile (push) Failing after 2s
Performance / S3 Read/Write Benchmark (rust volume) (push) Failing after 3s
pjdfstest POSIX Compliance / pjdfstest (push) Failing after 2s
Plugin Worker Integration Tests / Plugin Worker: erasure_coding (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: vacuum (push) Failing after 3s
Rust Volume Server Tests / Rust Integration Tests (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 1) (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 1) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 2) (push) Failing after 3s
Rust Volume Server Tests / Go Tests with Rust Volume (grpc - Shard 3) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Distributed Tests (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Performance Tests (push) Failing after 3s
Performance / Throughput Benchmark (go volume) (push) Failing after 3s
Performance / Throughput Benchmark (rust volume) (push) Failing after 2s
Performance / S3 Read/Write Benchmark (go volume) (push) Failing after 3s
Plugin Worker Integration Tests / Plugin Worker: volume_balance (push) Failing after 3s
PostgreSQL Gateway Tests / PostgreSQL Basic Tests (push) Failing after 3s
Rust Volume Server Tests / Rust Unit Tests (push) Failing after 2s
Rust Volume Server Tests / Go Tests with Rust Volume (http - Shard 3) (push) Failing after 3s
Rust Plugin Worker Tests / Rust Plugin Worker Build and Unit Tests (push) Failing after 3s
rust: build dev volume server binaries / cleanup (push) Successful in 3s
S3 ETag and ACL Tests / S3 ETag + ACL Integration Tests (push) Failing after 3s
S3 IAM Integration Tests / IAM Unit Tests (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (advanced) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (basic) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (group) (push) Failing after 3s
S3 IAM Integration Tests / S3 IAM Integration Tests (policy-enforcement) (push) Failing after 2s
S3 IAM Integration Tests / S3 IAM Integration Tests (sts) (push) Failing after 4s
S3 Keycloak Integration Tests / S3 Keycloak Integration Tests (push) Failing after 3s
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.11) (push) Failing after 3s
S3 PyArrow Parquet Tests / Go Unit Tests (Implicit Directory) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.12) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-amd64, x86_64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-darwin (darwin-arm64, aarch64-apple-darwin) (push) Has been cancelled
rust: build dev volume server binaries / build-rust-volume-dev-linux (linux-amd64, x86_64-unknown-linux-gnu) (push) Has been cancelled
S3 PyArrow Parquet Tests / PyArrow Parquet Tests (Python 3.9) (push) Has been cancelled
S3 Proxy Signature Tests / S3 Proxy Signature Verification Tests (push) Has been cancelled
S3 SDK V2 Route Disambiguation Tests / S3 SDK V2 Routing Tests (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (quick) (push) Has been cancelled
S3 SSE Tests / S3 SSE Compatibility Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Metadata Persistence Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Copy Operations Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Multipart Upload Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Performance Test (push) Has been cancelled
S3 SSE Tests / S3 Volume Encryption Test (push) Has been cancelled
S3 SSE Tests / S3 SSE Integration Tests (comprehensive) (push) Has been cancelled
Spark Integration Tests / Spark Integration Tests (push) Failing after 2s
Telemetry Integration Tests / telemetry-integration-test (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 1) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 1) (push) Failing after 3s
Ceph S3 tests / Basic S3 tests (KV store) (push) Has been cancelled
Ceph S3 tests / S3 Versioning & Object Lock tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Unit Tests (push) Failing after 3s
Ceph S3 tests / S3 CORS tests (push) Has been cancelled
Ceph S3 tests / SeaweedFS Custom S3 Copy tests (push) Has been cancelled
Ceph S3 tests / Basic S3 tests (SQL store) (push) Has been cancelled
test s3 over https using aws-cli / awscli-tests (push) Has been cancelled
TLS Rotation Integration Tests / TLS Rotation Integration Tests (push) Has been cancelled
Vacuum Integration Tests / Vacuum Integration Tests (push) Has been cancelled
S3 Policy Integration Tests / S3 Policy Variables Integration Tests (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (advanced-policy) (push) Failing after 3s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (basic-policy) (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Enforcement Integration Tests (contextual-policy) (push) Failing after 4s
S3 Policy Integration Tests / Trusted Proxy Detection Tests (push) Failing after 2s
S3 Policy Integration Tests / S3 Policy Shell Integration Tests (push) Failing after 3s
Samba on FUSE Integration / samba-integration (push) Failing after 2s
SFTP Integration Tests / SFTP Integration Testing (push) Failing after 3s
terraform: validate and test modules / fmt, validate, plan-level tests (push) Failing after 3s
terraform: validate and test modules / local cluster smoke test (real weed) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 3) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (grpc - Shard 2) (push) Failing after 3s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 2) (push) Failing after 2s
Volume Server Integration Tests / Volume Server Integration Tests (http - Shard 3) (push) Failing after 5s
mount: windows conformance / WinFsp conformance (push) Has been cancelled
mount: windows / Mount on Windows (push) Has been cancelled

This commit is contained in:
upstream-mirror
2026-09-16 03:23:42 +05:00
commit 89f5be005d
3997 changed files with 961970 additions and 0 deletions

View File

@@ -0,0 +1,111 @@
package azure
import (
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// Azure storage account names are 3 to 24 lowercase letters and digits. The
// name lands in the service URL, where a stray "/", "?" or "@" would move the
// authority somewhere else and send an authenticated request to that host.
var validAzureAccountName = regexp.MustCompile(`^[a-z0-9]{3,24}$`)
// NewAzBlobClient builds a blob service client for accountName.
//
// An empty accountKey selects Entra ID instead of a shared key: azidentity
// resolves a workload identity, a managed identity, or a developer login from
// the environment, and access is granted through RBAC. Fleets that cannot
// distribute and rotate storage account keys authenticate that way. clientID
// pins a user-assigned identity when the environment offers more than one.
//
// endpoint is the blob service URL, for accounts outside the public cloud. An
// empty endpoint derives the public one from accountName.
//
// A non-nil httpClient overrides the SDK transport; the volume server passes one
// whose dialer pins the validated endpoint against DNS rebinding.
func NewAzBlobClient(accountName, accountKey, clientID, endpoint string, httpClient *http.Client) (*azblob.Client, error) {
if accountName == "" {
return nil, fmt.Errorf("azure account name is required")
}
if !validAzureAccountName.MatchString(accountName) {
return nil, fmt.Errorf("invalid azure account name %q: expecting 3 to 24 lowercase letters and digits", accountName)
}
serviceURL, err := azureServiceURL(accountName, endpoint)
if err != nil {
return nil, err
}
options := DefaultAzBlobClientOptions()
if httpClient != nil {
options.Transport = httpClient
}
if accountKey == "" {
credential, err := newAzureTokenCredential(clientID)
if err != nil {
return nil, fmt.Errorf("failed to create Azure Entra ID credential for account %s: %w", accountName, err)
}
glog.V(1).Infof("azure %s: authenticating with Entra ID", accountName)
client, err := azblob.NewClient(serviceURL, credential, options)
if err != nil {
return nil, fmt.Errorf("failed to create Azure client: %w", err)
}
return client, nil
}
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
return nil, fmt.Errorf("failed to create Azure credential with account name:%s: %w", accountName, err)
}
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, options)
if err != nil {
return nil, fmt.Errorf("failed to create Azure client: %w", err)
}
return client, nil
}
// azureServiceURL locates the blob service. Sovereign clouds and private
// endpoints do not live under blob.core.windows.net, so they name the service
// URL outright rather than having it derived from the account.
func azureServiceURL(accountName, endpoint string) (string, error) {
if endpoint == "" {
return fmt.Sprintf("https://%s.blob.core.windows.net/", accountName), nil
}
parsed, err := url.Parse(endpoint)
if err != nil {
return "", fmt.Errorf("invalid azure endpoint %q: %w", endpoint, err)
}
// plain http would carry the account key or the bearer token in the clear
if parsed.Scheme != "https" || parsed.Hostname() == "" {
return "", fmt.Errorf("invalid azure endpoint %q: expecting an https service url, such as https://%s.blob.core.usgovcloudapi.net/", endpoint, accountName)
}
return endpoint, nil
}
// newAzureTokenCredential resolves an Entra ID credential. Without a pinned
// clientID the default chain discovers whatever the host offers. With one, the
// federated token file projected by the Azure workload identity webhook tells
// the two identity flavors apart.
func newAzureTokenCredential(clientID string) (azcore.TokenCredential, error) {
if clientID == "" {
return azidentity.NewDefaultAzureCredential(nil)
}
if os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" {
return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
ClientID: clientID,
})
}
return azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
ID: azidentity.ClientID(clientID),
})
}

View File

@@ -0,0 +1,98 @@
package azure
import (
"path/filepath"
"testing"
)
func TestNewAzBlobClientRequiresAccountName(t *testing.T) {
if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", "", "", nil); err == nil {
t.Error("expected an error without an account name")
}
}
func TestNewAzBlobClientRejectsMalformedAccountName(t *testing.T) {
for _, accountName := range []string{"ab", "TestAccount", "test-account", "evil.com/x", "evil@host.com", "account?x=1"} {
if _, err := NewAzBlobClient(accountName, "", "", "", nil); err == nil {
t.Errorf("expected an error for account name %q", accountName)
}
}
}
func TestNewAzBlobClientSharedKey(t *testing.T) {
client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "", "", nil)
if err != nil {
t.Fatalf("failed to create a shared key client: %v", err)
}
if client == nil {
t.Error("expected a client")
}
}
func TestNewAzBlobClientSharedKeyRejectsMalformedKey(t *testing.T) {
if _, err := NewAzBlobClient("testaccount", "not base64", "", "", nil); err == nil {
t.Error("expected an error with a malformed account key")
}
}
func TestAzureServiceURL(t *testing.T) {
serviceURL, err := azureServiceURL("testaccount", "")
if err != nil {
t.Fatalf("failed to derive the public service url: %v", err)
}
if serviceURL != "https://testaccount.blob.core.windows.net/" {
t.Errorf("unexpected public service url %q", serviceURL)
}
government := "https://testaccount.blob.core.usgovcloudapi.net/"
serviceURL, err = azureServiceURL("testaccount", government)
if err != nil {
t.Fatalf("failed to keep the configured service url: %v", err)
}
if serviceURL != government {
t.Errorf("expected %q, got %q", government, serviceURL)
}
for _, endpoint := range []string{"core.usgovcloudapi.net", "http://testaccount.blob.core.windows.net/", "https://", "https://:443/", "://x"} {
if _, err := azureServiceURL("testaccount", endpoint); err == nil {
t.Errorf("expected an error for endpoint %q", endpoint)
}
}
}
// no account key: authenticate through the Entra ID chain instead
func TestNewAzBlobClientEntraID(t *testing.T) {
client, err := NewAzBlobClient("testaccount", "", "", "", nil)
if err != nil {
t.Fatalf("failed to create an Entra ID client: %v", err)
}
if client == nil {
t.Error("expected a client")
}
}
func TestNewAzBlobClientWorkloadIdentity(t *testing.T) {
t.Setenv("AZURE_FEDERATED_TOKEN_FILE", filepath.Join(t.TempDir(), "token"))
t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000")
client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111", "", nil)
if err != nil {
t.Fatalf("failed to create a workload identity client: %v", err)
}
if client == nil {
t.Error("expected a client")
}
}
// a pinned client id without a projected token file is a plain managed identity
func TestNewAzureTokenCredentialManagedIdentity(t *testing.T) {
t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "")
credential, err := newAzureTokenCredential("11111111-1111-1111-1111-111111111111")
if err != nil {
t.Fatalf("failed to create a managed identity credential: %v", err)
}
if credential == nil {
t.Error("expected a credential")
}
}

View File

@@ -0,0 +1,589 @@
package azure
import (
"context"
"fmt"
"io"
"math"
"net/http"
"os"
"reflect"
"regexp"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
defaultBlockSize = 4 * 1024 * 1024
defaultConcurrency = 16
defaultReadConcurrency = 16
// DefaultAzureOpTimeout is the timeout for short metadata operations
// (GetProperties, Delete, conditional updates, etc.). Blob body transfers
// (ReadFile / WriteFile) are NOT bounded by this timeout: they use
// parallel 4 MiB block transfers and rely on the SDK's per-try TryTimeout
// to bound each individual block.
DefaultAzureOpTimeout = 60 * time.Second
)
// DefaultAzBlobClientOptions returns the default Azure blob client options
// with consistent retry configuration across the application.
// This centralizes the retry policy to ensure uniform behavior between
// remote storage and replication sink implementations.
//
// TryTimeout bounds a single HTTP attempt (headers + body) in the Azure SDK.
// With parallel 4 MiB block transfers (DownloadBuffer / UploadStream) a 60s
// TryTimeout is comfortable. A previous regression set this to 10s "to fail
// faster on auth issues" — that silently broke large-blob reads: a single
// non-parallelized Download of more than ~40 MiB over a typical WAN link
// cannot finish in 10s, so re-caching a large remote object (e.g. a 2 GB
// blob mounted via `remote.mount` and fetched through the S3 gateway) failed
// every attempt with a generic context-deadline error.
func DefaultAzBlobClientOptions() *azblob.ClientOptions {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Retry: policy.RetryOptions{
MaxRetries: 3,
TryTimeout: 60 * time.Second,
RetryDelay: 1 * time.Second,
MaxRetryDelay: 10 * time.Second,
},
},
}
}
// invalidMetadataChars matches any character that is not valid in Azure metadata keys.
// Azure metadata keys must be valid C# identifiers: letters, digits, and underscores only.
var invalidMetadataChars = regexp.MustCompile(`[^a-zA-Z0-9_]`)
// sanitizeMetadataKey converts an S3 metadata key to a valid Azure metadata key.
// Azure metadata keys must be valid C# identifiers (letters, digits, underscores only, cannot start with digit).
// To prevent collisions, invalid characters are replaced with their hex representation (_XX_).
// Examples:
// - "my-key" -> "my_2d_key"
// - "my.key" -> "my_2e_key"
// - "key@value" -> "key_40_value"
func sanitizeMetadataKey(key string) string {
// Replace each invalid character with _XX_ where XX is the hex code
result := invalidMetadataChars.ReplaceAllStringFunc(key, func(s string) string {
return fmt.Sprintf("_%02x_", s[0])
})
// Azure metadata keys cannot start with a digit
if len(result) > 0 && result[0] >= '0' && result[0] <= '9' {
result = "_" + result
}
return result
}
func init() {
remote_storage.RemoteStorageClientMakers["azure"] = new(azureRemoteStorageMaker)
}
// resolveAzureAccount completes the configured account from the environment. A
// configured client id asks for Entra ID, so a key left over in the environment
// must not quietly take the request back to shared key auth.
func resolveAzureAccount(conf *remote_pb.RemoteConf) (accountName, accountKey string) {
accountName, accountKey = conf.AzureAccountName, conf.AzureAccountKey
if len(accountName) == 0 {
accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
}
if len(accountKey) == 0 && len(conf.AzureClientId) == 0 {
accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
}
return
}
type azureRemoteStorageMaker struct{}
func (s azureRemoteStorageMaker) HasBucket() bool {
return true
}
func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
return MakeWithHTTPClient(conf, nil)
}
// MakeWithHTTPClient builds an azure client using the supplied *http.Client for
// its transport (or the SDK default when nil). Callers that need to pin the dial
// path against DNS rebinding pass a client whose transport has a guarded
// DialContext, mirroring the S3 backend.
func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
client := &azureRemoteStorageClient{
conf: conf,
}
accountName, accountKey := resolveAzureAccount(conf)
if len(accountName) == 0 {
return nil, fmt.Errorf("neither azure_account_name nor the AZURE_STORAGE_ACCOUNT environment variable is set")
}
azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId, conf.AzureEndpoint, httpClient)
if err != nil {
return nil, err
}
client.client = azClient
return client, nil
}
type azureRemoteStorageClient struct {
conf *remote_pb.RemoteConf
client *azblob.Client
}
var _ = remote_storage.RemoteStorageClient(&azureRemoteStorageClient{})
var _ = remote_storage.RemoteStorageConcurrentReader(&azureRemoteStorageClient{})
func (az *azureRemoteStorageClient) uploadConcurrency() int {
if n := int(az.conf.GetUploadConcurrency()); n > 0 {
return n
}
return defaultConcurrency
}
func (az *azureRemoteStorageClient) downloadConcurrency() int {
if n := int(az.conf.GetDownloadConcurrency()); n > 0 {
return n
}
return defaultReadConcurrency
}
func (az *azureRemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
pathKey := loc.Path[1:]
if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
pathKey += "/"
}
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
pager := containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
Prefix: &pathKey,
})
for pager.More() {
resp, pageErr := pager.NextPage(ctx)
if pageErr != nil {
return fmt.Errorf("azure list directory %s%s: %w", loc.Bucket, loc.Path, pageErr)
}
for _, prefix := range resp.Segment.BlobPrefixes {
if prefix.Name == nil {
continue
}
dirKey := "/" + strings.TrimSuffix(*prefix.Name, "/")
dir, name := util.FullPath(dirKey).DirAndName()
if err = visitFn(dir, name, true, nil); err != nil {
return fmt.Errorf("azure processing directory prefix %s: %w", *prefix.Name, err)
}
}
for _, blobItem := range resp.Segment.BlobItems {
if blobItem.Name == nil {
continue
}
key := "/" + *blobItem.Name
if strings.HasSuffix(key, "/") {
continue // skip directory markers
}
dir, name := util.FullPath(key).DirAndName()
remoteEntry := &filer_pb.RemoteEntry{
StorageName: az.conf.Name,
}
if blobItem.Properties != nil {
if blobItem.Properties.LastModified != nil {
remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
}
if blobItem.Properties.ContentLength != nil {
remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
}
if blobItem.Properties.ETag != nil {
remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
}
remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
}
if err = visitFn(dir, name, false, remoteEntry); err != nil {
return fmt.Errorf("azure processing blob %s: %w", *blobItem.Name, err)
}
}
}
return nil
}
func (az *azureRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
key := loc.Path[1:]
ctx, cancel := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
defer cancel()
resp, err := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key).GetProperties(ctx, nil)
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return nil, remote_storage.ErrRemoteObjectNotFound
}
return nil, fmt.Errorf("stat azure %s%s: %w", loc.Bucket, loc.Path, err)
}
remoteEntry = &filer_pb.RemoteEntry{
StorageName: az.conf.Name,
}
if resp.ContentLength != nil {
remoteEntry.RemoteSize = *resp.ContentLength
}
if resp.LastModified != nil {
remoteEntry.RemoteMtime = resp.LastModified.Unix()
}
if resp.ETag != nil {
remoteEntry.RemoteETag = string(*resp.ETag)
}
remoteEntry.RemoteContentEncoding = remoteContentEncoding(resp.ContentEncoding)
return remoteEntry, nil
}
// blob properties report no Content-Encoding as nil; the remote entry records
// that authoritatively as empty
func remoteContentEncoding(encoding *string) *string {
if encoding == nil {
empty := ""
return &empty
}
return encoding
}
func (az *azureRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
pathKey := loc.Path[1:]
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
// List blobs with pager
pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Prefix: &pathKey,
})
for pager.More() {
resp, err := pager.NextPage(context.Background())
if err != nil {
return fmt.Errorf("azure traverse %s%s: %w", loc.Bucket, loc.Path, err)
}
for _, blobItem := range resp.Segment.BlobItems {
if blobItem.Name == nil {
continue
}
key := "/" + *blobItem.Name
dir, name := util.FullPath(key).DirAndName()
remoteEntry := &filer_pb.RemoteEntry{
StorageName: az.conf.Name,
}
if blobItem.Properties != nil {
if blobItem.Properties.LastModified != nil {
remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
}
if blobItem.Properties.ContentLength != nil {
remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
}
if blobItem.Properties.ETag != nil {
remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
}
remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
}
err = visitFn(dir, name, false, remoteEntry)
if err != nil {
return fmt.Errorf("azure processing %s%s: %w", loc.Bucket, loc.Path, err)
}
}
}
return
}
func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
return az.ReadFileWithConcurrency(loc, offset, size, 0)
}
// ReadFileWithConcurrency fetches a byte range of a blob using the Azure SDK's
// parallel block downloader. The blob is split into `defaultBlockSize` (4 MiB)
// blocks that are fetched in parallel up to `concurrency` at a time. This
// keeps each individual HTTP GET small enough to complete well within the
// SDK's per-try TryTimeout, so large-blob reads remain reliable even on
// slower WAN links.
func (az *azureRemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error) {
if offset < 0 {
return nil, fmt.Errorf("invalid offset %d for %s%s", offset, loc.Bucket, loc.Path)
}
if size < 0 {
return nil, fmt.Errorf("invalid size %d for %s%s", size, loc.Bucket, loc.Path)
}
if concurrency <= 0 {
concurrency = az.downloadConcurrency()
}
if concurrency > math.MaxUint16 {
concurrency = math.MaxUint16
}
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
// DownloadBuffer requires a pre-sized destination. If the caller asked for
// "read to end" (size == 0), discover the blob length via GetProperties
// first so we can allocate accurately.
if size == 0 {
propsCtx, cancelProps := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
props, propsErr := blobClient.GetProperties(propsCtx, nil)
cancelProps()
if propsErr != nil {
return nil, fmt.Errorf("get properties %s%s: %w", loc.Bucket, loc.Path, propsErr)
}
if props.ContentLength == nil {
return nil, fmt.Errorf("azure %s%s: missing ContentLength", loc.Bucket, loc.Path)
}
size = *props.ContentLength - offset
if size <= 0 {
return []byte{}, nil
}
}
data = make([]byte, size)
n, err := blobClient.DownloadBuffer(context.Background(), data, &blob.DownloadBufferOptions{
Range: blob.HTTPRange{
Offset: offset,
Count: size,
},
BlockSize: defaultBlockSize,
Concurrency: uint16(concurrency),
})
if err != nil {
return nil, fmt.Errorf("failed to download file %s%s: %w", loc.Bucket, loc.Path, err)
}
// Pre-sized buffer: a short read stays zero-padded. Reject it rather than
// cache corrupt content.
if n != size {
return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size)
}
return data, nil
}
func (az *azureRemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
downloadResponse, err := blobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
Range: blob.HTTPRange{
Offset: offset,
Count: size,
},
})
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return nil, remote_storage.ErrRemoteObjectNotFound
}
return nil, fmt.Errorf("failed to open stream for %s%s: %v", loc.Bucket, loc.Path, err)
}
return downloadResponse.Body, nil
}
func (az *azureRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
return nil
}
func (az *azureRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
// the trailing slash keeps sibling prefixes that share the name intact
prefix := loc.Path[1:]
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
if prefix == "" {
// the mount root maps to the whole container; wiping every blob from a
// single namespace event is too destructive, so keep them
glog.Warningf("azure %s: skip removing directory mapped to the container root", loc.Bucket)
return nil
}
containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Prefix: &prefix,
})
for pager.More() {
resp, pageErr := pager.NextPage(context.Background())
if pageErr != nil {
return fmt.Errorf("azure list %s/%s: %w", loc.Bucket, prefix, pageErr)
}
for _, blobItem := range resp.Segment.BlobItems {
if blobItem.Name == nil {
continue
}
_, delErr := containerClient.NewBlobClient(*blobItem.Name).Delete(context.Background(), &blob.DeleteOptions{
DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
})
if delErr != nil && !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
return fmt.Errorf("azure delete %s/%s: %w", loc.Bucket, *blobItem.Name, delErr)
}
}
}
return nil
}
func (az *azureRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
// Upload from reader
metadata := toMetadata(entry.Extended)
httpHeaders := &blob.HTTPHeaders{}
if entry.Attributes != nil && entry.Attributes.Mime != "" {
httpHeaders.BlobContentType = &entry.Attributes.Mime
}
if contentEncoding := remote_storage.EntryContentEncoding(entry); contentEncoding != "" {
httpHeaders.BlobContentEncoding = &contentEncoding
}
_, err = blobClient.UploadStream(context.Background(), reader, &blockblob.UploadStreamOptions{
BlockSize: defaultBlockSize,
Concurrency: az.uploadConcurrency(),
HTTPHeaders: httpHeaders,
Metadata: metadata,
})
if err != nil {
return nil, fmt.Errorf("azure upload to %s%s: %w", loc.Bucket, loc.Path, err)
}
// read back the remote entry
return az.readFileRemoteEntry(loc)
}
func (az *azureRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
return az.StatFile(loc)
}
func toMetadata(attributes map[string][]byte) map[string]*string {
metadata := make(map[string]*string)
for k, v := range attributes {
if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) {
// S3 stores metadata keys in lowercase; normalize for consistency.
key := strings.ToLower(k[len(s3_constants.AmzUserMetaPrefix):])
// Sanitize key to prevent collisions and ensure Azure compliance
key = sanitizeMetadataKey(key)
val := string(v)
metadata[key] = &val
}
}
return metadata
}
func (az *azureRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
return nil
}
metadata := toMetadata(newEntry.Extended)
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
if !reflect.DeepEqual(toMetadata(oldEntry.Extended), metadata) {
if _, err = blobClient.SetMetadata(context.Background(), metadata, nil); err != nil {
return err
}
}
if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
// SetHTTPHeaders replaces the whole header set, so carry the rest over
props, getErr := blobClient.GetProperties(context.Background(), nil)
if getErr != nil {
return fmt.Errorf("azure get properties %s%s: %w", loc.Bucket, loc.Path, getErr)
}
httpHeaders := blob.ParseHTTPHeaders(props)
httpHeaders.BlobContentEncoding = nil
if encoding != "" {
httpHeaders.BlobContentEncoding = &encoding
}
if _, err = blobClient.SetHTTPHeaders(context.Background(), httpHeaders, nil); err != nil {
return fmt.Errorf("azure set http headers %s%s: %w", loc.Bucket, loc.Path, err)
}
}
return
}
func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
_, err = blobClient.Delete(context.Background(), &blob.DeleteOptions{
DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
})
if err != nil {
// Make delete idempotent - don't return error if blob doesn't exist
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return nil
}
return fmt.Errorf("azure delete %s%s: %w", loc.Bucket, loc.Path, err)
}
return
}
func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
pager := az.client.NewListContainersPager(nil)
for pager.More() {
resp, err := pager.NextPage(context.Background())
if err != nil {
return buckets, err
}
for _, containerItem := range resp.ContainerItems {
if containerItem.Name != nil {
bucket := &remote_storage.Bucket{
Name: *containerItem.Name,
}
if containerItem.Properties != nil && containerItem.Properties.LastModified != nil {
bucket.CreatedAt = *containerItem.Properties.LastModified
}
buckets = append(buckets, bucket)
}
}
}
return
}
func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) {
containerClient := az.client.ServiceClient().NewContainerClient(name)
_, err = containerClient.Create(context.Background(), nil)
if err != nil {
return fmt.Errorf("create bucket %s: %w", name, err)
}
return
}
func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) {
containerClient := az.client.ServiceClient().NewContainerClient(name)
_, err = containerClient.Delete(context.Background(), nil)
if err != nil {
return fmt.Errorf("delete bucket %s: %w", name, err)
}
return
}

View File

@@ -0,0 +1,432 @@
package azure
import (
"bytes"
"fmt"
"os"
"testing"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/require"
)
// TestAzureStorageClientBasic tests basic Azure storage client operations
func TestAzureStorageClientBasic(t *testing.T) {
// Skip if credentials not available
accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
testContainer := os.Getenv("AZURE_TEST_CONTAINER")
if accountName == "" || accountKey == "" {
t.Skip("Skipping Azure storage test: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
}
if testContainer == "" {
testContainer = "seaweedfs-test"
}
// Create client
maker := azureRemoteStorageMaker{}
conf := &remote_pb.RemoteConf{
Name: "test-azure",
AzureAccountName: accountName,
AzureAccountKey: accountKey,
}
client, err := maker.Make(conf)
if err != nil {
t.Fatalf("Failed to create Azure client: %v", err)
}
azClient := client.(*azureRemoteStorageClient)
// Test 1: Create bucket/container
t.Run("CreateBucket", func(t *testing.T) {
err := azClient.CreateBucket(testContainer)
// Ignore error if bucket already exists
if err != nil && !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
t.Fatalf("Failed to create bucket: %v", err)
}
})
// Test 2: List buckets
t.Run("ListBuckets", func(t *testing.T) {
buckets, err := azClient.ListBuckets()
if err != nil {
t.Fatalf("Failed to list buckets: %v", err)
}
if len(buckets) == 0 {
t.Log("No buckets found (might be expected)")
} else {
t.Logf("Found %d buckets", len(buckets))
}
})
// Test 3: Write file
testContent := []byte("Hello from SeaweedFS Azure SDK migration test!")
testKey := fmt.Sprintf("/test-file-%d.txt", time.Now().Unix())
loc := &remote_pb.RemoteStorageLocation{
Name: "test-azure",
Bucket: testContainer,
Path: testKey,
}
t.Run("WriteFile", func(t *testing.T) {
entry := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Mime: "text/plain",
},
Extended: map[string][]byte{
"x-amz-meta-test-key": []byte("test-value"),
},
}
reader := bytes.NewReader(testContent)
remoteEntry, err := azClient.WriteFile(loc, entry, reader)
if err != nil {
t.Fatalf("Failed to write file: %v", err)
}
if remoteEntry == nil {
t.Fatal("Remote entry is nil")
}
if remoteEntry.RemoteSize != int64(len(testContent)) {
t.Errorf("Expected size %d, got %d", len(testContent), remoteEntry.RemoteSize)
}
})
// Test 4: Read file
t.Run("ReadFile", func(t *testing.T) {
data, err := azClient.ReadFile(loc, 0, int64(len(testContent)))
if err != nil {
t.Fatalf("Failed to read file: %v", err)
}
if !bytes.Equal(data, testContent) {
t.Errorf("Content mismatch. Expected: %s, Got: %s", testContent, data)
}
})
// Test 5: Read partial file
t.Run("ReadPartialFile", func(t *testing.T) {
data, err := azClient.ReadFile(loc, 0, 5)
if err != nil {
t.Fatalf("Failed to read partial file: %v", err)
}
expected := testContent[:5]
if !bytes.Equal(data, expected) {
t.Errorf("Content mismatch. Expected: %s, Got: %s", expected, data)
}
})
// Test 6: Update metadata
t.Run("UpdateMetadata", func(t *testing.T) {
oldEntry := &filer_pb.Entry{
Extended: map[string][]byte{
"x-amz-meta-test-key": []byte("test-value"),
},
}
newEntry := &filer_pb.Entry{
Extended: map[string][]byte{
"x-amz-meta-test-key": []byte("test-value"),
"x-amz-meta-new-key": []byte("new-value"),
},
}
err := azClient.UpdateFileMetadata(loc, oldEntry, newEntry)
if err != nil {
t.Fatalf("Failed to update metadata: %v", err)
}
})
// Test 7: Traverse (list objects)
t.Run("Traverse", func(t *testing.T) {
foundFile := false
err := azClient.Traverse(loc, func(dir string, name string, isDir bool, remoteEntry *filer_pb.RemoteEntry) error {
if !isDir && name == testKey[1:] { // Remove leading slash
foundFile = true
}
return nil
})
if err != nil {
t.Fatalf("Failed to traverse: %v", err)
}
if !foundFile {
t.Log("Test file not found in traverse (might be expected due to path matching)")
}
})
// Test 8: Delete file
t.Run("DeleteFile", func(t *testing.T) {
err := azClient.DeleteFile(loc)
if err != nil {
t.Fatalf("Failed to delete file: %v", err)
}
})
// Test 9: Verify file deleted (should fail)
t.Run("VerifyDeleted", func(t *testing.T) {
_, err := azClient.ReadFile(loc, 0, 10)
if !bloberror.HasCode(err, bloberror.BlobNotFound) {
t.Errorf("Expected BlobNotFound error, but got: %v", err)
}
})
// Clean up: Try to delete the test container
// Comment out if you want to keep the container
/*
t.Run("DeleteBucket", func(t *testing.T) {
err := azClient.DeleteBucket(testContainer)
if err != nil {
t.Logf("Warning: Failed to delete bucket: %v", err)
}
})
*/
}
// TestToMetadata tests the metadata conversion function
func TestToMetadata(t *testing.T) {
tests := []struct {
name string
input map[string][]byte
expected map[string]*string
}{
{
name: "basic metadata",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "key1": []byte("value1"),
s3_constants.AmzUserMetaPrefix + "key2": []byte("value2"),
},
expected: map[string]*string{
"key1": stringPtr("value1"),
"key2": stringPtr("value2"),
},
},
{
name: "metadata with dashes",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "content-type": []byte("text/plain"),
},
expected: map[string]*string{
"content_2d_type": stringPtr("text/plain"), // dash (0x2d) -> _2d_
},
},
{
name: "non-metadata keys ignored",
input: map[string][]byte{
"some-other-key": []byte("ignored"),
s3_constants.AmzUserMetaPrefix + "included": []byte("included"),
},
expected: map[string]*string{
"included": stringPtr("included"),
},
},
{
name: "keys starting with digits",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "123key": []byte("value1"),
s3_constants.AmzUserMetaPrefix + "456-test": []byte("value2"),
s3_constants.AmzUserMetaPrefix + "789": []byte("value3"),
},
expected: map[string]*string{
"_123key": stringPtr("value1"), // starts with digit -> prefix _
"_456_2d_test": stringPtr("value2"), // starts with digit AND has dash
"_789": stringPtr("value3"),
},
},
{
name: "uppercase and mixed case keys",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "My-Key": []byte("value1"),
s3_constants.AmzUserMetaPrefix + "UPPERCASE": []byte("value2"),
s3_constants.AmzUserMetaPrefix + "MiXeD-CaSe": []byte("value3"),
},
expected: map[string]*string{
"my_2d_key": stringPtr("value1"), // lowercase + dash -> _2d_
"uppercase": stringPtr("value2"),
"mixed_2d_case": stringPtr("value3"),
},
},
{
name: "keys with invalid characters",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "my.key": []byte("value1"),
s3_constants.AmzUserMetaPrefix + "key+plus": []byte("value2"),
s3_constants.AmzUserMetaPrefix + "key@symbol": []byte("value3"),
s3_constants.AmzUserMetaPrefix + "key-with.": []byte("value4"),
s3_constants.AmzUserMetaPrefix + "key/slash": []byte("value5"),
},
expected: map[string]*string{
"my_2e_key": stringPtr("value1"), // dot (0x2e) -> _2e_
"key_2b_plus": stringPtr("value2"), // plus (0x2b) -> _2b_
"key_40_symbol": stringPtr("value3"), // @ (0x40) -> _40_
"key_2d_with_2e_": stringPtr("value4"), // dash and dot
"key_2f_slash": stringPtr("value5"), // slash (0x2f) -> _2f_
},
},
{
name: "collision prevention",
input: map[string][]byte{
s3_constants.AmzUserMetaPrefix + "my-key": []byte("value1"),
s3_constants.AmzUserMetaPrefix + "my.key": []byte("value2"),
s3_constants.AmzUserMetaPrefix + "my_key": []byte("value3"),
},
expected: map[string]*string{
"my_2d_key": stringPtr("value1"), // dash (0x2d)
"my_2e_key": stringPtr("value2"), // dot (0x2e)
"my_key": stringPtr("value3"), // underscore is valid, no encoding
},
},
{
name: "empty input",
input: map[string][]byte{},
expected: map[string]*string{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := toMetadata(tt.input)
if len(result) != len(tt.expected) {
t.Errorf("Expected %d keys, got %d", len(tt.expected), len(result))
}
for key, expectedVal := range tt.expected {
if resultVal, ok := result[key]; !ok {
t.Errorf("Expected key %s not found", key)
} else if resultVal == nil || expectedVal == nil {
if resultVal != expectedVal {
t.Errorf("For key %s: expected %v, got %v", key, expectedVal, resultVal)
}
} else if *resultVal != *expectedVal {
t.Errorf("For key %s: expected %s, got %s", key, *expectedVal, *resultVal)
}
}
})
}
}
func contains(s, substr string) bool {
return bytes.Contains([]byte(s), []byte(substr))
}
func stringPtr(s string) *string {
return &s
}
// Benchmark tests
func BenchmarkToMetadata(b *testing.B) {
input := map[string][]byte{
"x-amz-meta-key1": []byte("value1"),
"x-amz-meta-key2": []byte("value2"),
"x-amz-meta-content-type": []byte("text/plain"),
"other-key": []byte("ignored"),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
toMetadata(input)
}
}
// Test that the maker implements the interface
func TestAzureRemoteStorageMaker(t *testing.T) {
maker := azureRemoteStorageMaker{}
if !maker.HasBucket() {
t.Error("Expected HasBucket() to return true")
}
// Test with missing credentials - unset env vars (auto-restored by t.Setenv)
t.Setenv("AZURE_STORAGE_ACCOUNT", "")
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
conf := &remote_pb.RemoteConf{
Name: "test",
}
_, err := maker.Make(conf)
if err == nil {
t.Error("Expected error with missing credentials")
}
}
func TestResolveAzureAccount(t *testing.T) {
t.Setenv("AZURE_STORAGE_ACCOUNT", "envaccount")
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "ZW52a2V5")
accountName, accountKey := resolveAzureAccount(&remote_pb.RemoteConf{})
if accountName != "envaccount" || accountKey != "ZW52a2V5" {
t.Errorf("expected the environment account, got %q/%q", accountName, accountKey)
}
// a configured identity keeps a leftover environment key out of the way
accountName, accountKey = resolveAzureAccount(&remote_pb.RemoteConf{
AzureAccountName: "testaccount",
AzureClientId: "11111111-1111-1111-1111-111111111111",
})
if accountName != "testaccount" {
t.Errorf("expected the configured account name, got %q", accountName)
}
if accountKey != "" {
t.Errorf("expected no account key alongside a client id, got %q", accountKey)
}
}
// An account name without a key authenticates with Entra ID
func TestAzureRemoteStorageMakerWithoutAccountKey(t *testing.T) {
t.Setenv("AZURE_STORAGE_ACCOUNT", "")
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
maker := azureRemoteStorageMaker{}
conf := &remote_pb.RemoteConf{
Name: "test",
AzureAccountName: "testaccount",
}
client, err := maker.Make(conf)
if err != nil {
t.Fatalf("Failed to create client without an account key: %v", err)
}
if client.(*azureRemoteStorageClient).client == nil {
t.Error("Expected a blob client")
}
}
// Test error cases
func TestAzureStorageClientErrors(t *testing.T) {
// Test with invalid credentials
maker := azureRemoteStorageMaker{}
conf := &remote_pb.RemoteConf{
Name: "test",
AzureAccountName: "invalid",
AzureAccountKey: "aW52YWxpZGtleQ==", // base64 encoded "invalidkey"
}
client, err := maker.Make(conf)
if err != nil {
t.Skip("Invalid credentials correctly rejected at client creation")
}
// If client creation succeeded, operations should fail
azClient := client.(*azureRemoteStorageClient)
loc := &remote_pb.RemoteStorageLocation{
Name: "test",
Bucket: "nonexistent",
Path: "/test.txt",
}
// These operations should fail with invalid credentials
_, err = azClient.ReadFile(loc, 0, 10)
if err == nil {
t.Log("Expected error with invalid credentials on ReadFile, but got none (might be cached)")
}
}
func TestAzureRemoteStorageClientImplementsInterface(t *testing.T) {
var _ remote_storage.RemoteStorageClient = (*azureRemoteStorageClient)(nil)
}
func TestAzureErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
require.Error(t, remote_storage.ErrRemoteObjectNotFound)
require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
}