snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/remote_storage/azure/azure_credentials.go

@@ -0,0 +1,111 @@
+package azure
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+// Azure storage account names are 3 to 24 lowercase letters and digits. The
+// name lands in the service URL, where a stray "/", "?" or "@" would move the
+// authority somewhere else and send an authenticated request to that host.
+var validAzureAccountName = regexp.MustCompile(`^[a-z0-9]{3,24}$`)
+
+// NewAzBlobClient builds a blob service client for accountName.
+//
+// An empty accountKey selects Entra ID instead of a shared key: azidentity
+// resolves a workload identity, a managed identity, or a developer login from
+// the environment, and access is granted through RBAC. Fleets that cannot
+// distribute and rotate storage account keys authenticate that way. clientID
+// pins a user-assigned identity when the environment offers more than one.
+//
+// endpoint is the blob service URL, for accounts outside the public cloud. An
+// empty endpoint derives the public one from accountName.
+//
+// A non-nil httpClient overrides the SDK transport; the volume server passes one
+// whose dialer pins the validated endpoint against DNS rebinding.
+func NewAzBlobClient(accountName, accountKey, clientID, endpoint string, httpClient *http.Client) (*azblob.Client, error) {
+
+	if accountName == "" {
+		return nil, fmt.Errorf("azure account name is required")
+	}
+	if !validAzureAccountName.MatchString(accountName) {
+		return nil, fmt.Errorf("invalid azure account name %q: expecting 3 to 24 lowercase letters and digits", accountName)
+	}
+
+	serviceURL, err := azureServiceURL(accountName, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	options := DefaultAzBlobClientOptions()
+	if httpClient != nil {
+		options.Transport = httpClient
+	}
+
+	if accountKey == "" {
+		credential, err := newAzureTokenCredential(clientID)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Azure Entra ID credential for account %s: %w", accountName, err)
+		}
+		glog.V(1).Infof("azure %s: authenticating with Entra ID", accountName)
+		client, err := azblob.NewClient(serviceURL, credential, options)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Azure client: %w", err)
+		}
+		return client, nil
+	}
+
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Azure credential with account name:%s: %w", accountName, err)
+	}
+	client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, options)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Azure client: %w", err)
+	}
+	return client, nil
+}
+
+// azureServiceURL locates the blob service. Sovereign clouds and private
+// endpoints do not live under blob.core.windows.net, so they name the service
+// URL outright rather than having it derived from the account.
+func azureServiceURL(accountName, endpoint string) (string, error) {
+	if endpoint == "" {
+		return fmt.Sprintf("https://%s.blob.core.windows.net/", accountName), nil
+	}
+	parsed, err := url.Parse(endpoint)
+	if err != nil {
+		return "", fmt.Errorf("invalid azure endpoint %q: %w", endpoint, err)
+	}
+	// plain http would carry the account key or the bearer token in the clear
+	if parsed.Scheme != "https" || parsed.Hostname() == "" {
+		return "", fmt.Errorf("invalid azure endpoint %q: expecting an https service url, such as https://%s.blob.core.usgovcloudapi.net/", endpoint, accountName)
+	}
+	return endpoint, nil
+}
+
+// newAzureTokenCredential resolves an Entra ID credential. Without a pinned
+// clientID the default chain discovers whatever the host offers. With one, the
+// federated token file projected by the Azure workload identity webhook tells
+// the two identity flavors apart.
+func newAzureTokenCredential(clientID string) (azcore.TokenCredential, error) {
+	if clientID == "" {
+		return azidentity.NewDefaultAzureCredential(nil)
+	}
+	if os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" {
+		return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
+			ClientID: clientID,
+		})
+	}
+	return azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{
+		ID: azidentity.ClientID(clientID),
+	})
+}

weed/remote_storage/azure/azure_credentials_test.go

@@ -0,0 +1,98 @@
+package azure
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func TestNewAzBlobClientRequiresAccountName(t *testing.T) {
+	if _, err := NewAzBlobClient("", "aW52YWxpZGtleQ==", "", "", nil); err == nil {
+		t.Error("expected an error without an account name")
+	}
+}
+
+func TestNewAzBlobClientRejectsMalformedAccountName(t *testing.T) {
+	for _, accountName := range []string{"ab", "TestAccount", "test-account", "evil.com/x", "evil@host.com", "account?x=1"} {
+		if _, err := NewAzBlobClient(accountName, "", "", "", nil); err == nil {
+			t.Errorf("expected an error for account name %q", accountName)
+		}
+	}
+}
+
+func TestNewAzBlobClientSharedKey(t *testing.T) {
+	client, err := NewAzBlobClient("testaccount", "aW52YWxpZGtleQ==", "", "", nil)
+	if err != nil {
+		t.Fatalf("failed to create a shared key client: %v", err)
+	}
+	if client == nil {
+		t.Error("expected a client")
+	}
+}
+
+func TestNewAzBlobClientSharedKeyRejectsMalformedKey(t *testing.T) {
+	if _, err := NewAzBlobClient("testaccount", "not base64", "", "", nil); err == nil {
+		t.Error("expected an error with a malformed account key")
+	}
+}
+
+func TestAzureServiceURL(t *testing.T) {
+	serviceURL, err := azureServiceURL("testaccount", "")
+	if err != nil {
+		t.Fatalf("failed to derive the public service url: %v", err)
+	}
+	if serviceURL != "https://testaccount.blob.core.windows.net/" {
+		t.Errorf("unexpected public service url %q", serviceURL)
+	}
+
+	government := "https://testaccount.blob.core.usgovcloudapi.net/"
+	serviceURL, err = azureServiceURL("testaccount", government)
+	if err != nil {
+		t.Fatalf("failed to keep the configured service url: %v", err)
+	}
+	if serviceURL != government {
+		t.Errorf("expected %q, got %q", government, serviceURL)
+	}
+
+	for _, endpoint := range []string{"core.usgovcloudapi.net", "http://testaccount.blob.core.windows.net/", "https://", "https://:443/", "://x"} {
+		if _, err := azureServiceURL("testaccount", endpoint); err == nil {
+			t.Errorf("expected an error for endpoint %q", endpoint)
+		}
+	}
+}
+
+// no account key: authenticate through the Entra ID chain instead
+func TestNewAzBlobClientEntraID(t *testing.T) {
+	client, err := NewAzBlobClient("testaccount", "", "", "", nil)
+	if err != nil {
+		t.Fatalf("failed to create an Entra ID client: %v", err)
+	}
+	if client == nil {
+		t.Error("expected a client")
+	}
+}
+
+func TestNewAzBlobClientWorkloadIdentity(t *testing.T) {
+	t.Setenv("AZURE_FEDERATED_TOKEN_FILE", filepath.Join(t.TempDir(), "token"))
+	t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000000")
+
+	client, err := NewAzBlobClient("testaccount", "", "11111111-1111-1111-1111-111111111111", "", nil)
+	if err != nil {
+		t.Fatalf("failed to create a workload identity client: %v", err)
+	}
+	if client == nil {
+		t.Error("expected a client")
+	}
+}
+
+// a pinned client id without a projected token file is a plain managed identity
+func TestNewAzureTokenCredentialManagedIdentity(t *testing.T) {
+	t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "")
+
+	credential, err := newAzureTokenCredential("11111111-1111-1111-1111-111111111111")
+	if err != nil {
+		t.Fatalf("failed to create a managed identity credential: %v", err)
+	}
+	if credential == nil {
+		t.Error("expected a credential")
+	}
+}

weed/remote_storage/azure/azure_storage_client.go

@@ -0,0 +1,589 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"reflect"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+const (
+	defaultBlockSize       = 4 * 1024 * 1024
+	defaultConcurrency     = 16
+	defaultReadConcurrency = 16
+
+	// DefaultAzureOpTimeout is the timeout for short metadata operations
+	// (GetProperties, Delete, conditional updates, etc.). Blob body transfers
+	// (ReadFile / WriteFile) are NOT bounded by this timeout: they use
+	// parallel 4 MiB block transfers and rely on the SDK's per-try TryTimeout
+	// to bound each individual block.
+	DefaultAzureOpTimeout = 60 * time.Second
+)
+
+// DefaultAzBlobClientOptions returns the default Azure blob client options
+// with consistent retry configuration across the application.
+// This centralizes the retry policy to ensure uniform behavior between
+// remote storage and replication sink implementations.
+//
+// TryTimeout bounds a single HTTP attempt (headers + body) in the Azure SDK.
+// With parallel 4 MiB block transfers (DownloadBuffer / UploadStream) a 60s
+// TryTimeout is comfortable. A previous regression set this to 10s "to fail
+// faster on auth issues" — that silently broke large-blob reads: a single
+// non-parallelized Download of more than ~40 MiB over a typical WAN link
+// cannot finish in 10s, so re-caching a large remote object (e.g. a 2 GB
+// blob mounted via `remote.mount` and fetched through the S3 gateway) failed
+// every attempt with a generic context-deadline error.
+func DefaultAzBlobClientOptions() *azblob.ClientOptions {
+	return &azblob.ClientOptions{
+		ClientOptions: azcore.ClientOptions{
+			Retry: policy.RetryOptions{
+				MaxRetries:    3,
+				TryTimeout:    60 * time.Second,
+				RetryDelay:    1 * time.Second,
+				MaxRetryDelay: 10 * time.Second,
+			},
+		},
+	}
+}
+
+// invalidMetadataChars matches any character that is not valid in Azure metadata keys.
+// Azure metadata keys must be valid C# identifiers: letters, digits, and underscores only.
+var invalidMetadataChars = regexp.MustCompile(`[^a-zA-Z0-9_]`)
+
+// sanitizeMetadataKey converts an S3 metadata key to a valid Azure metadata key.
+// Azure metadata keys must be valid C# identifiers (letters, digits, underscores only, cannot start with digit).
+// To prevent collisions, invalid characters are replaced with their hex representation (_XX_).
+// Examples:
+//   - "my-key" -> "my_2d_key"
+//   - "my.key" -> "my_2e_key"
+//   - "key@value" -> "key_40_value"
+func sanitizeMetadataKey(key string) string {
+	// Replace each invalid character with _XX_ where XX is the hex code
+	result := invalidMetadataChars.ReplaceAllStringFunc(key, func(s string) string {
+		return fmt.Sprintf("_%02x_", s[0])
+	})
+
+	// Azure metadata keys cannot start with a digit
+	if len(result) > 0 && result[0] >= '0' && result[0] <= '9' {
+		result = "_" + result
+	}
+
+	return result
+}
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["azure"] = new(azureRemoteStorageMaker)
+}
+
+// resolveAzureAccount completes the configured account from the environment. A
+// configured client id asks for Entra ID, so a key left over in the environment
+// must not quietly take the request back to shared key auth.
+func resolveAzureAccount(conf *remote_pb.RemoteConf) (accountName, accountKey string) {
+	accountName, accountKey = conf.AzureAccountName, conf.AzureAccountKey
+	if len(accountName) == 0 {
+		accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	}
+	if len(accountKey) == 0 && len(conf.AzureClientId) == 0 {
+		accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	}
+	return
+}
+
+type azureRemoteStorageMaker struct{}
+
+func (s azureRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}
+
+// MakeWithHTTPClient builds an azure client using the supplied *http.Client for
+// its transport (or the SDK default when nil). Callers that need to pin the dial
+// path against DNS rebinding pass a client whose transport has a guarded
+// DialContext, mirroring the S3 backend.
+func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
+
+	client := &azureRemoteStorageClient{
+		conf: conf,
+	}
+
+	accountName, accountKey := resolveAzureAccount(conf)
+	if len(accountName) == 0 {
+		return nil, fmt.Errorf("neither azure_account_name nor the AZURE_STORAGE_ACCOUNT environment variable is set")
+	}
+
+	azClient, err := NewAzBlobClient(accountName, accountKey, conf.AzureClientId, conf.AzureEndpoint, httpClient)
+	if err != nil {
+		return nil, err
+	}
+
+	client.client = azClient
+
+	return client, nil
+}
+
+type azureRemoteStorageClient struct {
+	conf   *remote_pb.RemoteConf
+	client *azblob.Client
+}
+
+var _ = remote_storage.RemoteStorageClient(&azureRemoteStorageClient{})
+var _ = remote_storage.RemoteStorageConcurrentReader(&azureRemoteStorageClient{})
+
+func (az *azureRemoteStorageClient) uploadConcurrency() int {
+	if n := int(az.conf.GetUploadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultConcurrency
+}
+
+func (az *azureRemoteStorageClient) downloadConcurrency() int {
+	if n := int(az.conf.GetDownloadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultReadConcurrency
+}
+
+func (az *azureRemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+	pathKey := loc.Path[1:]
+	if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
+		pathKey += "/"
+	}
+
+	containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
+	pager := containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
+		Prefix: &pathKey,
+	})
+
+	for pager.More() {
+		resp, pageErr := pager.NextPage(ctx)
+		if pageErr != nil {
+			return fmt.Errorf("azure list directory %s%s: %w", loc.Bucket, loc.Path, pageErr)
+		}
+
+		for _, prefix := range resp.Segment.BlobPrefixes {
+			if prefix.Name == nil {
+				continue
+			}
+			dirKey := "/" + strings.TrimSuffix(*prefix.Name, "/")
+			dir, name := util.FullPath(dirKey).DirAndName()
+			if err = visitFn(dir, name, true, nil); err != nil {
+				return fmt.Errorf("azure processing directory prefix %s: %w", *prefix.Name, err)
+			}
+		}
+
+		for _, blobItem := range resp.Segment.BlobItems {
+			if blobItem.Name == nil {
+				continue
+			}
+			key := "/" + *blobItem.Name
+			if strings.HasSuffix(key, "/") {
+				continue // skip directory markers
+			}
+			dir, name := util.FullPath(key).DirAndName()
+
+			remoteEntry := &filer_pb.RemoteEntry{
+				StorageName: az.conf.Name,
+			}
+			if blobItem.Properties != nil {
+				if blobItem.Properties.LastModified != nil {
+					remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
+				}
+				if blobItem.Properties.ContentLength != nil {
+					remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
+				}
+				if blobItem.Properties.ETag != nil {
+					remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
+				}
+				remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
+			}
+
+			if err = visitFn(dir, name, false, remoteEntry); err != nil {
+				return fmt.Errorf("azure processing blob %s: %w", *blobItem.Name, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (az *azureRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
+	key := loc.Path[1:]
+	ctx, cancel := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
+	defer cancel()
+	resp, err := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key).GetProperties(ctx, nil)
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("stat azure %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	remoteEntry = &filer_pb.RemoteEntry{
+		StorageName: az.conf.Name,
+	}
+	if resp.ContentLength != nil {
+		remoteEntry.RemoteSize = *resp.ContentLength
+	}
+	if resp.LastModified != nil {
+		remoteEntry.RemoteMtime = resp.LastModified.Unix()
+	}
+	if resp.ETag != nil {
+		remoteEntry.RemoteETag = string(*resp.ETag)
+	}
+	remoteEntry.RemoteContentEncoding = remoteContentEncoding(resp.ContentEncoding)
+	return remoteEntry, nil
+}
+
+// blob properties report no Content-Encoding as nil; the remote entry records
+// that authoritatively as empty
+func remoteContentEncoding(encoding *string) *string {
+	if encoding == nil {
+		empty := ""
+		return &empty
+	}
+	return encoding
+}
+
+func (az *azureRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	pathKey := loc.Path[1:]
+	containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
+
+	// List blobs with pager
+	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Prefix: &pathKey,
+	})
+
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		if err != nil {
+			return fmt.Errorf("azure traverse %s%s: %w", loc.Bucket, loc.Path, err)
+		}
+
+		for _, blobItem := range resp.Segment.BlobItems {
+			if blobItem.Name == nil {
+				continue
+			}
+			key := "/" + *blobItem.Name
+			dir, name := util.FullPath(key).DirAndName()
+
+			remoteEntry := &filer_pb.RemoteEntry{
+				StorageName: az.conf.Name,
+			}
+			if blobItem.Properties != nil {
+				if blobItem.Properties.LastModified != nil {
+					remoteEntry.RemoteMtime = blobItem.Properties.LastModified.Unix()
+				}
+				if blobItem.Properties.ContentLength != nil {
+					remoteEntry.RemoteSize = *blobItem.Properties.ContentLength
+				}
+				if blobItem.Properties.ETag != nil {
+					remoteEntry.RemoteETag = string(*blobItem.Properties.ETag)
+				}
+				remoteEntry.RemoteContentEncoding = remoteContentEncoding(blobItem.Properties.ContentEncoding)
+			}
+
+			err = visitFn(dir, name, false, remoteEntry)
+			if err != nil {
+				return fmt.Errorf("azure processing %s%s: %w", loc.Bucket, loc.Path, err)
+			}
+		}
+	}
+
+	return
+}
+
+func (az *azureRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+	return az.ReadFileWithConcurrency(loc, offset, size, 0)
+}
+
+// ReadFileWithConcurrency fetches a byte range of a blob using the Azure SDK's
+// parallel block downloader. The blob is split into `defaultBlockSize` (4 MiB)
+// blocks that are fetched in parallel up to `concurrency` at a time. This
+// keeps each individual HTTP GET small enough to complete well within the
+// SDK's per-try TryTimeout, so large-blob reads remain reliable even on
+// slower WAN links.
+func (az *azureRemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error) {
+
+	if offset < 0 {
+		return nil, fmt.Errorf("invalid offset %d for %s%s", offset, loc.Bucket, loc.Path)
+	}
+	if size < 0 {
+		return nil, fmt.Errorf("invalid size %d for %s%s", size, loc.Bucket, loc.Path)
+	}
+
+	if concurrency <= 0 {
+		concurrency = az.downloadConcurrency()
+	}
+	if concurrency > math.MaxUint16 {
+		concurrency = math.MaxUint16
+	}
+
+	key := loc.Path[1:]
+	blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
+
+	// DownloadBuffer requires a pre-sized destination. If the caller asked for
+	// "read to end" (size == 0), discover the blob length via GetProperties
+	// first so we can allocate accurately.
+	if size == 0 {
+		propsCtx, cancelProps := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
+		props, propsErr := blobClient.GetProperties(propsCtx, nil)
+		cancelProps()
+		if propsErr != nil {
+			return nil, fmt.Errorf("get properties %s%s: %w", loc.Bucket, loc.Path, propsErr)
+		}
+		if props.ContentLength == nil {
+			return nil, fmt.Errorf("azure %s%s: missing ContentLength", loc.Bucket, loc.Path)
+		}
+		size = *props.ContentLength - offset
+		if size <= 0 {
+			return []byte{}, nil
+		}
+	}
+
+	data = make([]byte, size)
+	n, err := blobClient.DownloadBuffer(context.Background(), data, &blob.DownloadBufferOptions{
+		Range: blob.HTTPRange{
+			Offset: offset,
+			Count:  size,
+		},
+		BlockSize:   defaultBlockSize,
+		Concurrency: uint16(concurrency),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	// Pre-sized buffer: a short read stays zero-padded. Reject it rather than
+	// cache corrupt content.
+	if n != size {
+		return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size)
+	}
+
+	return data, nil
+}
+
+func (az *azureRemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
+	key := loc.Path[1:]
+	blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
+
+	downloadResponse, err := blobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
+		Range: blob.HTTPRange{
+			Offset: offset,
+			Count:  size,
+		},
+	})
+	if err != nil {
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("failed to open stream for %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	return downloadResponse.Body, nil
+}
+
+func (az *azureRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return nil
+}
+
+func (az *azureRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	// the trailing slash keeps sibling prefixes that share the name intact
+	prefix := loc.Path[1:]
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	if prefix == "" {
+		// the mount root maps to the whole container; wiping every blob from a
+		// single namespace event is too destructive, so keep them
+		glog.Warningf("azure %s: skip removing directory mapped to the container root", loc.Bucket)
+		return nil
+	}
+
+	containerClient := az.client.ServiceClient().NewContainerClient(loc.Bucket)
+	pager := containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
+		Prefix: &prefix,
+	})
+	for pager.More() {
+		resp, pageErr := pager.NextPage(context.Background())
+		if pageErr != nil {
+			return fmt.Errorf("azure list %s/%s: %w", loc.Bucket, prefix, pageErr)
+		}
+		for _, blobItem := range resp.Segment.BlobItems {
+			if blobItem.Name == nil {
+				continue
+			}
+			_, delErr := containerClient.NewBlobClient(*blobItem.Name).Delete(context.Background(), &blob.DeleteOptions{
+				DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
+			})
+			if delErr != nil && !bloberror.HasCode(delErr, bloberror.BlobNotFound) {
+				return fmt.Errorf("azure delete %s/%s: %w", loc.Bucket, *blobItem.Name, delErr)
+			}
+		}
+	}
+	return nil
+}
+
+func (az *azureRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	key := loc.Path[1:]
+	blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
+
+	// Upload from reader
+	metadata := toMetadata(entry.Extended)
+	httpHeaders := &blob.HTTPHeaders{}
+	if entry.Attributes != nil && entry.Attributes.Mime != "" {
+		httpHeaders.BlobContentType = &entry.Attributes.Mime
+	}
+	if contentEncoding := remote_storage.EntryContentEncoding(entry); contentEncoding != "" {
+		httpHeaders.BlobContentEncoding = &contentEncoding
+	}
+
+	_, err = blobClient.UploadStream(context.Background(), reader, &blockblob.UploadStreamOptions{
+		BlockSize:   defaultBlockSize,
+		Concurrency: az.uploadConcurrency(),
+		HTTPHeaders: httpHeaders,
+		Metadata:    metadata,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("azure upload to %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+
+	// read back the remote entry
+	return az.readFileRemoteEntry(loc)
+}
+
+func (az *azureRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	return az.StatFile(loc)
+}
+
+func toMetadata(attributes map[string][]byte) map[string]*string {
+	metadata := make(map[string]*string)
+	for k, v := range attributes {
+		if strings.HasPrefix(k, s3_constants.AmzUserMetaPrefix) {
+			// S3 stores metadata keys in lowercase; normalize for consistency.
+			key := strings.ToLower(k[len(s3_constants.AmzUserMetaPrefix):])
+
+			// Sanitize key to prevent collisions and ensure Azure compliance
+			key = sanitizeMetadataKey(key)
+
+			val := string(v)
+			metadata[key] = &val
+		}
+	}
+	return metadata
+}
+
+func (az *azureRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
+		return nil
+	}
+	metadata := toMetadata(newEntry.Extended)
+
+	key := loc.Path[1:]
+	blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
+
+	if !reflect.DeepEqual(toMetadata(oldEntry.Extended), metadata) {
+		if _, err = blobClient.SetMetadata(context.Background(), metadata, nil); err != nil {
+			return err
+		}
+	}
+
+	if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
+		// SetHTTPHeaders replaces the whole header set, so carry the rest over
+		props, getErr := blobClient.GetProperties(context.Background(), nil)
+		if getErr != nil {
+			return fmt.Errorf("azure get properties %s%s: %w", loc.Bucket, loc.Path, getErr)
+		}
+		httpHeaders := blob.ParseHTTPHeaders(props)
+		httpHeaders.BlobContentEncoding = nil
+		if encoding != "" {
+			httpHeaders.BlobContentEncoding = &encoding
+		}
+		if _, err = blobClient.SetHTTPHeaders(context.Background(), httpHeaders, nil); err != nil {
+			return fmt.Errorf("azure set http headers %s%s: %w", loc.Bucket, loc.Path, err)
+		}
+	}
+
+	return
+}
+
+func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	key := loc.Path[1:]
+	blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key)
+
+	_, err = blobClient.Delete(context.Background(), &blob.DeleteOptions{
+		DeleteSnapshots: to.Ptr(blob.DeleteSnapshotsOptionTypeInclude),
+	})
+	if err != nil {
+		// Make delete idempotent - don't return error if blob doesn't exist
+		if bloberror.HasCode(err, bloberror.BlobNotFound) {
+			return nil
+		}
+		return fmt.Errorf("azure delete %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	pager := az.client.NewListContainersPager(nil)
+
+	for pager.More() {
+		resp, err := pager.NextPage(context.Background())
+		if err != nil {
+			return buckets, err
+		}
+
+		for _, containerItem := range resp.ContainerItems {
+			if containerItem.Name != nil {
+				bucket := &remote_storage.Bucket{
+					Name: *containerItem.Name,
+				}
+				if containerItem.Properties != nil && containerItem.Properties.LastModified != nil {
+					bucket.CreatedAt = *containerItem.Properties.LastModified
+				}
+				buckets = append(buckets, bucket)
+			}
+		}
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) {
+	containerClient := az.client.ServiceClient().NewContainerClient(name)
+	_, err = containerClient.Create(context.Background(), nil)
+	if err != nil {
+		return fmt.Errorf("create bucket %s: %w", name, err)
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) {
+	containerClient := az.client.ServiceClient().NewContainerClient(name)
+	_, err = containerClient.Delete(context.Background(), nil)
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %w", name, err)
+	}
+	return
+}

weed/remote_storage/azure/azure_storage_client_test.go

@@ -0,0 +1,432 @@
+package azure
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAzureStorageClientBasic tests basic Azure storage client operations
+func TestAzureStorageClientBasic(t *testing.T) {
+	// Skip if credentials not available
+	accountName := os.Getenv("AZURE_STORAGE_ACCOUNT")
+	accountKey := os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	testContainer := os.Getenv("AZURE_TEST_CONTAINER")
+
+	if accountName == "" || accountKey == "" {
+		t.Skip("Skipping Azure storage test: AZURE_STORAGE_ACCOUNT or AZURE_STORAGE_ACCESS_KEY not set")
+	}
+	if testContainer == "" {
+		testContainer = "seaweedfs-test"
+	}
+
+	// Create client
+	maker := azureRemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Name:             "test-azure",
+		AzureAccountName: accountName,
+		AzureAccountKey:  accountKey,
+	}
+
+	client, err := maker.Make(conf)
+	if err != nil {
+		t.Fatalf("Failed to create Azure client: %v", err)
+	}
+
+	azClient := client.(*azureRemoteStorageClient)
+
+	// Test 1: Create bucket/container
+	t.Run("CreateBucket", func(t *testing.T) {
+		err := azClient.CreateBucket(testContainer)
+		// Ignore error if bucket already exists
+		if err != nil && !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
+			t.Fatalf("Failed to create bucket: %v", err)
+		}
+	})
+
+	// Test 2: List buckets
+	t.Run("ListBuckets", func(t *testing.T) {
+		buckets, err := azClient.ListBuckets()
+		if err != nil {
+			t.Fatalf("Failed to list buckets: %v", err)
+		}
+		if len(buckets) == 0 {
+			t.Log("No buckets found (might be expected)")
+		} else {
+			t.Logf("Found %d buckets", len(buckets))
+		}
+	})
+
+	// Test 3: Write file
+	testContent := []byte("Hello from SeaweedFS Azure SDK migration test!")
+	testKey := fmt.Sprintf("/test-file-%d.txt", time.Now().Unix())
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test-azure",
+		Bucket: testContainer,
+		Path:   testKey,
+	}
+
+	t.Run("WriteFile", func(t *testing.T) {
+		entry := &filer_pb.Entry{
+			Attributes: &filer_pb.FuseAttributes{
+				Mtime: time.Now().Unix(),
+				Mime:  "text/plain",
+			},
+			Extended: map[string][]byte{
+				"x-amz-meta-test-key": []byte("test-value"),
+			},
+		}
+
+		reader := bytes.NewReader(testContent)
+		remoteEntry, err := azClient.WriteFile(loc, entry, reader)
+		if err != nil {
+			t.Fatalf("Failed to write file: %v", err)
+		}
+		if remoteEntry == nil {
+			t.Fatal("Remote entry is nil")
+		}
+		if remoteEntry.RemoteSize != int64(len(testContent)) {
+			t.Errorf("Expected size %d, got %d", len(testContent), remoteEntry.RemoteSize)
+		}
+	})
+
+	// Test 4: Read file
+	t.Run("ReadFile", func(t *testing.T) {
+		data, err := azClient.ReadFile(loc, 0, int64(len(testContent)))
+		if err != nil {
+			t.Fatalf("Failed to read file: %v", err)
+		}
+		if !bytes.Equal(data, testContent) {
+			t.Errorf("Content mismatch. Expected: %s, Got: %s", testContent, data)
+		}
+	})
+
+	// Test 5: Read partial file
+	t.Run("ReadPartialFile", func(t *testing.T) {
+		data, err := azClient.ReadFile(loc, 0, 5)
+		if err != nil {
+			t.Fatalf("Failed to read partial file: %v", err)
+		}
+		expected := testContent[:5]
+		if !bytes.Equal(data, expected) {
+			t.Errorf("Content mismatch. Expected: %s, Got: %s", expected, data)
+		}
+	})
+
+	// Test 6: Update metadata
+	t.Run("UpdateMetadata", func(t *testing.T) {
+		oldEntry := &filer_pb.Entry{
+			Extended: map[string][]byte{
+				"x-amz-meta-test-key": []byte("test-value"),
+			},
+		}
+		newEntry := &filer_pb.Entry{
+			Extended: map[string][]byte{
+				"x-amz-meta-test-key": []byte("test-value"),
+				"x-amz-meta-new-key":  []byte("new-value"),
+			},
+		}
+		err := azClient.UpdateFileMetadata(loc, oldEntry, newEntry)
+		if err != nil {
+			t.Fatalf("Failed to update metadata: %v", err)
+		}
+	})
+
+	// Test 7: Traverse (list objects)
+	t.Run("Traverse", func(t *testing.T) {
+		foundFile := false
+		err := azClient.Traverse(loc, func(dir string, name string, isDir bool, remoteEntry *filer_pb.RemoteEntry) error {
+			if !isDir && name == testKey[1:] { // Remove leading slash
+				foundFile = true
+			}
+			return nil
+		})
+		if err != nil {
+			t.Fatalf("Failed to traverse: %v", err)
+		}
+		if !foundFile {
+			t.Log("Test file not found in traverse (might be expected due to path matching)")
+		}
+	})
+
+	// Test 8: Delete file
+	t.Run("DeleteFile", func(t *testing.T) {
+		err := azClient.DeleteFile(loc)
+		if err != nil {
+			t.Fatalf("Failed to delete file: %v", err)
+		}
+	})
+
+	// Test 9: Verify file deleted (should fail)
+	t.Run("VerifyDeleted", func(t *testing.T) {
+		_, err := azClient.ReadFile(loc, 0, 10)
+		if !bloberror.HasCode(err, bloberror.BlobNotFound) {
+			t.Errorf("Expected BlobNotFound error, but got: %v", err)
+		}
+	})
+
+	// Clean up: Try to delete the test container
+	// Comment out if you want to keep the container
+	/*
+		t.Run("DeleteBucket", func(t *testing.T) {
+			err := azClient.DeleteBucket(testContainer)
+			if err != nil {
+				t.Logf("Warning: Failed to delete bucket: %v", err)
+			}
+		})
+	*/
+}
+
+// TestToMetadata tests the metadata conversion function
+func TestToMetadata(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    map[string][]byte
+		expected map[string]*string
+	}{
+		{
+			name: "basic metadata",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "key1": []byte("value1"),
+				s3_constants.AmzUserMetaPrefix + "key2": []byte("value2"),
+			},
+			expected: map[string]*string{
+				"key1": stringPtr("value1"),
+				"key2": stringPtr("value2"),
+			},
+		},
+		{
+			name: "metadata with dashes",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "content-type": []byte("text/plain"),
+			},
+			expected: map[string]*string{
+				"content_2d_type": stringPtr("text/plain"), // dash (0x2d) -> _2d_
+			},
+		},
+		{
+			name: "non-metadata keys ignored",
+			input: map[string][]byte{
+				"some-other-key": []byte("ignored"),
+				s3_constants.AmzUserMetaPrefix + "included": []byte("included"),
+			},
+			expected: map[string]*string{
+				"included": stringPtr("included"),
+			},
+		},
+		{
+			name: "keys starting with digits",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "123key":   []byte("value1"),
+				s3_constants.AmzUserMetaPrefix + "456-test": []byte("value2"),
+				s3_constants.AmzUserMetaPrefix + "789":      []byte("value3"),
+			},
+			expected: map[string]*string{
+				"_123key":      stringPtr("value1"), // starts with digit -> prefix _
+				"_456_2d_test": stringPtr("value2"), // starts with digit AND has dash
+				"_789":         stringPtr("value3"),
+			},
+		},
+		{
+			name: "uppercase and mixed case keys",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "My-Key":     []byte("value1"),
+				s3_constants.AmzUserMetaPrefix + "UPPERCASE":  []byte("value2"),
+				s3_constants.AmzUserMetaPrefix + "MiXeD-CaSe": []byte("value3"),
+			},
+			expected: map[string]*string{
+				"my_2d_key":     stringPtr("value1"), // lowercase + dash -> _2d_
+				"uppercase":     stringPtr("value2"),
+				"mixed_2d_case": stringPtr("value3"),
+			},
+		},
+		{
+			name: "keys with invalid characters",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "my.key":     []byte("value1"),
+				s3_constants.AmzUserMetaPrefix + "key+plus":   []byte("value2"),
+				s3_constants.AmzUserMetaPrefix + "key@symbol": []byte("value3"),
+				s3_constants.AmzUserMetaPrefix + "key-with.":  []byte("value4"),
+				s3_constants.AmzUserMetaPrefix + "key/slash":  []byte("value5"),
+			},
+			expected: map[string]*string{
+				"my_2e_key":       stringPtr("value1"), // dot (0x2e) -> _2e_
+				"key_2b_plus":     stringPtr("value2"), // plus (0x2b) -> _2b_
+				"key_40_symbol":   stringPtr("value3"), // @ (0x40) -> _40_
+				"key_2d_with_2e_": stringPtr("value4"), // dash and dot
+				"key_2f_slash":    stringPtr("value5"), // slash (0x2f) -> _2f_
+			},
+		},
+		{
+			name: "collision prevention",
+			input: map[string][]byte{
+				s3_constants.AmzUserMetaPrefix + "my-key": []byte("value1"),
+				s3_constants.AmzUserMetaPrefix + "my.key": []byte("value2"),
+				s3_constants.AmzUserMetaPrefix + "my_key": []byte("value3"),
+			},
+			expected: map[string]*string{
+				"my_2d_key": stringPtr("value1"), // dash (0x2d)
+				"my_2e_key": stringPtr("value2"), // dot (0x2e)
+				"my_key":    stringPtr("value3"), // underscore is valid, no encoding
+			},
+		},
+		{
+			name:     "empty input",
+			input:    map[string][]byte{},
+			expected: map[string]*string{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := toMetadata(tt.input)
+			if len(result) != len(tt.expected) {
+				t.Errorf("Expected %d keys, got %d", len(tt.expected), len(result))
+			}
+			for key, expectedVal := range tt.expected {
+				if resultVal, ok := result[key]; !ok {
+					t.Errorf("Expected key %s not found", key)
+				} else if resultVal == nil || expectedVal == nil {
+					if resultVal != expectedVal {
+						t.Errorf("For key %s: expected %v, got %v", key, expectedVal, resultVal)
+					}
+				} else if *resultVal != *expectedVal {
+					t.Errorf("For key %s: expected %s, got %s", key, *expectedVal, *resultVal)
+				}
+			}
+		})
+	}
+}
+
+func contains(s, substr string) bool {
+	return bytes.Contains([]byte(s), []byte(substr))
+}
+
+func stringPtr(s string) *string {
+	return &s
+}
+
+// Benchmark tests
+func BenchmarkToMetadata(b *testing.B) {
+	input := map[string][]byte{
+		"x-amz-meta-key1":         []byte("value1"),
+		"x-amz-meta-key2":         []byte("value2"),
+		"x-amz-meta-content-type": []byte("text/plain"),
+		"other-key":               []byte("ignored"),
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		toMetadata(input)
+	}
+}
+
+// Test that the maker implements the interface
+func TestAzureRemoteStorageMaker(t *testing.T) {
+	maker := azureRemoteStorageMaker{}
+
+	if !maker.HasBucket() {
+		t.Error("Expected HasBucket() to return true")
+	}
+
+	// Test with missing credentials - unset env vars (auto-restored by t.Setenv)
+	t.Setenv("AZURE_STORAGE_ACCOUNT", "")
+	t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
+
+	conf := &remote_pb.RemoteConf{
+		Name: "test",
+	}
+	_, err := maker.Make(conf)
+	if err == nil {
+		t.Error("Expected error with missing credentials")
+	}
+}
+
+func TestResolveAzureAccount(t *testing.T) {
+	t.Setenv("AZURE_STORAGE_ACCOUNT", "envaccount")
+	t.Setenv("AZURE_STORAGE_ACCESS_KEY", "ZW52a2V5")
+
+	accountName, accountKey := resolveAzureAccount(&remote_pb.RemoteConf{})
+	if accountName != "envaccount" || accountKey != "ZW52a2V5" {
+		t.Errorf("expected the environment account, got %q/%q", accountName, accountKey)
+	}
+
+	// a configured identity keeps a leftover environment key out of the way
+	accountName, accountKey = resolveAzureAccount(&remote_pb.RemoteConf{
+		AzureAccountName: "testaccount",
+		AzureClientId:    "11111111-1111-1111-1111-111111111111",
+	})
+	if accountName != "testaccount" {
+		t.Errorf("expected the configured account name, got %q", accountName)
+	}
+	if accountKey != "" {
+		t.Errorf("expected no account key alongside a client id, got %q", accountKey)
+	}
+}
+
+// An account name without a key authenticates with Entra ID
+func TestAzureRemoteStorageMakerWithoutAccountKey(t *testing.T) {
+	t.Setenv("AZURE_STORAGE_ACCOUNT", "")
+	t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
+
+	maker := azureRemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Name:             "test",
+		AzureAccountName: "testaccount",
+	}
+	client, err := maker.Make(conf)
+	if err != nil {
+		t.Fatalf("Failed to create client without an account key: %v", err)
+	}
+	if client.(*azureRemoteStorageClient).client == nil {
+		t.Error("Expected a blob client")
+	}
+}
+
+// Test error cases
+func TestAzureStorageClientErrors(t *testing.T) {
+	// Test with invalid credentials
+	maker := azureRemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Name:             "test",
+		AzureAccountName: "invalid",
+		AzureAccountKey:  "aW52YWxpZGtleQ==", // base64 encoded "invalidkey"
+	}
+
+	client, err := maker.Make(conf)
+	if err != nil {
+		t.Skip("Invalid credentials correctly rejected at client creation")
+	}
+
+	// If client creation succeeded, operations should fail
+	azClient := client.(*azureRemoteStorageClient)
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test",
+		Bucket: "nonexistent",
+		Path:   "/test.txt",
+	}
+
+	// These operations should fail with invalid credentials
+	_, err = azClient.ReadFile(loc, 0, 10)
+	if err == nil {
+		t.Log("Expected error with invalid credentials on ReadFile, but got none (might be cached)")
+	}
+}
+
+func TestAzureRemoteStorageClientImplementsInterface(t *testing.T) {
+	var _ remote_storage.RemoteStorageClient = (*azureRemoteStorageClient)(nil)
+}
+
+func TestAzureErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
+	require.Error(t, remote_storage.ErrRemoteObjectNotFound)
+	require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
+}

weed/remote_storage/gcs/gcs_storage_client.go

@@ -0,0 +1,421 @@
+package gcs
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"reflect"
+	"slices"
+	"strings"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+	"google.golang.org/protobuf/proto"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["gcs"] = new(gcsRemoteStorageMaker)
+}
+
+// defaultTokenURL is where the SDK sends the token request when the credentials
+// leave token_uri unset.
+const defaultTokenURL = "https://oauth2.googleapis.com/token"
+
+// StaticKeyCredentialTypes are the credential types that carry their own key
+// material. Every other type tells the SDK to fetch the token from a url, file
+// or executable named inside the credentials.
+var StaticKeyCredentialTypes = []string{
+	string(google.ServiceAccount),
+	string(google.AuthorizedUser),
+}
+
+// ParseInlineCredentials reports the credential type of an inline credentials
+// document and the token endpoint it makes the SDK dial.
+func ParseInlineCredentials(creds string) (credType string, tokenURL string, err error) {
+	var doc struct {
+		Type     string `json:"type"`
+		TokenURI string `json:"token_uri"`
+	}
+	if err := json.Unmarshal([]byte(creds), &doc); err != nil {
+		return "", "", fmt.Errorf("parse gcs credentials: %w", err)
+	}
+	if doc.TokenURI == "" {
+		return doc.Type, defaultTokenURL, nil
+	}
+	return doc.Type, doc.TokenURI, nil
+}
+
+type gcsRemoteStorageMaker struct{}
+
+func (s gcsRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}
+
+// MakeWithHTTPClient builds a gcs client whose token exchange and object reads
+// both go through the supplied *http.Client (or the SDK default when nil).
+// Callers that need to pin the dial path against DNS rebinding pass a client
+// whose transport has a guarded DialContext, mirroring the S3 backend. Callers
+// handling credentials they do not control pass the types they accept.
+func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client, allowedTypes ...string) (remote_storage.RemoteStorageClient, error) {
+	client := &gcsRemoteStorageClient{
+		conf: conf,
+	}
+
+	ctx := context.Background()
+	if httpClient != nil {
+		ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient)
+	}
+
+	googleApplicationCredentials := conf.GcsGoogleApplicationCredentials
+
+	if googleApplicationCredentials == "" {
+		if creds, found := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); found {
+			googleApplicationCredentials = creds
+		} else {
+			glog.Warningf("no GOOGLE_APPLICATION_CREDENTIALS env variable found, falling back to Application Default Credentials")
+		}
+	}
+
+	projectID := conf.GcsProjectId
+	if projectID == "" {
+		if pid, found := os.LookupEnv("GOOGLE_CLOUD_PROJECT"); found {
+			projectID = pid
+		} else {
+			glog.Warningf("need to specify GOOGLE_CLOUD_PROJECT env variable")
+		}
+	}
+
+	var clientOpts []option.ClientOption
+
+	if googleApplicationCredentials != "" {
+		googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials)
+		var data []byte
+		var err error
+		if strings.HasPrefix(googleApplicationCredentials, "{") {
+			data = []byte(googleApplicationCredentials)
+		} else {
+			data, err = os.ReadFile(googleApplicationCredentials)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read credentials file %s: %w", googleApplicationCredentials, err)
+			}
+		}
+		credType, _, parseErr := ParseInlineCredentials(string(data))
+		if parseErr != nil {
+			return nil, parseErr
+		}
+		if len(allowedTypes) > 0 && !slices.Contains(allowedTypes, credType) {
+			return nil, fmt.Errorf("gcs credential type %q is not accepted here", credType)
+		}
+		// Declaring the type keeps the SDK from reading the document as anything
+		// else; the untyped loader is deprecated for exactly that reason.
+		creds, err := google.CredentialsFromJSONWithType(ctx, data, google.CredentialsType(credType), storage.ScopeFullControl)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse credentials: %w", err)
+		}
+		clientOpts = append(clientOpts, option.WithHTTPClient(oauth2.NewClient(ctx, creds.TokenSource)), option.WithoutAuthentication())
+	}
+
+	c, err := storage.NewClient(ctx, clientOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create client: %w", err)
+	}
+
+	client.client = c
+	client.projectID = projectID
+	return client, nil
+}
+
+type gcsRemoteStorageClient struct {
+	conf      *remote_pb.RemoteConf
+	client    *storage.Client
+	projectID string
+}
+
+var _ = remote_storage.RemoteStorageClient(&gcsRemoteStorageClient{})
+
+func (gcs *gcsRemoteStorageClient) toRemoteEntry(attr *storage.ObjectAttrs) *filer_pb.RemoteEntry {
+	return &filer_pb.RemoteEntry{
+		StorageName:           gcs.conf.Name,
+		RemoteMtime:           attr.Updated.Unix(),
+		RemoteSize:            attr.Size,
+		RemoteETag:            attr.Etag,
+		RemoteContentEncoding: proto.String(attr.ContentEncoding),
+	}
+}
+
+func (gcs *gcsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	pathKey := loc.Path[1:]
+
+	objectIterator := gcs.client.Bucket(loc.Bucket).Objects(context.Background(), &storage.Query{
+		Delimiter: "",
+		Prefix:    pathKey,
+		Versions:  false,
+	})
+
+	var objectAttr *storage.ObjectAttrs
+	for err == nil {
+		objectAttr, err = objectIterator.Next()
+		if err != nil {
+			if err == iterator.Done {
+				return nil
+			}
+			return err
+		}
+
+		key := objectAttr.Name
+		key = "/" + key
+		dir, name := util.FullPath(key).DirAndName()
+		err = visitFn(dir, name, false, gcs.toRemoteEntry(objectAttr))
+	}
+	return
+}
+
+const defaultGCSOpTimeout = 30 * time.Second
+
+func (gcs *gcsRemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+	pathKey := loc.Path[1:]
+	if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
+		pathKey += "/"
+	}
+
+	objectIterator := gcs.client.Bucket(loc.Bucket).Objects(ctx, &storage.Query{
+		Delimiter: "/",
+		Prefix:    pathKey,
+		Versions:  false,
+	})
+
+	for {
+		objectAttr, iterErr := objectIterator.Next()
+		if iterErr != nil {
+			if iterErr == iterator.Done {
+				return nil
+			}
+			return fmt.Errorf("list directory %s%s: %w", loc.Bucket, loc.Path, iterErr)
+		}
+
+		if objectAttr.Prefix != "" {
+			// Common prefix → subdirectory
+			dirKey := "/" + strings.TrimSuffix(objectAttr.Prefix, "/")
+			dir, name := util.FullPath(dirKey).DirAndName()
+			if err = visitFn(dir, name, true, nil); err != nil {
+				return err
+			}
+		} else {
+			key := "/" + objectAttr.Name
+			if strings.HasSuffix(key, "/") {
+				continue // skip directory markers
+			}
+			dir, name := util.FullPath(key).DirAndName()
+			if err = visitFn(dir, name, false, gcs.toRemoteEntry(objectAttr)); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func (gcs *gcsRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
+	key := loc.Path[1:]
+	ctx, cancel := context.WithTimeout(context.Background(), defaultGCSOpTimeout)
+	defer cancel()
+	attr, err := gcs.client.Bucket(loc.Bucket).Object(key).Attrs(ctx)
+	if err != nil {
+		if errors.Is(err, storage.ErrObjectNotExist) {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("stat gcs %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	return gcs.toRemoteEntry(attr), nil
+}
+
+func (gcs *gcsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+
+	key := loc.Path[1:]
+	// read the stored bytes: decompressive transcoding of gzip-encoded objects
+	// breaks range reads and returns sizes that disagree with RemoteSize
+	rangeReader, readErr := gcs.client.Bucket(loc.Bucket).Object(key).ReadCompressed(true).NewRangeReader(context.Background(), offset, size)
+	if readErr != nil {
+		return nil, readErr
+	}
+	data, err = io.ReadAll(rangeReader)
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
+	key := loc.Path[1:]
+	reader, err = gcs.client.Bucket(loc.Bucket).Object(key).ReadCompressed(true).NewRangeReader(ctx, offset, size)
+	if err != nil {
+		if errors.Is(err, storage.ErrObjectNotExist) {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("failed to open stream for %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	return reader, nil
+}
+
+func (gcs *gcsRemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return nil
+}
+
+func (gcs *gcsRemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	// the trailing slash keeps sibling prefixes that share the name intact
+	prefix := loc.Path[1:]
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	if prefix == "" {
+		// the mount root maps to the whole bucket; wiping every object from a
+		// single namespace event is too destructive, so keep them
+		glog.Warningf("gcs %s: skip removing directory mapped to the bucket root", loc.Bucket)
+		return nil
+	}
+
+	bucket := gcs.client.Bucket(loc.Bucket)
+	objectIterator := bucket.Objects(context.Background(), &storage.Query{
+		Prefix: prefix,
+	})
+	for {
+		objectAttr, iterErr := objectIterator.Next()
+		if iterErr == iterator.Done {
+			return nil
+		}
+		if iterErr != nil {
+			return fmt.Errorf("gcs list %s/%s: %w", loc.Bucket, prefix, iterErr)
+		}
+		if delErr := bucket.Object(objectAttr.Name).Delete(context.Background()); delErr != nil && !errors.Is(delErr, storage.ErrObjectNotExist) {
+			return fmt.Errorf("gcs delete %s/%s: %w", loc.Bucket, objectAttr.Name, delErr)
+		}
+	}
+}
+
+func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	key := loc.Path[1:]
+
+	metadata := toMetadata(entry.Extended)
+	wc := gcs.client.Bucket(loc.Bucket).Object(key).NewWriter(context.Background())
+	wc.Metadata = metadata
+	if entry.Attributes != nil && entry.Attributes.Mime != "" {
+		wc.ContentType = entry.Attributes.Mime
+	}
+	wc.ContentEncoding = remote_storage.EntryContentEncoding(entry)
+	if _, err = io.Copy(wc, reader); err != nil {
+		return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
+	}
+	if err = wc.Close(); err != nil {
+		return nil, fmt.Errorf("close gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
+	}
+
+	// read back the remote entry
+	return gcs.readFileRemoteEntry(loc)
+
+}
+
+func (gcs *gcsRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	return gcs.StatFile(loc)
+}
+
+func toMetadata(attributes map[string][]byte) map[string]string {
+	metadata := make(map[string]string)
+	for k, v := range attributes {
+		if strings.HasPrefix(k, "X-") {
+			continue
+		}
+		metadata[k] = string(v)
+	}
+	return metadata
+}
+
+func (gcs *gcsRemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
+		return nil
+	}
+	attrsToUpdate := storage.ObjectAttrsToUpdate{}
+	if metadata := toMetadata(newEntry.Extended); len(metadata) > 0 {
+		attrsToUpdate.Metadata = metadata
+	} else {
+		// no way to delete the metadata yet
+	}
+	if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
+		attrsToUpdate.ContentEncoding = encoding // empty clears the header
+	}
+
+	if attrsToUpdate.Metadata == nil && attrsToUpdate.ContentEncoding == nil {
+		return nil
+	}
+	key := loc.Path[1:]
+	_, err = gcs.client.Bucket(loc.Bucket).Object(key).Update(context.Background(), attrsToUpdate)
+	return
+}
+func (gcs *gcsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	key := loc.Path[1:]
+	if err = gcs.client.Bucket(loc.Bucket).Object(key).Delete(context.Background()); err != nil {
+		if errors.Is(err, storage.ErrObjectNotExist) {
+			return remote_storage.ErrRemoteObjectNotFound
+		}
+		return fmt.Errorf("gcs delete %s%s: %v", loc.Bucket, key, err)
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	if gcs.projectID == "" {
+		return nil, fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	iter := gcs.client.Buckets(context.Background(), gcs.projectID)
+	for {
+		b, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return buckets, err
+		}
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      b.Name,
+			CreatedAt: b.Created,
+		})
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) CreateBucket(name string) (err error) {
+	if gcs.projectID == "" {
+		return fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	err = gcs.client.Bucket(name).Create(context.Background(), gcs.projectID, &storage.BucketAttrs{})
+	if err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) DeleteBucket(name string) (err error) {
+	err = gcs.client.Bucket(name).Delete(context.Background())
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/gcs/gcs_storage_client_test.go

@@ -0,0 +1,46 @@
+package gcs
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGCSRemoteStorageClientImplementsInterface(t *testing.T) {
+	var _ remote_storage.RemoteStorageClient = (*gcsRemoteStorageClient)(nil)
+}
+
+func TestGCSErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
+	require.Error(t, remote_storage.ErrRemoteObjectNotFound)
+	require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
+}
+
+// TestMakeWithHTTPClientAllowedTypes covers the restriction a caller applies to
+// credentials it does not control: a federated document never reaches the SDK,
+// while an unrestricted caller keeps loading whatever the operator configured.
+func TestMakeWithHTTPClientAllowedTypes(t *testing.T) {
+	federated := `{"type":"external_account","audience":"a","subject_token_type":"t","token_url":"http://127.0.0.1:9/v1/token","credential_source":{"url":"http://169.254.169.254/"}}`
+	conf := &remote_pb.RemoteConf{Type: "gcs", GcsGoogleApplicationCredentials: federated}
+
+	_, err := MakeWithHTTPClient(conf, nil, StaticKeyCredentialTypes...)
+	require.ErrorContains(t, err, `"external_account" is not accepted`)
+
+	_, err = MakeWithHTTPClient(conf, nil)
+	require.NoError(t, err)
+}
+
+func TestParseInlineCredentials(t *testing.T) {
+	credType, tokenURL, err := ParseInlineCredentials(`{"type":"service_account"}`)
+	require.NoError(t, err)
+	require.Equal(t, "service_account", credType)
+	require.Equal(t, defaultTokenURL, tokenURL)
+
+	_, tokenURL, err = ParseInlineCredentials(`{"type":"service_account","token_uri":"https://example.com/t"}`)
+	require.NoError(t, err)
+	require.Equal(t, "https://example.com/t", tokenURL)
+
+	_, _, err = ParseInlineCredentials(`not json`)
+	require.Error(t, err)
+}

weed/remote_storage/remote_storage.go

@@ -0,0 +1,196 @@
+package remote_storage
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+const slash = "/"
+
+func ParseLocationName(remote string) (locationName string) {
+	remote = strings.TrimSuffix(remote, slash)
+	parts := strings.SplitN(remote, slash, 2)
+	if len(parts) >= 1 {
+		return parts[0]
+	}
+	return
+}
+
+func parseBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+	loc = &remote_pb.RemoteStorageLocation{}
+	remote = strings.TrimSuffix(remote, slash)
+	parts := strings.SplitN(remote, slash, 3)
+	if len(parts) >= 1 {
+		loc.Name = parts[0]
+	}
+	if len(parts) >= 2 {
+		loc.Bucket = parts[1]
+	}
+	loc.Path = remote[len(loc.Name)+1+len(loc.Bucket):]
+	if loc.Path == "" {
+		loc.Path = slash
+	}
+	return
+}
+
+func parseNoBucketLocation(remote string) (loc *remote_pb.RemoteStorageLocation) {
+	loc = &remote_pb.RemoteStorageLocation{}
+	remote = strings.TrimSuffix(remote, slash)
+	parts := strings.SplitN(remote, slash, 2)
+	if len(parts) >= 1 {
+		loc.Name = parts[0]
+	}
+	loc.Path = remote[len(loc.Name):]
+	if loc.Path == "" {
+		loc.Path = slash
+	}
+	return
+}
+
+func FormatLocation(loc *remote_pb.RemoteStorageLocation) string {
+	if loc.Bucket == "" {
+		return fmt.Sprintf("%s%s", loc.Name, loc.Path)
+	}
+	return fmt.Sprintf("%s/%s%s", loc.Name, loc.Bucket, loc.Path)
+}
+
+type VisitFunc func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error
+
+// EntryContentEncoding returns the Content-Encoding stored in the entry
+// extended attributes, for clients to set on uploaded remote objects.
+func EntryContentEncoding(entry *filer_pb.Entry) string {
+	if entry == nil {
+		return ""
+	}
+	return string(entry.Extended["Content-Encoding"])
+}
+
+type Bucket struct {
+	Name      string
+	CreatedAt time.Time
+}
+
+// ErrRemoteObjectNotFound is returned by StatFile when the object does not exist in the remote storage backend.
+var ErrRemoteObjectNotFound = errors.New("remote object not found")
+
+type RemoteStorageClient interface {
+	Traverse(loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error
+	ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn VisitFunc) error
+	StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error)
+	ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error)
+	WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error)
+	RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error)
+	WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error)
+	UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
+	DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error)
+	ListBuckets() ([]*Bucket, error)
+	CreateBucket(name string) (err error)
+	DeleteBucket(name string) (err error)
+}
+
+// RemoteStorageConcurrentReader is an optional interface for remote storage clients
+// that support configurable download concurrency for multipart downloads.
+type RemoteStorageConcurrentReader interface {
+	ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error)
+}
+
+// RemoteStorageStreamReader is an optional interface for remote storage clients
+// that support streaming reads with io.Reader for efficient memory usage.
+type RemoteStorageStreamReader interface {
+	ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error)
+}
+
+// CacheWaitTimeout is how long a read of an uncached remote-only object waits
+// for the local cache before serving another way: small files wait longer since
+// their cache completes quickly, large files fail fast for better TTFB. A mount
+// carrying cache_wait_ms replaces the size tiers, and 0 means never wait.
+func CacheWaitTimeout(remoteSize int64, mountedLocation *remote_pb.RemoteStorageLocation) time.Duration {
+	if mountedLocation != nil && mountedLocation.CacheWaitMs != nil {
+		return max(0, time.Duration(*mountedLocation.CacheWaitMs)*time.Millisecond)
+	}
+	switch {
+	case remoteSize > 500*1024*1024:
+		return 2 * time.Second
+	case remoteSize > 0 && remoteSize < 50*1024*1024:
+		return 10 * time.Second
+	default:
+		return 5 * time.Second
+	}
+}
+
+type RemoteStorageClientMaker interface {
+	Make(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error)
+	HasBucket() bool
+}
+
+type CachedRemoteStorageClient struct {
+	*remote_pb.RemoteConf
+	RemoteStorageClient
+}
+
+var (
+	RemoteStorageClientMakers = make(map[string]RemoteStorageClientMaker)
+	remoteStorageClients      = make(map[string]CachedRemoteStorageClient)
+	remoteStorageClientsLock  sync.Mutex
+)
+
+func GetAllRemoteStorageNames() string {
+	var storageNames []string
+	for k := range RemoteStorageClientMakers {
+		storageNames = append(storageNames, k)
+	}
+	sort.Strings(storageNames)
+	return strings.Join(storageNames, "|")
+}
+
+func ParseRemoteLocation(remoteConfType string, remote string) (remoteStorageLocation *remote_pb.RemoteStorageLocation, err error) {
+	maker, found := RemoteStorageClientMakers[remoteConfType]
+	if !found {
+		return nil, fmt.Errorf("remote storage type %s not found", remoteConfType)
+	}
+
+	if !maker.HasBucket() {
+		return parseNoBucketLocation(remote), nil
+	}
+	return parseBucketLocation(remote), nil
+}
+
+func makeRemoteStorageClient(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
+	maker, found := RemoteStorageClientMakers[remoteConf.Type]
+	if !found {
+		return nil, fmt.Errorf("remote storage type %s not found", remoteConf.Type)
+	}
+	return maker.Make(remoteConf)
+}
+
+func GetRemoteStorage(remoteConf *remote_pb.RemoteConf) (RemoteStorageClient, error) {
+	remoteStorageClientsLock.Lock()
+	defer remoteStorageClientsLock.Unlock()
+
+	existingRemoteStorageClient, found := remoteStorageClients[remoteConf.Name]
+	if found && proto.Equal(existingRemoteStorageClient.RemoteConf, remoteConf) {
+		return existingRemoteStorageClient.RemoteStorageClient, nil
+	}
+
+	newRemoteStorageClient, err := makeRemoteStorageClient(remoteConf)
+	if err != nil {
+		return nil, fmt.Errorf("make remote storage client %s: %v", remoteConf.Name, err)
+	}
+
+	remoteStorageClients[remoteConf.Name] = CachedRemoteStorageClient{
+		RemoteConf:          remoteConf,
+		RemoteStorageClient: newRemoteStorageClient,
+	}
+
+	return newRemoteStorageClient, nil
+}

weed/remote_storage/remote_storage_test.go

@@ -0,0 +1,54 @@
+package remote_storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"google.golang.org/protobuf/proto"
+)
+
+func TestCacheWaitTimeout(t *testing.T) {
+	tests := []struct {
+		name            string
+		remoteSize      int64
+		mountedLocation *remote_pb.RemoteStorageLocation
+		want            time.Duration
+	}{
+		{name: "large file fails fast", remoteSize: 600 * 1024 * 1024, want: 2 * time.Second},
+		{name: "small file waits longer", remoteSize: 1024, want: 10 * time.Second},
+		{name: "medium file", remoteSize: 100 * 1024 * 1024, want: 5 * time.Second},
+		{name: "unknown size", want: 5 * time.Second},
+		{
+			name:            "mount replaces the size tier",
+			remoteSize:      600 * 1024 * 1024,
+			mountedLocation: &remote_pb.RemoteStorageLocation{CacheWaitMs: proto.Int32(30000)},
+			want:            30 * time.Second,
+		},
+		{
+			name:            "mount opts out of caching",
+			remoteSize:      1024,
+			mountedLocation: &remote_pb.RemoteStorageLocation{CacheWaitMs: proto.Int32(0)},
+			want:            0,
+		},
+		{
+			name:            "negative wait never waits",
+			remoteSize:      1024,
+			mountedLocation: &remote_pb.RemoteStorageLocation{CacheWaitMs: proto.Int32(-1)},
+			want:            0,
+		},
+		{
+			name:            "unset wait keeps the size tier",
+			remoteSize:      600 * 1024 * 1024,
+			mountedLocation: &remote_pb.RemoteStorageLocation{},
+			want:            2 * time.Second,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := CacheWaitTimeout(tt.remoteSize, tt.mountedLocation); got != tt.want {
+				t.Errorf("CacheWaitTimeout(%d) = %v, want %v", tt.remoteSize, got, tt.want)
+			}
+		})
+	}
+}

weed/remote_storage/s3/aliyun.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["aliyun"] = new(AliyunRemoteStorageMaker)
+}
+
+type AliyunRemoteStorageMaker struct{}
+
+func (s AliyunRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/backblaze.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["b2"] = new(BackBlazeRemoteStorageMaker)
+}
+
+type BackBlazeRemoteStorageMaker struct{}
+
+func (s BackBlazeRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/baidu.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["baidu"] = new(BaiduRemoteStorageMaker)
+}
+
+type BaiduRemoteStorageMaker struct{}
+
+func (s BaiduRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/contabo.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["contabo"] = new(ContaboRemoteStorageMaker)
+}
+
+type ContaboRemoteStorageMaker struct{}
+
+func (s ContaboRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s ContaboRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/endpoint_test.go

@@ -0,0 +1,44 @@
+package s3
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+)
+
+// TestS3CompatibleEndpointCoversAllProviders locks the type-to-endpoint mapping
+// the volume server's SSRF guard relies on. Every S3-SDK-backed provider dials
+// a caller-supplied endpoint, so each must surface it here; a new sibling that
+// forgets to would silently bypass the guard.
+func TestS3CompatibleEndpointCoversAllProviders(t *testing.T) {
+	cases := []struct {
+		conf *remote_pb.RemoteConf
+		want string
+	}{
+		{&remote_pb.RemoteConf{Type: "s3", S3Endpoint: "http://s3.internal"}, "http://s3.internal"},
+		{&remote_pb.RemoteConf{Type: "wasabi", WasabiEndpoint: "http://wasabi.internal"}, "http://wasabi.internal"},
+		{&remote_pb.RemoteConf{Type: "b2", BackblazeEndpoint: "http://b2.internal"}, "http://b2.internal"},
+		{&remote_pb.RemoteConf{Type: "aliyun", AliyunEndpoint: "http://aliyun.internal"}, "http://aliyun.internal"},
+		{&remote_pb.RemoteConf{Type: "tencent", TencentEndpoint: "http://tencent.internal"}, "http://tencent.internal"},
+		{&remote_pb.RemoteConf{Type: "baidu", BaiduEndpoint: "http://baidu.internal"}, "http://baidu.internal"},
+		{&remote_pb.RemoteConf{Type: "filebase", FilebaseEndpoint: "http://filebase.internal"}, "http://filebase.internal"},
+		{&remote_pb.RemoteConf{Type: "storj", StorjEndpoint: "http://storj.internal"}, "http://storj.internal"},
+		{&remote_pb.RemoteConf{Type: "contabo", ContaboEndpoint: "http://contabo.internal"}, "http://contabo.internal"},
+	}
+	for _, tc := range cases {
+		endpoint, ok := S3CompatibleEndpoint(tc.conf)
+		if !ok {
+			t.Errorf("type %q: expected an S3-compatible endpoint, got ok=false", tc.conf.Type)
+			continue
+		}
+		if endpoint != tc.want {
+			t.Errorf("type %q: endpoint = %q, want %q", tc.conf.Type, endpoint, tc.want)
+		}
+	}
+
+	// Every registered maker must be an S3-compatible type; the guard treats
+	// anything else (gcs, azure, ...) as not dialing a caller-supplied URL.
+	if _, ok := S3CompatibleEndpoint(&remote_pb.RemoteConf{Type: "gcs"}); ok {
+		t.Error("gcs must not be reported as an S3-compatible endpoint")
+	}
+}

weed/remote_storage/s3/filebase.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["filebase"] = new(FilebaseRemoteStorageMaker)
+}
+
+type FilebaseRemoteStorageMaker struct{}
+
+func (s FilebaseRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s FilebaseRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/s3_storage_client.go

@@ -0,0 +1,703 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["s3"] = new(s3RemoteStorageMaker)
+}
+
+type s3RemoteStorageMaker struct{}
+
+func (s s3RemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}
+
+// s3CompatibleOptions carries the per-provider AWS SDK knobs the shared client
+// builder needs. Every S3-SDK-backed provider (s3 plus the wasabi/b2/... family)
+// converges on the same s3RemoteStorageClient and dials a caller-supplied
+// endpoint, differing only in which RemoteConf fields it reads.
+type s3CompatibleOptions struct {
+	name           string
+	endpoint       string
+	region         string
+	accessKey      string
+	secretKey      string
+	forcePathStyle bool
+	signV4         bool
+	// anonymousWhenNoCreds disables SigV4 signing for public buckets when no
+	// credentials are supplied. Only the generic "s3" provider does this.
+	anonymousWhenNoCreds bool
+	// setUserAgent adds the SeaweedFS User-Agent header (generic "s3" only).
+	setUserAgent bool
+}
+
+// s3CompatibleClientOptions returns the client knobs for an S3-SDK-backed
+// RemoteConf and whether conf is such a type. Keeping the type-to-fields
+// mapping in one place lets the client builder and the SSRF endpoint guard
+// agree on exactly which endpoint each provider dials.
+func s3CompatibleClientOptions(conf *remote_pb.RemoteConf) (s3CompatibleOptions, bool) {
+	switch conf.Type {
+	case "s3":
+		return s3CompatibleOptions{
+			name:                 "s3",
+			endpoint:             conf.S3Endpoint,
+			region:               conf.S3Region,
+			accessKey:            conf.S3AccessKey,
+			secretKey:            conf.S3SecretKey,
+			forcePathStyle:       conf.S3ForcePathStyle,
+			signV4:               conf.S3V4Signature,
+			anonymousWhenNoCreds: true,
+			setUserAgent:         true,
+		}, true
+	case "aliyun":
+		return s3CompatibleOptions{
+			name:      "aliyun",
+			endpoint:  conf.AliyunEndpoint,
+			region:    conf.AliyunRegion,
+			accessKey: util.Nvl(conf.AliyunAccessKey, os.Getenv("ALICLOUD_ACCESS_KEY_ID")),
+			secretKey: util.Nvl(conf.AliyunSecretKey, os.Getenv("ALICLOUD_ACCESS_KEY_SECRET")),
+		}, true
+	case "b2":
+		return s3CompatibleOptions{
+			name:           "backblaze",
+			endpoint:       conf.BackblazeEndpoint,
+			region:         conf.BackblazeRegion,
+			accessKey:      conf.BackblazeKeyId,
+			secretKey:      conf.BackblazeApplicationKey,
+			forcePathStyle: true,
+		}, true
+	case "baidu":
+		return s3CompatibleOptions{
+			name:      "baidu",
+			endpoint:  conf.BaiduEndpoint,
+			region:    conf.BaiduRegion,
+			accessKey: util.Nvl(conf.BaiduAccessKey, os.Getenv("BDCLOUD_ACCESS_KEY")),
+			secretKey: util.Nvl(conf.BaiduSecretKey, os.Getenv("BDCLOUD_SECRET_KEY")),
+			signV4:    true,
+		}, true
+	case "contabo":
+		return s3CompatibleOptions{
+			name:           "contabo",
+			endpoint:       conf.ContaboEndpoint,
+			region:         conf.ContaboRegion,
+			accessKey:      util.Nvl(conf.ContaboAccessKey, os.Getenv("ACCESS_KEY")),
+			secretKey:      util.Nvl(conf.ContaboSecretKey, os.Getenv("SECRET_KEY")),
+			forcePathStyle: true,
+		}, true
+	case "filebase":
+		return s3CompatibleOptions{
+			name:           "filebase",
+			endpoint:       conf.FilebaseEndpoint,
+			region:         "us-east-1",
+			accessKey:      util.Nvl(conf.FilebaseAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")),
+			secretKey:      util.Nvl(conf.FilebaseSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")),
+			forcePathStyle: true,
+			signV4:         true,
+		}, true
+	case "storj":
+		return s3CompatibleOptions{
+			name:           "storj",
+			endpoint:       conf.StorjEndpoint,
+			region:         "us-west-2",
+			accessKey:      util.Nvl(conf.StorjAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")),
+			secretKey:      util.Nvl(conf.StorjSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")),
+			forcePathStyle: true,
+		}, true
+	case "tencent":
+		return s3CompatibleOptions{
+			name:           "tencent",
+			endpoint:       conf.TencentEndpoint,
+			region:         "us-west-2",
+			accessKey:      util.Nvl(conf.TencentSecretId, os.Getenv("COS_SECRETID")),
+			secretKey:      util.Nvl(conf.TencentSecretKey, os.Getenv("COS_SECRETKEY")),
+			forcePathStyle: true,
+		}, true
+	case "wasabi":
+		return s3CompatibleOptions{
+			name:           "wasabi",
+			endpoint:       conf.WasabiEndpoint,
+			region:         conf.WasabiRegion,
+			accessKey:      conf.WasabiAccessKey,
+			secretKey:      conf.WasabiSecretKey,
+			forcePathStyle: true,
+		}, true
+	}
+	return s3CompatibleOptions{}, false
+}
+
+// S3CompatibleEndpoint returns the endpoint an S3-SDK-backed RemoteConf would
+// dial directly, and whether conf is such a type. The volume server validates
+// this endpoint against the SSRF deny-list for every S3-compatible provider,
+// not just the generic "s3" type.
+func S3CompatibleEndpoint(conf *remote_pb.RemoteConf) (string, bool) {
+	opt, ok := s3CompatibleClientOptions(conf)
+	if !ok {
+		return "", false
+	}
+	return opt.endpoint, true
+}
+
+// MakeWithHTTPClient builds the client for any S3-SDK-backed remote storage
+// type using the supplied *http.Client (or the AWS SDK default when nil).
+// Callers that need to pin the dial path against DNS rebinding can pass a
+// client whose transport has a guarded DialContext.
+func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
+	opt, ok := s3CompatibleClientOptions(conf)
+	if !ok {
+		return nil, fmt.Errorf("%q is not an S3-compatible remote storage type", conf.Type)
+	}
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	config := &aws.Config{
+		Region:                        aws.String(opt.region),
+		Endpoint:                      aws.String(opt.endpoint),
+		S3ForcePathStyle:              aws.Bool(opt.forcePathStyle),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if httpClient != nil {
+		config.HTTPClient = httpClient
+	}
+	if opt.accessKey != "" && opt.secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(opt.accessKey, opt.secretKey, "")
+	} else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" {
+		// Explicitly disable signing for public buckets.
+		config.Credentials = credentials.AnonymousCredentials
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create %s session: %w", opt.name, err)
+	}
+	if opt.signV4 {
+		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	}
+	if opt.setUserAgent {
+		sess.Handlers.Build.PushBack(func(r *request.Request) {
+			r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
+		})
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}
+
+var skipSha256PayloadSigning = func(r *request.Request) {
+	// see https://github.com/ceph/ceph/pull/15965/files
+	if r.ClientInfo.ServiceID != "S3" {
+		return
+	}
+	if r.Operation.Name == "PutObject" || r.Operation.Name == "UploadPart" {
+		if len(r.HTTPRequest.Header.Get("X-Amz-Content-Sha256")) == 0 {
+			r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
+		}
+	}
+}
+
+type s3RemoteStorageClient struct {
+	conf *remote_pb.RemoteConf
+	conn s3iface.S3API
+}
+
+const (
+	defaultUploadConcurrency = 1
+	defaultReadConcurrency   = 5
+)
+
+func (s *s3RemoteStorageClient) uploadConcurrency() int {
+	if n := int(s.conf.GetUploadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultUploadConcurrency
+}
+
+func (s *s3RemoteStorageClient) downloadConcurrency() int {
+	if n := int(s.conf.GetDownloadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultReadConcurrency
+}
+
+var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{})
+
+func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	pathKey := remote.Path[1:]
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket:              aws.String(remote.Bucket),
+		ContinuationToken:   nil,
+		Delimiter:           nil, // not aws.String("/"), iterate through all entries
+		EncodingType:        nil,
+		ExpectedBucketOwner: nil,
+		FetchOwner:          nil,
+		MaxKeys:             nil, // aws.Int64(1000),
+		Prefix:              aws.String(pathKey),
+		RequestPayer:        nil,
+		StartAfter:          nil,
+	}
+	isLastPage := false
+	for !isLastPage && err == nil {
+		var localErr error
+		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, content := range page.Contents {
+				key := *content.Key
+				key = "/" + key
+				dir, name := util.FullPath(key).DirAndName()
+				remoteEntry := &filer_pb.RemoteEntry{
+					StorageName: s.conf.Name,
+				}
+				if content.LastModified != nil {
+					remoteEntry.RemoteMtime = content.LastModified.Unix()
+				}
+				if content.Size != nil {
+					remoteEntry.RemoteSize = *content.Size
+				}
+				if content.ETag != nil {
+					remoteEntry.RemoteETag = *content.ETag
+				}
+				if err := visitFn(dir, name, false, remoteEntry); err != nil {
+					localErr = err
+					return false
+				}
+			}
+			listInput.ContinuationToken = page.NextContinuationToken
+			isLastPage = lastPage
+			return true
+		})
+		if listErr != nil {
+			err = fmt.Errorf("list %v: %w", remote, listErr)
+		}
+		if localErr != nil {
+			err = fmt.Errorf("process %v: %w", remote, localErr)
+		}
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
+	pathKey := loc.Path[1:]
+	if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
+		pathKey += "/"
+	}
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket:    aws.String(loc.Bucket),
+		Prefix:    aws.String(pathKey),
+		Delimiter: aws.String("/"),
+	}
+
+	var localErr error
+	listErr := s.conn.ListObjectsV2PagesWithContext(ctx, listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, prefix := range page.CommonPrefixes {
+			if prefix.Prefix == nil {
+				continue
+			}
+			dirKey := "/" + strings.TrimSuffix(*prefix.Prefix, "/")
+			dir, name := util.FullPath(dirKey).DirAndName()
+			if err := visitFn(dir, name, true, nil); err != nil {
+				localErr = err
+				return false
+			}
+		}
+		for _, content := range page.Contents {
+			key := "/" + *content.Key
+			if strings.HasSuffix(key, "/") {
+				continue // skip directory markers
+			}
+			dir, name := util.FullPath(key).DirAndName()
+			remoteEntry := &filer_pb.RemoteEntry{
+				StorageName: s.conf.Name,
+			}
+			if content.LastModified != nil {
+				remoteEntry.RemoteMtime = content.LastModified.Unix()
+			}
+			if content.Size != nil {
+				remoteEntry.RemoteSize = *content.Size
+			}
+			if content.ETag != nil {
+				remoteEntry.RemoteETag = *content.ETag
+			}
+			if err := visitFn(dir, name, false, remoteEntry); err != nil {
+				localErr = err
+				return false
+			}
+		}
+		return true
+	})
+	if listErr != nil {
+		return fmt.Errorf("list directory %v: %w", loc, listErr)
+	}
+	if localErr != nil {
+		return fmt.Errorf("process directory %v: %w", loc, localErr)
+	}
+	return nil
+}
+
+func (s *s3RemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
+	resp, err := s.conn.HeadObject(&s3.HeadObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+	})
+	if err != nil {
+		if reqErr, ok := err.(awserr.RequestFailure); ok && reqErr.StatusCode() == http.StatusNotFound {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("stat s3 %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	remoteEntry = &filer_pb.RemoteEntry{
+		StorageName: s.conf.Name,
+	}
+	if resp.ContentLength != nil {
+		remoteEntry.RemoteSize = *resp.ContentLength
+	}
+	if resp.LastModified != nil {
+		remoteEntry.RemoteMtime = resp.LastModified.Unix()
+	}
+	if resp.ETag != nil {
+		remoteEntry.RemoteETag = *resp.ETag
+	}
+	// a HeadObject response is authoritative: no header means no encoding
+	remoteEntry.RemoteContentEncoding = aws.String(aws.StringValue(resp.ContentEncoding))
+	return remoteEntry, nil
+}
+
+func (s *s3RemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+	return s.ReadFileWithConcurrency(loc, offset, size, 0)
+}
+
+func (s *s3RemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error) {
+	if concurrency <= 0 {
+		concurrency = s.downloadConcurrency()
+	}
+	downloader := s3manager.NewDownloaderWithClient(s.conn, func(u *s3manager.Downloader) {
+		u.PartSize = int64(4 * 1024 * 1024)
+		u.Concurrency = concurrency
+	})
+
+	dataSlice := make([]byte, int(size))
+	writerAt := aws.NewWriteAtBuffer(dataSlice)
+
+	n, err := downloader.Download(writerAt, &s3.GetObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	// The buffer is pre-sized to size, so a short read leaves the tail
+	// zero-padded and would be cached as valid-looking but corrupt content.
+	// Reject it instead.
+	if n != size {
+		return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size)
+	}
+
+	return writerAt.Bytes(), nil
+}
+
+func (s *s3RemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
+	output, err := s.conn.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchKey {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("failed to open stream for %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	return output.Body, nil
+}
+
+func (s *s3RemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return nil
+}
+
+func (s *s3RemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	// the trailing slash keeps sibling prefixes that share the name intact
+	prefix := loc.Path[1:]
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	if prefix == "" {
+		// the mount root maps to the whole bucket; wiping every object from a
+		// single namespace event is too destructive, so keep them
+		glog.Warningf("s3 %s: skip removing directory mapped to the bucket root", loc.Bucket)
+		return nil
+	}
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket: aws.String(loc.Bucket),
+		Prefix: aws.String(prefix),
+	}
+	var deleteErr error
+	listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		var objects []*s3.ObjectIdentifier
+		for _, content := range page.Contents {
+			objects = append(objects, &s3.ObjectIdentifier{Key: content.Key})
+		}
+		if len(objects) == 0 {
+			return true
+		}
+		// a listing page holds at most 1000 keys, the DeleteObjects limit
+		resp, batchErr := s.conn.DeleteObjects(&s3.DeleteObjectsInput{
+			Bucket: aws.String(loc.Bucket),
+			Delete: &s3.Delete{
+				Objects: objects,
+				Quiet:   aws.Bool(true),
+			},
+		})
+		if batchErr != nil {
+			deleteErr = batchErr
+			return false
+		}
+		if len(resp.Errors) > 0 {
+			// a batch can fail 1000 keys; report the scope, not every key
+			failed := resp.Errors[0]
+			deleteErr = fmt.Errorf("%d keys failed, first is %s: %s %s", len(resp.Errors), aws.StringValue(failed.Key), aws.StringValue(failed.Code), aws.StringValue(failed.Message))
+			return false
+		}
+		return true
+	})
+	if listErr != nil {
+		return fmt.Errorf("list %s/%s: %w", loc.Bucket, prefix, listErr)
+	}
+	if deleteErr != nil {
+		return fmt.Errorf("remove directory %s/%s: %w", loc.Bucket, prefix, deleteErr)
+	}
+	return nil
+}
+
+func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	fileSize := int64(filer.FileSize(entry))
+
+	partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB
+	for partSize*1000 < fileSize {
+		partSize *= 4
+	}
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
+		u.PartSize = partSize
+		u.Concurrency = s.uploadConcurrency()
+	})
+
+	// process tagging
+	tags := ""
+	var awsTags *string
+	// openstack swift doesn't support s3 object tagging
+	if s.conf.S3SupportTagging {
+		for k, v := range entry.Extended {
+			if len(tags) > 0 {
+				tags = tags + "&"
+			}
+			tags = tags + k + "=" + string(v)
+		}
+		awsTags = aws.String(tags)
+	}
+
+	// Upload the file to S3.
+	uploadInput := &s3manager.UploadInput{
+		Bucket:  aws.String(loc.Bucket),
+		Key:     aws.String(loc.Path[1:]),
+		Body:    reader,
+		Tagging: awsTags,
+	}
+	if entry.Attributes != nil && entry.Attributes.Mime != "" {
+		uploadInput.ContentType = aws.String(entry.Attributes.Mime)
+	}
+	if contentEncoding := remote_storage.EntryContentEncoding(entry); contentEncoding != "" {
+		uploadInput.ContentEncoding = aws.String(contentEncoding)
+	}
+	if s.conf.S3StorageClass != "" {
+		uploadInput.StorageClass = aws.String(s.conf.S3StorageClass)
+	}
+	_, err = uploader.Upload(uploadInput)
+
+	//in case it fails to upload
+	if err != nil {
+		return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
+	}
+
+	// read back the remote entry
+	return s.readFileRemoteEntry(loc)
+
+}
+
+func toTagging(attributes map[string][]byte) *s3.Tagging {
+	tagging := &s3.Tagging{}
+	for k, v := range attributes {
+		tagging.TagSet = append(tagging.TagSet, &s3.Tag{
+			Key:   aws.String(k),
+			Value: aws.String(string(v)),
+		})
+	}
+	return tagging
+}
+
+func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	return s.StatFile(loc)
+}
+
+// the largest object a single CopyObject call accepts
+const s3CopyObjectSizeLimit = 5 * 1024 * 1024 * 1024
+
+func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
+		return nil
+	}
+
+	// Content-Encoding is S3 system metadata, changeable without a content
+	// rewrite only through an in-place copy
+	if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
+		key := loc.Path[1:]
+		if fileSize := int64(filer.FileSize(newEntry)); fileSize > s3CopyObjectSizeLimit {
+			glog.Warningf("s3 %s/%s: applying the Content-Encoding change needs an object copy, but %d bytes exceeds the copy limit; it will apply on the next content write", loc.Bucket, key, fileSize)
+		} else {
+			// the replace directive drops everything not resent, so read the
+			// object's current metadata and carry it over
+			headOut, headErr := s.conn.HeadObject(&s3.HeadObjectInput{
+				Bucket: aws.String(loc.Bucket),
+				Key:    aws.String(key),
+			})
+			if headErr != nil {
+				return fmt.Errorf("stat %s/%s before metadata copy: %w", loc.Bucket, key, headErr)
+			}
+			copyInput := &s3.CopyObjectInput{
+				Bucket:                  aws.String(loc.Bucket),
+				Key:                     aws.String(key),
+				CopySource:              aws.String(url.PathEscape(loc.Bucket + "/" + key)),
+				MetadataDirective:       aws.String(s3.MetadataDirectiveReplace),
+				Metadata:                headOut.Metadata,
+				ContentType:             headOut.ContentType,
+				CacheControl:            headOut.CacheControl,
+				ContentDisposition:      headOut.ContentDisposition,
+				ContentLanguage:         headOut.ContentLanguage,
+				WebsiteRedirectLocation: headOut.WebsiteRedirectLocation,
+				ServerSideEncryption:    headOut.ServerSideEncryption,
+				SSEKMSKeyId:             headOut.SSEKMSKeyId,
+				StorageClass:            headOut.StorageClass,
+			}
+			if headOut.Expires != nil {
+				if expires, parseErr := http.ParseTime(*headOut.Expires); parseErr == nil {
+					copyInput.Expires = aws.Time(expires)
+				}
+			}
+			if encoding != "" {
+				copyInput.ContentEncoding = aws.String(encoding)
+			}
+			if newEntry.Attributes != nil && newEntry.Attributes.Mime != "" {
+				copyInput.ContentType = aws.String(newEntry.Attributes.Mime)
+			}
+			if s.conf.S3StorageClass != "" {
+				copyInput.StorageClass = aws.String(s.conf.S3StorageClass)
+			}
+			if _, err = s.conn.CopyObject(copyInput); err != nil {
+				return fmt.Errorf("update content encoding of %s/%s: %w", loc.Bucket, key, err)
+			}
+		}
+	}
+
+	// same as the write path: a remote without tagging support rejects both
+	// PutObjectTagging and DeleteObjectTagging
+	if !s.conf.S3SupportTagging {
+		return
+	}
+
+	tagging := toTagging(newEntry.Extended)
+	if len(tagging.TagSet) > 0 {
+		_, err = s.conn.PutObjectTagging(&s3.PutObjectTaggingInput{
+			Bucket:  aws.String(loc.Bucket),
+			Key:     aws.String(loc.Path[1:]),
+			Tagging: tagging,
+		})
+	} else {
+		_, err = s.conn.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+			Bucket: aws.String(loc.Bucket),
+			Key:    aws.String(loc.Path[1:]),
+		})
+	}
+	return
+}
+func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	_, err = s.conn.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+	})
+	return
+}
+
+func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return nil, fmt.Errorf("list buckets: %w", err)
+	}
+	for _, b := range resp.Buckets {
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      *b.Name,
+			CreatedAt: *b.CreationDate,
+		})
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
+	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
+		ACL:                        nil,
+		Bucket:                     aws.String(name),
+		CreateBucketConfiguration:  nil,
+		GrantFullControl:           nil,
+		GrantRead:                  nil,
+		GrantReadACP:               nil,
+		GrantWrite:                 nil,
+		GrantWriteACP:              nil,
+		ObjectLockEnabledForBucket: nil,
+	})
+	if err != nil {
+		return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err)
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
+	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(name),
+	})
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/s3/s3_storage_client_test.go

@@ -0,0 +1,347 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/stretchr/testify/require"
+)
+
+func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) {
+	maker := s3RemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+	}
+
+	remoteClient, err := maker.Make(conf)
+	require.NoError(t, err)
+
+	client, ok := remoteClient.(*s3RemoteStorageClient)
+	require.True(t, ok)
+
+	s3Client, ok := client.conn.(*awss3.S3)
+	require.True(t, ok)
+	require.Same(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+}
+
+func TestS3MakeUsesStaticCredentialsWhenKeysAreProvided(t *testing.T) {
+	maker := s3RemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-access",
+		S3SecretKey:      "test-secret",
+	}
+
+	remoteClient, err := maker.Make(conf)
+	require.NoError(t, err)
+
+	client, ok := remoteClient.(*s3RemoteStorageClient)
+	require.True(t, ok)
+
+	s3Client, ok := client.conn.(*awss3.S3)
+	require.True(t, ok)
+	require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+
+	credValue, err := s3Client.Config.Credentials.Get()
+	require.NoError(t, err)
+	require.Equal(t, conf.S3AccessKey, credValue.AccessKeyID)
+	require.Equal(t, conf.S3SecretKey, credValue.SecretAccessKey)
+}
+
+func TestS3RemoteStorageClientImplementsInterface(t *testing.T) {
+	var _ remote_storage.RemoteStorageClient = (*s3RemoteStorageClient)(nil)
+}
+
+func TestS3ErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
+	require.Error(t, remote_storage.ErrRemoteObjectNotFound)
+	require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
+}
+
+// removeDirectoryMock serves canned listing pages and records the delete batches.
+type removeDirectoryMock struct {
+	s3iface.S3API
+	pages        []*awss3.ListObjectsV2Output
+	listInputs   []*awss3.ListObjectsV2Input
+	deleteInputs []*awss3.DeleteObjectsInput
+	deleteResp   *awss3.DeleteObjectsOutput
+	deleteErr    error
+}
+
+func (m *removeDirectoryMock) ListObjectsV2Pages(input *awss3.ListObjectsV2Input, fn func(*awss3.ListObjectsV2Output, bool) bool) error {
+	m.listInputs = append(m.listInputs, input)
+	for i, page := range m.pages {
+		if !fn(page, i == len(m.pages)-1) {
+			return nil
+		}
+	}
+	return nil
+}
+
+func (m *removeDirectoryMock) DeleteObjects(input *awss3.DeleteObjectsInput) (*awss3.DeleteObjectsOutput, error) {
+	m.deleteInputs = append(m.deleteInputs, input)
+	if m.deleteErr != nil {
+		return nil, m.deleteErr
+	}
+	if m.deleteResp != nil {
+		return m.deleteResp, nil
+	}
+	return &awss3.DeleteObjectsOutput{}, nil
+}
+
+func listPage(keys ...string) *awss3.ListObjectsV2Output {
+	page := &awss3.ListObjectsV2Output{}
+	for _, key := range keys {
+		page.Contents = append(page.Contents, &awss3.Object{Key: aws.String(key)})
+	}
+	return page
+}
+
+func deletedKeys(input *awss3.DeleteObjectsInput) (keys []string) {
+	for _, object := range input.Delete.Objects {
+		keys = append(keys, aws.StringValue(object.Key))
+	}
+	return
+}
+
+func TestS3RemoveDirectoryDeletesEveryListedObject(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages: []*awss3.ListObjectsV2Output{
+			listPage("testdir/a.bin", "testdir/sub/b.bin"),
+			listPage("testdir/z.bin"),
+		},
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+
+	require.Len(t, mock.listInputs, 1)
+	// without the trailing slash the listing would also match /testdir2
+	require.Equal(t, "testdir/", aws.StringValue(mock.listInputs[0].Prefix))
+	require.Len(t, mock.deleteInputs, 2)
+	require.Equal(t, []string{"testdir/a.bin", "testdir/sub/b.bin"}, deletedKeys(mock.deleteInputs[0]))
+	require.Equal(t, []string{"testdir/z.bin"}, deletedKeys(mock.deleteInputs[1]))
+}
+
+func TestS3RemoveDirectoryEmptyListingSendsNoDeletes(t *testing.T) {
+	mock := &removeDirectoryMock{pages: []*awss3.ListObjectsV2Output{listPage()}}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+	require.Empty(t, mock.deleteInputs)
+}
+
+func TestS3RemoveDirectoryRefusesBucketRoot(t *testing.T) {
+	mock := &removeDirectoryMock{pages: []*awss3.ListObjectsV2Output{listPage("a.bin")}}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+	require.Empty(t, mock.listInputs)
+	require.Empty(t, mock.deleteInputs)
+}
+
+func TestS3RemoveDirectoryReturnsBatchError(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages:     []*awss3.ListObjectsV2Output{listPage("testdir/a.bin")},
+		deleteErr: fmt.Errorf("access denied"),
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.ErrorContains(t, client.RemoveDirectory(loc), "access denied")
+}
+
+func TestS3RemoveDirectoryReturnsPerKeyError(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages: []*awss3.ListObjectsV2Output{listPage("testdir/a.bin", "testdir/b.bin")},
+		deleteResp: &awss3.DeleteObjectsOutput{
+			Errors: []*awss3.Error{{
+				Key:     aws.String("testdir/a.bin"),
+				Code:    aws.String("InternalError"),
+				Message: aws.String("try again"),
+			}, {
+				Key:     aws.String("testdir/b.bin"),
+				Code:    aws.String("InternalError"),
+				Message: aws.String("try again"),
+			}},
+		},
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	err := client.RemoveDirectory(loc)
+	require.ErrorContains(t, err, "2 keys failed")
+	require.ErrorContains(t, err, "testdir/a.bin")
+}
+
+// captureRoundTripper records the PUT request that the s3manager uploader
+// sends, and short-circuits all calls with a 200 so the SDK is satisfied.
+type captureRoundTripper struct {
+	uploadReq *http.Request
+}
+
+func (c *captureRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Method == http.MethodPut {
+		c.uploadReq = req.Clone(req.Context())
+	}
+	if req.Body != nil {
+		_, _ = io.Copy(io.Discard, req.Body)
+		_ = req.Body.Close()
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Header: http.Header{
+			"ETag": []string{"\"etag\""},
+		},
+		Request: req,
+	}, nil
+}
+
+func (c *captureRoundTripper) uploadContentType() string {
+	if c.uploadReq == nil {
+		return ""
+	}
+	return c.uploadReq.Header.Get("Content-Type")
+}
+
+func newCapturingS3Client(t *testing.T) (*s3RemoteStorageClient, *captureRoundTripper) {
+	t.Helper()
+	rt := &captureRoundTripper{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		Name:             "test",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "https://example.invalid",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-key",
+		S3SecretKey:      "test-secret",
+	}
+	httpClient := &http.Client{Transport: rt}
+	rs, err := MakeWithHTTPClient(conf, httpClient)
+	require.NoError(t, err)
+	return rs.(*s3RemoteStorageClient), rt
+}
+
+func TestS3WriteFilePassesMimeAsContentType(t *testing.T) {
+	client, rt := newCapturingS3Client(t)
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test",
+		Bucket: "bucket",
+		Path:   "/dir/test.html",
+	}
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{Mime: "text/html"},
+	}
+
+	_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("<html></html>")))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	require.Equal(t, "text/html", rt.uploadContentType(), "Content-Type should match entry.Attributes.Mime")
+}
+
+// recordingRoundTripper records every request and answers all of them with a 200.
+type recordingRoundTripper struct {
+	requests []*http.Request
+}
+
+func (c *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	c.requests = append(c.requests, req.Clone(req.Context()))
+	if req.Body != nil {
+		_, _ = io.Copy(io.Discard, req.Body)
+		_ = req.Body.Close()
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Header:     http.Header{},
+		Request:    req,
+	}, nil
+}
+
+func newRecordingS3Client(t *testing.T, supportTagging bool) (*s3RemoteStorageClient, *recordingRoundTripper) {
+	t.Helper()
+	rt := &recordingRoundTripper{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		Name:             "test",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "https://example.invalid",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-key",
+		S3SecretKey:      "test-secret",
+		S3SupportTagging: supportTagging,
+	}
+	rs, err := MakeWithHTTPClient(conf, &http.Client{Transport: rt})
+	require.NoError(t, err)
+	return rs.(*s3RemoteStorageClient), rt
+}
+
+func TestS3UpdateFileMetadataSkipsTaggingWhenUnsupported(t *testing.T) {
+	client, rt := newRecordingS3Client(t, false)
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/dir/file.bin"}
+	plain := &filer_pb.Entry{}
+	tagged := &filer_pb.Entry{Extended: map[string][]byte{"k1": []byte("v1")}}
+
+	require.NoError(t, client.UpdateFileMetadata(loc, plain, tagged))
+	require.Empty(t, rt.requests, "adding attributes must not send PutObjectTagging")
+
+	require.NoError(t, client.UpdateFileMetadata(loc, tagged, plain))
+	require.Empty(t, rt.requests, "removing attributes must not send DeleteObjectTagging")
+}
+
+func TestS3UpdateFileMetadataSendsTaggingWhenSupported(t *testing.T) {
+	client, rt := newRecordingS3Client(t, true)
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/dir/file.bin"}
+	plain := &filer_pb.Entry{}
+	tagged := &filer_pb.Entry{Extended: map[string][]byte{"k1": []byte("v1")}}
+
+	require.NoError(t, client.UpdateFileMetadata(loc, plain, tagged))
+	require.Len(t, rt.requests, 1)
+	require.Equal(t, http.MethodPut, rt.requests[0].Method)
+	require.Contains(t, rt.requests[0].URL.RawQuery, "tagging")
+
+	rt.requests = nil
+	require.NoError(t, client.UpdateFileMetadata(loc, tagged, plain))
+	require.Len(t, rt.requests, 1)
+	require.Equal(t, http.MethodDelete, rt.requests[0].Method)
+	require.Contains(t, rt.requests[0].URL.RawQuery, "tagging")
+}
+
+func TestS3WriteFileOmitsContentTypeWhenMimeMissing(t *testing.T) {
+	client, rt := newCapturingS3Client(t)
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test",
+		Bucket: "bucket",
+		Path:   "/dir/test.bin",
+	}
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{},
+	}
+
+	_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("data")))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	// When entry.Attributes.Mime is empty we don't force a Content-Type so the
+	// remote can apply its own default rather than getting a misleading one.
+	require.Equal(t, "", rt.uploadContentType())
+}

weed/remote_storage/s3/storj.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["storj"] = new(StorjRemoteStorageMaker)
+}
+
+type StorjRemoteStorageMaker struct{}
+
+func (s StorjRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s StorjRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/tencent.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["tencent"] = new(TencentRemoteStorageMaker)
+}
+
+type TencentRemoteStorageMaker struct{}
+
+func (s TencentRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/wasabi.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["wasabi"] = new(WasabiRemoteStorageMaker)
+}
+
+type WasabiRemoteStorageMaker struct{}
+
+func (s WasabiRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/track_sync_offset.go

@@ -0,0 +1,74 @@
+package remote_storage
+
+import (
+	"context"
+	"errors"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+)
+
+const (
+	SyncKeyPrefix = "remote.sync."
+)
+
+func GetSyncOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, dir string) (lastOffsetTsNs int64, readErr error) {
+
+	dirHash := uint32(util.HashStringToLong(dir))
+
+	readErr = pb.WithFilerClient(false, 0, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		syncKey := []byte(SyncKeyPrefix + "____")
+		util.Uint32toBytes(syncKey[len(SyncKeyPrefix):len(SyncKeyPrefix)+4], dirHash)
+
+		resp, err := client.KvGet(context.Background(), &filer_pb.KvGetRequest{Key: syncKey})
+		if err != nil {
+			return err
+		}
+
+		if len(resp.Error) != 0 {
+			return errors.New(resp.Error)
+		}
+		if len(resp.Value) < 8 {
+			return nil
+		}
+
+		lastOffsetTsNs = int64(util.BytesToUint64(resp.Value))
+
+		return nil
+	})
+
+	return
+
+}
+
+func SetSyncOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, dir string, offsetTsNs int64) error {
+
+	dirHash := uint32(util.HashStringToLong(dir))
+
+	return pb.WithFilerClient(false, 0, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+
+		syncKey := []byte(SyncKeyPrefix + "____")
+		util.Uint32toBytes(syncKey[len(SyncKeyPrefix):len(SyncKeyPrefix)+4], dirHash)
+
+		valueBuf := make([]byte, 8)
+		util.Uint64toBytes(valueBuf, uint64(offsetTsNs))
+
+		resp, err := client.KvPut(context.Background(), &filer_pb.KvPutRequest{
+			Key:   syncKey,
+			Value: valueBuf,
+		})
+		if err != nil {
+			return err
+		}
+
+		if len(resp.Error) != 0 {
+			return errors.New(resp.Error)
+		}
+
+		return nil
+
+	})
+
+}