snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/remote_storage/s3/aliyun.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["aliyun"] = new(AliyunRemoteStorageMaker)
+}
+
+type AliyunRemoteStorageMaker struct{}
+
+func (s AliyunRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s AliyunRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/backblaze.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["b2"] = new(BackBlazeRemoteStorageMaker)
+}
+
+type BackBlazeRemoteStorageMaker struct{}
+
+func (s BackBlazeRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BackBlazeRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/baidu.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["baidu"] = new(BaiduRemoteStorageMaker)
+}
+
+type BaiduRemoteStorageMaker struct{}
+
+func (s BaiduRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s BaiduRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/contabo.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["contabo"] = new(ContaboRemoteStorageMaker)
+}
+
+type ContaboRemoteStorageMaker struct{}
+
+func (s ContaboRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s ContaboRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/endpoint_test.go

@@ -0,0 +1,44 @@
+package s3
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+)
+
+// TestS3CompatibleEndpointCoversAllProviders locks the type-to-endpoint mapping
+// the volume server's SSRF guard relies on. Every S3-SDK-backed provider dials
+// a caller-supplied endpoint, so each must surface it here; a new sibling that
+// forgets to would silently bypass the guard.
+func TestS3CompatibleEndpointCoversAllProviders(t *testing.T) {
+	cases := []struct {
+		conf *remote_pb.RemoteConf
+		want string
+	}{
+		{&remote_pb.RemoteConf{Type: "s3", S3Endpoint: "http://s3.internal"}, "http://s3.internal"},
+		{&remote_pb.RemoteConf{Type: "wasabi", WasabiEndpoint: "http://wasabi.internal"}, "http://wasabi.internal"},
+		{&remote_pb.RemoteConf{Type: "b2", BackblazeEndpoint: "http://b2.internal"}, "http://b2.internal"},
+		{&remote_pb.RemoteConf{Type: "aliyun", AliyunEndpoint: "http://aliyun.internal"}, "http://aliyun.internal"},
+		{&remote_pb.RemoteConf{Type: "tencent", TencentEndpoint: "http://tencent.internal"}, "http://tencent.internal"},
+		{&remote_pb.RemoteConf{Type: "baidu", BaiduEndpoint: "http://baidu.internal"}, "http://baidu.internal"},
+		{&remote_pb.RemoteConf{Type: "filebase", FilebaseEndpoint: "http://filebase.internal"}, "http://filebase.internal"},
+		{&remote_pb.RemoteConf{Type: "storj", StorjEndpoint: "http://storj.internal"}, "http://storj.internal"},
+		{&remote_pb.RemoteConf{Type: "contabo", ContaboEndpoint: "http://contabo.internal"}, "http://contabo.internal"},
+	}
+	for _, tc := range cases {
+		endpoint, ok := S3CompatibleEndpoint(tc.conf)
+		if !ok {
+			t.Errorf("type %q: expected an S3-compatible endpoint, got ok=false", tc.conf.Type)
+			continue
+		}
+		if endpoint != tc.want {
+			t.Errorf("type %q: endpoint = %q, want %q", tc.conf.Type, endpoint, tc.want)
+		}
+	}
+
+	// Every registered maker must be an S3-compatible type; the guard treats
+	// anything else (gcs, azure, ...) as not dialing a caller-supplied URL.
+	if _, ok := S3CompatibleEndpoint(&remote_pb.RemoteConf{Type: "gcs"}); ok {
+		t.Error("gcs must not be reported as an S3-compatible endpoint")
+	}
+}

weed/remote_storage/s3/filebase.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["filebase"] = new(FilebaseRemoteStorageMaker)
+}
+
+type FilebaseRemoteStorageMaker struct{}
+
+func (s FilebaseRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s FilebaseRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/s3_storage_client.go

@@ -0,0 +1,703 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["s3"] = new(s3RemoteStorageMaker)
+}
+
+type s3RemoteStorageMaker struct{}
+
+func (s s3RemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s s3RemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}
+
+// s3CompatibleOptions carries the per-provider AWS SDK knobs the shared client
+// builder needs. Every S3-SDK-backed provider (s3 plus the wasabi/b2/... family)
+// converges on the same s3RemoteStorageClient and dials a caller-supplied
+// endpoint, differing only in which RemoteConf fields it reads.
+type s3CompatibleOptions struct {
+	name           string
+	endpoint       string
+	region         string
+	accessKey      string
+	secretKey      string
+	forcePathStyle bool
+	signV4         bool
+	// anonymousWhenNoCreds disables SigV4 signing for public buckets when no
+	// credentials are supplied. Only the generic "s3" provider does this.
+	anonymousWhenNoCreds bool
+	// setUserAgent adds the SeaweedFS User-Agent header (generic "s3" only).
+	setUserAgent bool
+}
+
+// s3CompatibleClientOptions returns the client knobs for an S3-SDK-backed
+// RemoteConf and whether conf is such a type. Keeping the type-to-fields
+// mapping in one place lets the client builder and the SSRF endpoint guard
+// agree on exactly which endpoint each provider dials.
+func s3CompatibleClientOptions(conf *remote_pb.RemoteConf) (s3CompatibleOptions, bool) {
+	switch conf.Type {
+	case "s3":
+		return s3CompatibleOptions{
+			name:                 "s3",
+			endpoint:             conf.S3Endpoint,
+			region:               conf.S3Region,
+			accessKey:            conf.S3AccessKey,
+			secretKey:            conf.S3SecretKey,
+			forcePathStyle:       conf.S3ForcePathStyle,
+			signV4:               conf.S3V4Signature,
+			anonymousWhenNoCreds: true,
+			setUserAgent:         true,
+		}, true
+	case "aliyun":
+		return s3CompatibleOptions{
+			name:      "aliyun",
+			endpoint:  conf.AliyunEndpoint,
+			region:    conf.AliyunRegion,
+			accessKey: util.Nvl(conf.AliyunAccessKey, os.Getenv("ALICLOUD_ACCESS_KEY_ID")),
+			secretKey: util.Nvl(conf.AliyunSecretKey, os.Getenv("ALICLOUD_ACCESS_KEY_SECRET")),
+		}, true
+	case "b2":
+		return s3CompatibleOptions{
+			name:           "backblaze",
+			endpoint:       conf.BackblazeEndpoint,
+			region:         conf.BackblazeRegion,
+			accessKey:      conf.BackblazeKeyId,
+			secretKey:      conf.BackblazeApplicationKey,
+			forcePathStyle: true,
+		}, true
+	case "baidu":
+		return s3CompatibleOptions{
+			name:      "baidu",
+			endpoint:  conf.BaiduEndpoint,
+			region:    conf.BaiduRegion,
+			accessKey: util.Nvl(conf.BaiduAccessKey, os.Getenv("BDCLOUD_ACCESS_KEY")),
+			secretKey: util.Nvl(conf.BaiduSecretKey, os.Getenv("BDCLOUD_SECRET_KEY")),
+			signV4:    true,
+		}, true
+	case "contabo":
+		return s3CompatibleOptions{
+			name:           "contabo",
+			endpoint:       conf.ContaboEndpoint,
+			region:         conf.ContaboRegion,
+			accessKey:      util.Nvl(conf.ContaboAccessKey, os.Getenv("ACCESS_KEY")),
+			secretKey:      util.Nvl(conf.ContaboSecretKey, os.Getenv("SECRET_KEY")),
+			forcePathStyle: true,
+		}, true
+	case "filebase":
+		return s3CompatibleOptions{
+			name:           "filebase",
+			endpoint:       conf.FilebaseEndpoint,
+			region:         "us-east-1",
+			accessKey:      util.Nvl(conf.FilebaseAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")),
+			secretKey:      util.Nvl(conf.FilebaseSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")),
+			forcePathStyle: true,
+			signV4:         true,
+		}, true
+	case "storj":
+		return s3CompatibleOptions{
+			name:           "storj",
+			endpoint:       conf.StorjEndpoint,
+			region:         "us-west-2",
+			accessKey:      util.Nvl(conf.StorjAccessKey, os.Getenv("AWS_ACCESS_KEY_ID")),
+			secretKey:      util.Nvl(conf.StorjSecretKey, os.Getenv("AWS_SECRET_ACCESS_KEY")),
+			forcePathStyle: true,
+		}, true
+	case "tencent":
+		return s3CompatibleOptions{
+			name:           "tencent",
+			endpoint:       conf.TencentEndpoint,
+			region:         "us-west-2",
+			accessKey:      util.Nvl(conf.TencentSecretId, os.Getenv("COS_SECRETID")),
+			secretKey:      util.Nvl(conf.TencentSecretKey, os.Getenv("COS_SECRETKEY")),
+			forcePathStyle: true,
+		}, true
+	case "wasabi":
+		return s3CompatibleOptions{
+			name:           "wasabi",
+			endpoint:       conf.WasabiEndpoint,
+			region:         conf.WasabiRegion,
+			accessKey:      conf.WasabiAccessKey,
+			secretKey:      conf.WasabiSecretKey,
+			forcePathStyle: true,
+		}, true
+	}
+	return s3CompatibleOptions{}, false
+}
+
+// S3CompatibleEndpoint returns the endpoint an S3-SDK-backed RemoteConf would
+// dial directly, and whether conf is such a type. The volume server validates
+// this endpoint against the SSRF deny-list for every S3-compatible provider,
+// not just the generic "s3" type.
+func S3CompatibleEndpoint(conf *remote_pb.RemoteConf) (string, bool) {
+	opt, ok := s3CompatibleClientOptions(conf)
+	if !ok {
+		return "", false
+	}
+	return opt.endpoint, true
+}
+
+// MakeWithHTTPClient builds the client for any S3-SDK-backed remote storage
+// type using the supplied *http.Client (or the AWS SDK default when nil).
+// Callers that need to pin the dial path against DNS rebinding can pass a
+// client whose transport has a guarded DialContext.
+func MakeWithHTTPClient(conf *remote_pb.RemoteConf, httpClient *http.Client) (remote_storage.RemoteStorageClient, error) {
+	opt, ok := s3CompatibleClientOptions(conf)
+	if !ok {
+		return nil, fmt.Errorf("%q is not an S3-compatible remote storage type", conf.Type)
+	}
+	client := &s3RemoteStorageClient{
+		conf: conf,
+	}
+	config := &aws.Config{
+		Region:                        aws.String(opt.region),
+		Endpoint:                      aws.String(opt.endpoint),
+		S3ForcePathStyle:              aws.Bool(opt.forcePathStyle),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if httpClient != nil {
+		config.HTTPClient = httpClient
+	}
+	if opt.accessKey != "" && opt.secretKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(opt.accessKey, opt.secretKey, "")
+	} else if opt.anonymousWhenNoCreds && opt.accessKey == "" && opt.secretKey == "" {
+		// Explicitly disable signing for public buckets.
+		config.Credentials = credentials.AnonymousCredentials
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create %s session: %w", opt.name, err)
+	}
+	if opt.signV4 {
+		sess.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
+	}
+	if opt.setUserAgent {
+		sess.Handlers.Build.PushBack(func(r *request.Request) {
+			r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
+		})
+	}
+	sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
+	client.conn = s3.New(sess)
+	return client, nil
+}
+
+var skipSha256PayloadSigning = func(r *request.Request) {
+	// see https://github.com/ceph/ceph/pull/15965/files
+	if r.ClientInfo.ServiceID != "S3" {
+		return
+	}
+	if r.Operation.Name == "PutObject" || r.Operation.Name == "UploadPart" {
+		if len(r.HTTPRequest.Header.Get("X-Amz-Content-Sha256")) == 0 {
+			r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
+		}
+	}
+}
+
+type s3RemoteStorageClient struct {
+	conf *remote_pb.RemoteConf
+	conn s3iface.S3API
+}
+
+const (
+	defaultUploadConcurrency = 1
+	defaultReadConcurrency   = 5
+)
+
+func (s *s3RemoteStorageClient) uploadConcurrency() int {
+	if n := int(s.conf.GetUploadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultUploadConcurrency
+}
+
+func (s *s3RemoteStorageClient) downloadConcurrency() int {
+	if n := int(s.conf.GetDownloadConcurrency()); n > 0 {
+		return n
+	}
+	return defaultReadConcurrency
+}
+
+var _ = remote_storage.RemoteStorageClient(&s3RemoteStorageClient{})
+
+func (s *s3RemoteStorageClient) Traverse(remote *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
+
+	pathKey := remote.Path[1:]
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket:              aws.String(remote.Bucket),
+		ContinuationToken:   nil,
+		Delimiter:           nil, // not aws.String("/"), iterate through all entries
+		EncodingType:        nil,
+		ExpectedBucketOwner: nil,
+		FetchOwner:          nil,
+		MaxKeys:             nil, // aws.Int64(1000),
+		Prefix:              aws.String(pathKey),
+		RequestPayer:        nil,
+		StartAfter:          nil,
+	}
+	isLastPage := false
+	for !isLastPage && err == nil {
+		var localErr error
+		listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+			for _, content := range page.Contents {
+				key := *content.Key
+				key = "/" + key
+				dir, name := util.FullPath(key).DirAndName()
+				remoteEntry := &filer_pb.RemoteEntry{
+					StorageName: s.conf.Name,
+				}
+				if content.LastModified != nil {
+					remoteEntry.RemoteMtime = content.LastModified.Unix()
+				}
+				if content.Size != nil {
+					remoteEntry.RemoteSize = *content.Size
+				}
+				if content.ETag != nil {
+					remoteEntry.RemoteETag = *content.ETag
+				}
+				if err := visitFn(dir, name, false, remoteEntry); err != nil {
+					localErr = err
+					return false
+				}
+			}
+			listInput.ContinuationToken = page.NextContinuationToken
+			isLastPage = lastPage
+			return true
+		})
+		if listErr != nil {
+			err = fmt.Errorf("list %v: %w", remote, listErr)
+		}
+		if localErr != nil {
+			err = fmt.Errorf("process %v: %w", remote, localErr)
+		}
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) ListDirectory(ctx context.Context, loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {
+	pathKey := loc.Path[1:]
+	if pathKey != "" && !strings.HasSuffix(pathKey, "/") {
+		pathKey += "/"
+	}
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket:    aws.String(loc.Bucket),
+		Prefix:    aws.String(pathKey),
+		Delimiter: aws.String("/"),
+	}
+
+	var localErr error
+	listErr := s.conn.ListObjectsV2PagesWithContext(ctx, listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, prefix := range page.CommonPrefixes {
+			if prefix.Prefix == nil {
+				continue
+			}
+			dirKey := "/" + strings.TrimSuffix(*prefix.Prefix, "/")
+			dir, name := util.FullPath(dirKey).DirAndName()
+			if err := visitFn(dir, name, true, nil); err != nil {
+				localErr = err
+				return false
+			}
+		}
+		for _, content := range page.Contents {
+			key := "/" + *content.Key
+			if strings.HasSuffix(key, "/") {
+				continue // skip directory markers
+			}
+			dir, name := util.FullPath(key).DirAndName()
+			remoteEntry := &filer_pb.RemoteEntry{
+				StorageName: s.conf.Name,
+			}
+			if content.LastModified != nil {
+				remoteEntry.RemoteMtime = content.LastModified.Unix()
+			}
+			if content.Size != nil {
+				remoteEntry.RemoteSize = *content.Size
+			}
+			if content.ETag != nil {
+				remoteEntry.RemoteETag = *content.ETag
+			}
+			if err := visitFn(dir, name, false, remoteEntry); err != nil {
+				localErr = err
+				return false
+			}
+		}
+		return true
+	})
+	if listErr != nil {
+		return fmt.Errorf("list directory %v: %w", loc, listErr)
+	}
+	if localErr != nil {
+		return fmt.Errorf("process directory %v: %w", loc, localErr)
+	}
+	return nil
+}
+
+func (s *s3RemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
+	resp, err := s.conn.HeadObject(&s3.HeadObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+	})
+	if err != nil {
+		if reqErr, ok := err.(awserr.RequestFailure); ok && reqErr.StatusCode() == http.StatusNotFound {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("stat s3 %s%s: %w", loc.Bucket, loc.Path, err)
+	}
+	remoteEntry = &filer_pb.RemoteEntry{
+		StorageName: s.conf.Name,
+	}
+	if resp.ContentLength != nil {
+		remoteEntry.RemoteSize = *resp.ContentLength
+	}
+	if resp.LastModified != nil {
+		remoteEntry.RemoteMtime = resp.LastModified.Unix()
+	}
+	if resp.ETag != nil {
+		remoteEntry.RemoteETag = *resp.ETag
+	}
+	// a HeadObject response is authoritative: no header means no encoding
+	remoteEntry.RemoteContentEncoding = aws.String(aws.StringValue(resp.ContentEncoding))
+	return remoteEntry, nil
+}
+
+func (s *s3RemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
+	return s.ReadFileWithConcurrency(loc, offset, size, 0)
+}
+
+func (s *s3RemoteStorageClient) ReadFileWithConcurrency(loc *remote_pb.RemoteStorageLocation, offset int64, size int64, concurrency int) (data []byte, err error) {
+	if concurrency <= 0 {
+		concurrency = s.downloadConcurrency()
+	}
+	downloader := s3manager.NewDownloaderWithClient(s.conn, func(u *s3manager.Downloader) {
+		u.PartSize = int64(4 * 1024 * 1024)
+		u.Concurrency = concurrency
+	})
+
+	dataSlice := make([]byte, int(size))
+	writerAt := aws.NewWriteAtBuffer(dataSlice)
+
+	n, err := downloader.Download(writerAt, &s3.GetObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	// The buffer is pre-sized to size, so a short read leaves the tail
+	// zero-padded and would be cached as valid-looking but corrupt content.
+	// Reject it instead.
+	if n != size {
+		return nil, fmt.Errorf("short read from %s%s at offset %d: got %d bytes, want %d", loc.Bucket, loc.Path, offset, n, size)
+	}
+
+	return writerAt.Bytes(), nil
+}
+
+func (s *s3RemoteStorageClient) ReadFileAsStream(ctx context.Context, loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (reader io.ReadCloser, err error) {
+	output, err := s.conn.GetObjectWithContext(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+		Range:  aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)),
+	})
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchKey {
+			return nil, remote_storage.ErrRemoteObjectNotFound
+		}
+		return nil, fmt.Errorf("failed to open stream for %s%s: %v", loc.Bucket, loc.Path, err)
+	}
+	return output.Body, nil
+}
+
+func (s *s3RemoteStorageClient) WriteDirectory(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry) (err error) {
+	return nil
+}
+
+func (s *s3RemoteStorageClient) RemoveDirectory(loc *remote_pb.RemoteStorageLocation) (err error) {
+	// the trailing slash keeps sibling prefixes that share the name intact
+	prefix := loc.Path[1:]
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+	if prefix == "" {
+		// the mount root maps to the whole bucket; wiping every object from a
+		// single namespace event is too destructive, so keep them
+		glog.Warningf("s3 %s: skip removing directory mapped to the bucket root", loc.Bucket)
+		return nil
+	}
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket: aws.String(loc.Bucket),
+		Prefix: aws.String(prefix),
+	}
+	var deleteErr error
+	listErr := s.conn.ListObjectsV2Pages(listInput, func(page *s3.ListObjectsV2Output, lastPage bool) bool {
+		var objects []*s3.ObjectIdentifier
+		for _, content := range page.Contents {
+			objects = append(objects, &s3.ObjectIdentifier{Key: content.Key})
+		}
+		if len(objects) == 0 {
+			return true
+		}
+		// a listing page holds at most 1000 keys, the DeleteObjects limit
+		resp, batchErr := s.conn.DeleteObjects(&s3.DeleteObjectsInput{
+			Bucket: aws.String(loc.Bucket),
+			Delete: &s3.Delete{
+				Objects: objects,
+				Quiet:   aws.Bool(true),
+			},
+		})
+		if batchErr != nil {
+			deleteErr = batchErr
+			return false
+		}
+		if len(resp.Errors) > 0 {
+			// a batch can fail 1000 keys; report the scope, not every key
+			failed := resp.Errors[0]
+			deleteErr = fmt.Errorf("%d keys failed, first is %s: %s %s", len(resp.Errors), aws.StringValue(failed.Key), aws.StringValue(failed.Code), aws.StringValue(failed.Message))
+			return false
+		}
+		return true
+	})
+	if listErr != nil {
+		return fmt.Errorf("list %s/%s: %w", loc.Bucket, prefix, listErr)
+	}
+	if deleteErr != nil {
+		return fmt.Errorf("remove directory %s/%s: %w", loc.Bucket, prefix, deleteErr)
+	}
+	return nil
+}
+
+func (s *s3RemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocation, entry *filer_pb.Entry, reader io.Reader) (remoteEntry *filer_pb.RemoteEntry, err error) {
+
+	fileSize := int64(filer.FileSize(entry))
+
+	partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB
+	for partSize*1000 < fileSize {
+		partSize *= 4
+	}
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
+		u.PartSize = partSize
+		u.Concurrency = s.uploadConcurrency()
+	})
+
+	// process tagging
+	tags := ""
+	var awsTags *string
+	// openstack swift doesn't support s3 object tagging
+	if s.conf.S3SupportTagging {
+		for k, v := range entry.Extended {
+			if len(tags) > 0 {
+				tags = tags + "&"
+			}
+			tags = tags + k + "=" + string(v)
+		}
+		awsTags = aws.String(tags)
+	}
+
+	// Upload the file to S3.
+	uploadInput := &s3manager.UploadInput{
+		Bucket:  aws.String(loc.Bucket),
+		Key:     aws.String(loc.Path[1:]),
+		Body:    reader,
+		Tagging: awsTags,
+	}
+	if entry.Attributes != nil && entry.Attributes.Mime != "" {
+		uploadInput.ContentType = aws.String(entry.Attributes.Mime)
+	}
+	if contentEncoding := remote_storage.EntryContentEncoding(entry); contentEncoding != "" {
+		uploadInput.ContentEncoding = aws.String(contentEncoding)
+	}
+	if s.conf.S3StorageClass != "" {
+		uploadInput.StorageClass = aws.String(s.conf.S3StorageClass)
+	}
+	_, err = uploader.Upload(uploadInput)
+
+	//in case it fails to upload
+	if err != nil {
+		return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
+	}
+
+	// read back the remote entry
+	return s.readFileRemoteEntry(loc)
+
+}
+
+func toTagging(attributes map[string][]byte) *s3.Tagging {
+	tagging := &s3.Tagging{}
+	for k, v := range attributes {
+		tagging.TagSet = append(tagging.TagSet, &s3.Tag{
+			Key:   aws.String(k),
+			Value: aws.String(string(v)),
+		})
+	}
+	return tagging
+}
+
+func (s *s3RemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
+	return s.StatFile(loc)
+}
+
+// the largest object a single CopyObject call accepts
+const s3CopyObjectSizeLimit = 5 * 1024 * 1024 * 1024
+
+func (s *s3RemoteStorageClient) UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error) {
+	if reflect.DeepEqual(oldEntry.Extended, newEntry.Extended) {
+		return nil
+	}
+
+	// Content-Encoding is S3 system metadata, changeable without a content
+	// rewrite only through an in-place copy
+	if encoding := remote_storage.EntryContentEncoding(newEntry); encoding != remote_storage.EntryContentEncoding(oldEntry) {
+		key := loc.Path[1:]
+		if fileSize := int64(filer.FileSize(newEntry)); fileSize > s3CopyObjectSizeLimit {
+			glog.Warningf("s3 %s/%s: applying the Content-Encoding change needs an object copy, but %d bytes exceeds the copy limit; it will apply on the next content write", loc.Bucket, key, fileSize)
+		} else {
+			// the replace directive drops everything not resent, so read the
+			// object's current metadata and carry it over
+			headOut, headErr := s.conn.HeadObject(&s3.HeadObjectInput{
+				Bucket: aws.String(loc.Bucket),
+				Key:    aws.String(key),
+			})
+			if headErr != nil {
+				return fmt.Errorf("stat %s/%s before metadata copy: %w", loc.Bucket, key, headErr)
+			}
+			copyInput := &s3.CopyObjectInput{
+				Bucket:                  aws.String(loc.Bucket),
+				Key:                     aws.String(key),
+				CopySource:              aws.String(url.PathEscape(loc.Bucket + "/" + key)),
+				MetadataDirective:       aws.String(s3.MetadataDirectiveReplace),
+				Metadata:                headOut.Metadata,
+				ContentType:             headOut.ContentType,
+				CacheControl:            headOut.CacheControl,
+				ContentDisposition:      headOut.ContentDisposition,
+				ContentLanguage:         headOut.ContentLanguage,
+				WebsiteRedirectLocation: headOut.WebsiteRedirectLocation,
+				ServerSideEncryption:    headOut.ServerSideEncryption,
+				SSEKMSKeyId:             headOut.SSEKMSKeyId,
+				StorageClass:            headOut.StorageClass,
+			}
+			if headOut.Expires != nil {
+				if expires, parseErr := http.ParseTime(*headOut.Expires); parseErr == nil {
+					copyInput.Expires = aws.Time(expires)
+				}
+			}
+			if encoding != "" {
+				copyInput.ContentEncoding = aws.String(encoding)
+			}
+			if newEntry.Attributes != nil && newEntry.Attributes.Mime != "" {
+				copyInput.ContentType = aws.String(newEntry.Attributes.Mime)
+			}
+			if s.conf.S3StorageClass != "" {
+				copyInput.StorageClass = aws.String(s.conf.S3StorageClass)
+			}
+			if _, err = s.conn.CopyObject(copyInput); err != nil {
+				return fmt.Errorf("update content encoding of %s/%s: %w", loc.Bucket, key, err)
+			}
+		}
+	}
+
+	// same as the write path: a remote without tagging support rejects both
+	// PutObjectTagging and DeleteObjectTagging
+	if !s.conf.S3SupportTagging {
+		return
+	}
+
+	tagging := toTagging(newEntry.Extended)
+	if len(tagging.TagSet) > 0 {
+		_, err = s.conn.PutObjectTagging(&s3.PutObjectTaggingInput{
+			Bucket:  aws.String(loc.Bucket),
+			Key:     aws.String(loc.Path[1:]),
+			Tagging: tagging,
+		})
+	} else {
+		_, err = s.conn.DeleteObjectTagging(&s3.DeleteObjectTaggingInput{
+			Bucket: aws.String(loc.Bucket),
+			Key:    aws.String(loc.Path[1:]),
+		})
+	}
+	return
+}
+func (s *s3RemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error) {
+	_, err = s.conn.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(loc.Bucket),
+		Key:    aws.String(loc.Path[1:]),
+	})
+	return
+}
+
+func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	resp, err := s.conn.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return nil, fmt.Errorf("list buckets: %w", err)
+	}
+	for _, b := range resp.Buckets {
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      *b.Name,
+			CreatedAt: *b.CreationDate,
+		})
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
+	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
+		ACL:                        nil,
+		Bucket:                     aws.String(name),
+		CreateBucketConfiguration:  nil,
+		GrantFullControl:           nil,
+		GrantRead:                  nil,
+		GrantReadACP:               nil,
+		GrantWrite:                 nil,
+		GrantWriteACP:              nil,
+		ObjectLockEnabledForBucket: nil,
+	})
+	if err != nil {
+		return fmt.Errorf("%s create bucket %s: %v", s.conf.Name, name, err)
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
+	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket: aws.String(name),
+	})
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}

weed/remote_storage/s3/s3_storage_client_test.go

@@ -0,0 +1,347 @@
+package s3
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+	"github.com/stretchr/testify/require"
+)
+
+func TestS3MakeUsesAnonymousCredentialsWhenKeysAreEmpty(t *testing.T) {
+	maker := s3RemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+	}
+
+	remoteClient, err := maker.Make(conf)
+	require.NoError(t, err)
+
+	client, ok := remoteClient.(*s3RemoteStorageClient)
+	require.True(t, ok)
+
+	s3Client, ok := client.conn.(*awss3.S3)
+	require.True(t, ok)
+	require.Same(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+}
+
+func TestS3MakeUsesStaticCredentialsWhenKeysAreProvided(t *testing.T) {
+	maker := s3RemoteStorageMaker{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "http://localhost:8333",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-access",
+		S3SecretKey:      "test-secret",
+	}
+
+	remoteClient, err := maker.Make(conf)
+	require.NoError(t, err)
+
+	client, ok := remoteClient.(*s3RemoteStorageClient)
+	require.True(t, ok)
+
+	s3Client, ok := client.conn.(*awss3.S3)
+	require.True(t, ok)
+	require.NotSame(t, credentials.AnonymousCredentials, s3Client.Config.Credentials)
+
+	credValue, err := s3Client.Config.Credentials.Get()
+	require.NoError(t, err)
+	require.Equal(t, conf.S3AccessKey, credValue.AccessKeyID)
+	require.Equal(t, conf.S3SecretKey, credValue.SecretAccessKey)
+}
+
+func TestS3RemoteStorageClientImplementsInterface(t *testing.T) {
+	var _ remote_storage.RemoteStorageClient = (*s3RemoteStorageClient)(nil)
+}
+
+func TestS3ErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
+	require.Error(t, remote_storage.ErrRemoteObjectNotFound)
+	require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
+}
+
+// removeDirectoryMock serves canned listing pages and records the delete batches.
+type removeDirectoryMock struct {
+	s3iface.S3API
+	pages        []*awss3.ListObjectsV2Output
+	listInputs   []*awss3.ListObjectsV2Input
+	deleteInputs []*awss3.DeleteObjectsInput
+	deleteResp   *awss3.DeleteObjectsOutput
+	deleteErr    error
+}
+
+func (m *removeDirectoryMock) ListObjectsV2Pages(input *awss3.ListObjectsV2Input, fn func(*awss3.ListObjectsV2Output, bool) bool) error {
+	m.listInputs = append(m.listInputs, input)
+	for i, page := range m.pages {
+		if !fn(page, i == len(m.pages)-1) {
+			return nil
+		}
+	}
+	return nil
+}
+
+func (m *removeDirectoryMock) DeleteObjects(input *awss3.DeleteObjectsInput) (*awss3.DeleteObjectsOutput, error) {
+	m.deleteInputs = append(m.deleteInputs, input)
+	if m.deleteErr != nil {
+		return nil, m.deleteErr
+	}
+	if m.deleteResp != nil {
+		return m.deleteResp, nil
+	}
+	return &awss3.DeleteObjectsOutput{}, nil
+}
+
+func listPage(keys ...string) *awss3.ListObjectsV2Output {
+	page := &awss3.ListObjectsV2Output{}
+	for _, key := range keys {
+		page.Contents = append(page.Contents, &awss3.Object{Key: aws.String(key)})
+	}
+	return page
+}
+
+func deletedKeys(input *awss3.DeleteObjectsInput) (keys []string) {
+	for _, object := range input.Delete.Objects {
+		keys = append(keys, aws.StringValue(object.Key))
+	}
+	return
+}
+
+func TestS3RemoveDirectoryDeletesEveryListedObject(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages: []*awss3.ListObjectsV2Output{
+			listPage("testdir/a.bin", "testdir/sub/b.bin"),
+			listPage("testdir/z.bin"),
+		},
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+
+	require.Len(t, mock.listInputs, 1)
+	// without the trailing slash the listing would also match /testdir2
+	require.Equal(t, "testdir/", aws.StringValue(mock.listInputs[0].Prefix))
+	require.Len(t, mock.deleteInputs, 2)
+	require.Equal(t, []string{"testdir/a.bin", "testdir/sub/b.bin"}, deletedKeys(mock.deleteInputs[0]))
+	require.Equal(t, []string{"testdir/z.bin"}, deletedKeys(mock.deleteInputs[1]))
+}
+
+func TestS3RemoveDirectoryEmptyListingSendsNoDeletes(t *testing.T) {
+	mock := &removeDirectoryMock{pages: []*awss3.ListObjectsV2Output{listPage()}}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+	require.Empty(t, mock.deleteInputs)
+}
+
+func TestS3RemoveDirectoryRefusesBucketRoot(t *testing.T) {
+	mock := &removeDirectoryMock{pages: []*awss3.ListObjectsV2Output{listPage("a.bin")}}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/"}
+
+	require.NoError(t, client.RemoveDirectory(loc))
+	require.Empty(t, mock.listInputs)
+	require.Empty(t, mock.deleteInputs)
+}
+
+func TestS3RemoveDirectoryReturnsBatchError(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages:     []*awss3.ListObjectsV2Output{listPage("testdir/a.bin")},
+		deleteErr: fmt.Errorf("access denied"),
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	require.ErrorContains(t, client.RemoveDirectory(loc), "access denied")
+}
+
+func TestS3RemoveDirectoryReturnsPerKeyError(t *testing.T) {
+	mock := &removeDirectoryMock{
+		pages: []*awss3.ListObjectsV2Output{listPage("testdir/a.bin", "testdir/b.bin")},
+		deleteResp: &awss3.DeleteObjectsOutput{
+			Errors: []*awss3.Error{{
+				Key:     aws.String("testdir/a.bin"),
+				Code:    aws.String("InternalError"),
+				Message: aws.String("try again"),
+			}, {
+				Key:     aws.String("testdir/b.bin"),
+				Code:    aws.String("InternalError"),
+				Message: aws.String("try again"),
+			}},
+		},
+	}
+	client := &s3RemoteStorageClient{conf: &remote_pb.RemoteConf{Name: "test"}, conn: mock}
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/testdir"}
+
+	err := client.RemoveDirectory(loc)
+	require.ErrorContains(t, err, "2 keys failed")
+	require.ErrorContains(t, err, "testdir/a.bin")
+}
+
+// captureRoundTripper records the PUT request that the s3manager uploader
+// sends, and short-circuits all calls with a 200 so the SDK is satisfied.
+type captureRoundTripper struct {
+	uploadReq *http.Request
+}
+
+func (c *captureRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Method == http.MethodPut {
+		c.uploadReq = req.Clone(req.Context())
+	}
+	if req.Body != nil {
+		_, _ = io.Copy(io.Discard, req.Body)
+		_ = req.Body.Close()
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Header: http.Header{
+			"ETag": []string{"\"etag\""},
+		},
+		Request: req,
+	}, nil
+}
+
+func (c *captureRoundTripper) uploadContentType() string {
+	if c.uploadReq == nil {
+		return ""
+	}
+	return c.uploadReq.Header.Get("Content-Type")
+}
+
+func newCapturingS3Client(t *testing.T) (*s3RemoteStorageClient, *captureRoundTripper) {
+	t.Helper()
+	rt := &captureRoundTripper{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		Name:             "test",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "https://example.invalid",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-key",
+		S3SecretKey:      "test-secret",
+	}
+	httpClient := &http.Client{Transport: rt}
+	rs, err := MakeWithHTTPClient(conf, httpClient)
+	require.NoError(t, err)
+	return rs.(*s3RemoteStorageClient), rt
+}
+
+func TestS3WriteFilePassesMimeAsContentType(t *testing.T) {
+	client, rt := newCapturingS3Client(t)
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test",
+		Bucket: "bucket",
+		Path:   "/dir/test.html",
+	}
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{Mime: "text/html"},
+	}
+
+	_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("<html></html>")))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	require.Equal(t, "text/html", rt.uploadContentType(), "Content-Type should match entry.Attributes.Mime")
+}
+
+// recordingRoundTripper records every request and answers all of them with a 200.
+type recordingRoundTripper struct {
+	requests []*http.Request
+}
+
+func (c *recordingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	c.requests = append(c.requests, req.Clone(req.Context()))
+	if req.Body != nil {
+		_, _ = io.Copy(io.Discard, req.Body)
+		_ = req.Body.Close()
+	}
+	return &http.Response{
+		StatusCode: http.StatusOK,
+		Body:       io.NopCloser(strings.NewReader("")),
+		Header:     http.Header{},
+		Request:    req,
+	}, nil
+}
+
+func newRecordingS3Client(t *testing.T, supportTagging bool) (*s3RemoteStorageClient, *recordingRoundTripper) {
+	t.Helper()
+	rt := &recordingRoundTripper{}
+	conf := &remote_pb.RemoteConf{
+		Type:             "s3",
+		Name:             "test",
+		S3Region:         "us-east-1",
+		S3Endpoint:       "https://example.invalid",
+		S3ForcePathStyle: true,
+		S3AccessKey:      "test-key",
+		S3SecretKey:      "test-secret",
+		S3SupportTagging: supportTagging,
+	}
+	rs, err := MakeWithHTTPClient(conf, &http.Client{Transport: rt})
+	require.NoError(t, err)
+	return rs.(*s3RemoteStorageClient), rt
+}
+
+func TestS3UpdateFileMetadataSkipsTaggingWhenUnsupported(t *testing.T) {
+	client, rt := newRecordingS3Client(t, false)
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/dir/file.bin"}
+	plain := &filer_pb.Entry{}
+	tagged := &filer_pb.Entry{Extended: map[string][]byte{"k1": []byte("v1")}}
+
+	require.NoError(t, client.UpdateFileMetadata(loc, plain, tagged))
+	require.Empty(t, rt.requests, "adding attributes must not send PutObjectTagging")
+
+	require.NoError(t, client.UpdateFileMetadata(loc, tagged, plain))
+	require.Empty(t, rt.requests, "removing attributes must not send DeleteObjectTagging")
+}
+
+func TestS3UpdateFileMetadataSendsTaggingWhenSupported(t *testing.T) {
+	client, rt := newRecordingS3Client(t, true)
+	loc := &remote_pb.RemoteStorageLocation{Name: "test", Bucket: "bucket", Path: "/dir/file.bin"}
+	plain := &filer_pb.Entry{}
+	tagged := &filer_pb.Entry{Extended: map[string][]byte{"k1": []byte("v1")}}
+
+	require.NoError(t, client.UpdateFileMetadata(loc, plain, tagged))
+	require.Len(t, rt.requests, 1)
+	require.Equal(t, http.MethodPut, rt.requests[0].Method)
+	require.Contains(t, rt.requests[0].URL.RawQuery, "tagging")
+
+	rt.requests = nil
+	require.NoError(t, client.UpdateFileMetadata(loc, tagged, plain))
+	require.Len(t, rt.requests, 1)
+	require.Equal(t, http.MethodDelete, rt.requests[0].Method)
+	require.Contains(t, rt.requests[0].URL.RawQuery, "tagging")
+}
+
+func TestS3WriteFileOmitsContentTypeWhenMimeMissing(t *testing.T) {
+	client, rt := newCapturingS3Client(t)
+	loc := &remote_pb.RemoteStorageLocation{
+		Name:   "test",
+		Bucket: "bucket",
+		Path:   "/dir/test.bin",
+	}
+	entry := &filer_pb.Entry{
+		Attributes: &filer_pb.FuseAttributes{},
+	}
+
+	_, _ = client.WriteFile(loc, entry, bytes.NewReader([]byte("data")))
+
+	require.NotNil(t, rt.uploadReq, "uploader should have issued a PUT")
+	// When entry.Attributes.Mime is empty we don't force a Content-Type so the
+	// remote can apply its own default rather than getting a misleading one.
+	require.Equal(t, "", rt.uploadContentType())
+}

weed/remote_storage/s3/storj.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["storj"] = new(StorjRemoteStorageMaker)
+}
+
+type StorjRemoteStorageMaker struct{}
+
+func (s StorjRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s StorjRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/tencent.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["tencent"] = new(TencentRemoteStorageMaker)
+}
+
+type TencentRemoteStorageMaker struct{}
+
+func (s TencentRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s TencentRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}

weed/remote_storage/s3/wasabi.go

@@ -0,0 +1,20 @@
+package s3
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
+	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
+)
+
+func init() {
+	remote_storage.RemoteStorageClientMakers["wasabi"] = new(WasabiRemoteStorageMaker)
+}
+
+type WasabiRemoteStorageMaker struct{}
+
+func (s WasabiRemoteStorageMaker) HasBucket() bool {
+	return true
+}
+
+func (s WasabiRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
+	return MakeWithHTTPClient(conf, nil)
+}