snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/backend/s3_backend/s3_backend.go

@@ -0,0 +1,247 @@
+package s3_backend
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/google/uuid"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+)
+
+func init() {
+	backend.BackendStorageFactories["s3"] = &S3BackendFactory{}
+}
+
+type S3BackendFactory struct {
+}
+
+func (factory *S3BackendFactory) StorageType() backend.StorageType {
+	return backend.StorageType("s3")
+}
+func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, configPrefix string, id string) (backend.BackendStorage, error) {
+	return newS3BackendStorage(configuration, configPrefix, id)
+}
+
+type S3BackendStorage struct {
+	id                    string
+	aws_access_key_id     string
+	aws_secret_access_key string
+	region                string
+	bucket                string
+	endpoint              string
+	storageClass          string
+	forcePathStyle        bool
+	uploadConcurrency     int
+	downloadConcurrency   int
+	conn                  s3iface.S3API
+}
+
+func newS3BackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *S3BackendStorage, err error) {
+	s = &S3BackendStorage{}
+	s.id = id
+	s.aws_access_key_id = configuration.GetString(configPrefix + "aws_access_key_id")
+	s.aws_secret_access_key = configuration.GetString(configPrefix + "aws_secret_access_key")
+	s.region = configuration.GetString(configPrefix + "region")
+	s.bucket = configuration.GetString(configPrefix + "bucket")
+	s.endpoint = configuration.GetString(configPrefix + "endpoint")
+	s.storageClass = configuration.GetString(configPrefix + "storage_class")
+	s.forcePathStyle = util.ParseBool(configuration.GetString(configPrefix+"force_path_style"), true)
+	s.uploadConcurrency = parseConcurrency(configuration.GetString(configPrefix+"upload_concurrency"), defaultUploadConcurrency)
+	s.downloadConcurrency = parseConcurrency(configuration.GetString(configPrefix+"download_concurrency"), defaultDownloadConcurrency)
+	if s.storageClass == "" {
+		s.storageClass = "STANDARD_IA"
+	}
+
+	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region, s.endpoint, s.forcePathStyle)
+
+	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
+	return
+}
+
+func (s *S3BackendStorage) ToProperties() map[string]string {
+	m := make(map[string]string)
+	m["aws_access_key_id"] = s.aws_access_key_id
+	m["aws_secret_access_key"] = s.aws_secret_access_key
+	m["region"] = s.region
+	m["bucket"] = s.bucket
+	m["endpoint"] = s.endpoint
+	m["storage_class"] = s.storageClass
+	m["force_path_style"] = util.BoolToString(s.forcePathStyle)
+	m["upload_concurrency"] = strconv.Itoa(s.uploadConcurrency)
+	m["download_concurrency"] = strconv.Itoa(s.downloadConcurrency)
+	return m
+}
+
+const (
+	defaultUploadConcurrency   = 5
+	defaultDownloadConcurrency = 5
+)
+
+func parseConcurrency(value string, def int) int {
+	if value == "" {
+		return def
+	}
+	if n, err := strconv.Atoi(value); err == nil && n > 0 {
+		return n
+	}
+	glog.Warningf("invalid concurrency value %q, using default %d", value, def)
+	return def
+}
+
+func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+
+	f := &S3BackendStorageFile{
+		backendStorage: s,
+		key:            key,
+		tierInfo:       tierInfo,
+	}
+
+	return f
+}
+
+func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error) {
+	randomUuid, _ := uuid.NewRandom()
+	key = randomUuid.String()
+
+	if concurrency <= 0 {
+		concurrency = s.uploadConcurrency
+	}
+
+	glog.V(1).Infof("copying dat file of %s to remote s3.%s as %s", f.Name(), s.id, key)
+
+	util.Retry("upload to S3", func() error {
+		size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, s.storageClass, fn, concurrency)
+		return err
+	})
+
+	return
+}
+
+func (s *S3BackendStorage) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error) {
+	if concurrency <= 0 {
+		concurrency = s.downloadConcurrency
+	}
+
+	glog.V(1).Infof("download dat file of %s from remote s3.%s as %s", fileName, s.id, key)
+
+	size, err = downloadFromS3(s.conn, fileName, s.bucket, key, fn, concurrency)
+
+	return
+}
+
+func (s *S3BackendStorage) DeleteFile(key string) (err error) {
+
+	glog.V(1).Infof("delete dat file %s from remote", key)
+
+	err = deleteFromS3(s.conn, s.bucket, key)
+
+	return
+}
+
+type S3BackendStorageFile struct {
+	backendStorage *S3BackendStorage
+	key            string
+	tierInfo       *volume_server_pb.VolumeInfo
+}
+
+func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {
+	if off < 0 {
+		return 0, fmt.Errorf("negative offset %d", off)
+	}
+
+	requested := len(p)
+	if requested == 0 {
+		return 0, nil
+	}
+
+	datSize, _, _ := s3backendStorageFile.GetStat()
+
+	if datSize > 0 && off >= datSize {
+		return 0, io.EOF
+	}
+
+	bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)
+
+	getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
+		Bucket: &s3backendStorageFile.backendStorage.bucket,
+		Key:    &s3backendStorageFile.key,
+		Range:  &bytesRange,
+	})
+
+	if getObjectErr != nil {
+		return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
+	}
+	defer getObjectOutput.Body.Close()
+
+	// glog.V(3).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
+	// glog.V(3).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)
+
+	var readCount int
+	for {
+		p = p[readCount:]
+		if len(p) == 0 {
+			break
+		}
+		readCount, err = getObjectOutput.Body.Read(p)
+		n += readCount
+
+		if err != nil {
+			break
+		}
+	}
+
+	if err == io.EOF && n == requested {
+		err = nil
+	}
+
+	return
+}
+
+func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
+	panic("not implemented")
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Truncate(off int64) error {
+	panic("not implemented")
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Close() error {
+	return nil
+}
+
+func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
+
+	files := s3backendStorageFile.tierInfo.GetFiles()
+
+	if len(files) == 0 {
+		err = fmt.Errorf("remote file info not found")
+		return
+	}
+
+	datSize = int64(files[0].FileSize)
+	modTime = time.Unix(int64(files[0].ModifiedTime), 0)
+
+	return
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Name() string {
+	return s3backendStorageFile.key
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Sync() error {
+	return nil
+}

weed/storage/backend/s3_backend/s3_backend_test.go

@@ -0,0 +1,156 @@
+package s3_backend
+
+import (
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"testing/iotest"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+type testProperties map[string]string
+
+func (m testProperties) GetString(key string) string {
+	return m[key]
+}
+
+type stubS3Client struct {
+	s3iface.S3API
+	getObject func(*s3.GetObjectInput) (*s3.GetObjectOutput, error)
+}
+
+func (s *stubS3Client) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	return s.getObject(input)
+}
+
+func TestReadAtReturnsEOFForShortRead(t *testing.T) {
+	client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+		return &s3.GetObjectOutput{Body: io.NopCloser(strings.NewReader("abc"))}, nil
+	}}
+	file := S3BackendStorageFile{
+		backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+		key:            "key",
+		tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+			{FileSize: 10},
+		}},
+	}
+
+	buffer := make([]byte, 5)
+	n, err := file.ReadAt(buffer, 0)
+	if n != 3 {
+		t.Fatalf("ReadAt() read %d bytes, want 3", n)
+	}
+	if !errors.Is(err, io.EOF) {
+		t.Fatalf("ReadAt() error = %v, want io.EOF", err)
+	}
+}
+
+func TestReadAtClearsEOFOnFullRead(t *testing.T) {
+	client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+		return &s3.GetObjectOutput{Body: io.NopCloser(iotest.DataErrReader(strings.NewReader("abcde")))}, nil
+	}}
+	file := S3BackendStorageFile{
+		backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+		key:            "key",
+		tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+			{FileSize: 10},
+		}},
+	}
+
+	buffer := make([]byte, 5)
+	n, err := file.ReadAt(buffer, 0)
+	if n != 5 || err != nil {
+		t.Fatalf("ReadAt() = (%d, %v), want (5, nil)", n, err)
+	}
+	if string(buffer) != "abcde" {
+		t.Fatalf("ReadAt() buffer = %q, want %q", buffer, "abcde")
+	}
+}
+
+func TestReadAtRejectsInvalidRequestsLocally(t *testing.T) {
+	tests := []struct {
+		name    string
+		buffer  []byte
+		offset  int64
+		wantErr bool
+	}{
+		{name: "empty buffer", buffer: nil},
+		{name: "negative offset", buffer: make([]byte, 1), offset: -1, wantErr: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			called := false
+			client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+				called = true
+				return nil, errors.New("unexpected GetObject")
+			}}
+			file := S3BackendStorageFile{
+				backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+				key:            "key",
+				tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+					{FileSize: 10},
+				}},
+			}
+
+			n, err := file.ReadAt(tt.buffer, tt.offset)
+			if n != 0 || (err != nil) != tt.wantErr {
+				t.Fatalf("ReadAt() = (%d, %v), want error %t", n, err, tt.wantErr)
+			}
+			if called {
+				t.Fatal("ReadAt() called S3 for an invalid request")
+			}
+		})
+	}
+}
+
+func TestParseConcurrency(t *testing.T) {
+	cases := []struct {
+		value    string
+		def      int
+		expected int
+	}{
+		{"", 5, 5},    // unset -> default
+		{"0", 5, 5},   // explicit zero means default
+		{"-3", 5, 5},  // invalid -> default
+		{"abc", 5, 5}, // invalid -> default
+		{"1", 5, 1},   // override
+		{"64", 5, 64}, // override
+	}
+	for _, tt := range cases {
+		if got := parseConcurrency(tt.value, tt.def); got != tt.expected {
+			t.Errorf("parseConcurrency(%q, %d) = %d, want %d", tt.value, tt.def, got, tt.expected)
+		}
+	}
+}
+
+func TestS3BackendStorageConcurrencyConfigRoundTrip(t *testing.T) {
+	s, err := newS3BackendStorage(testProperties{}, "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.uploadConcurrency != defaultUploadConcurrency || s.downloadConcurrency != defaultDownloadConcurrency {
+		t.Fatalf("defaults: upload=%d download=%d, want %d/%d",
+			s.uploadConcurrency, s.downloadConcurrency, defaultUploadConcurrency, defaultDownloadConcurrency)
+	}
+
+	s, err = newS3BackendStorage(testProperties{
+		"upload_concurrency":   "1",
+		"download_concurrency": "17",
+	}, "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.uploadConcurrency != 1 || s.downloadConcurrency != 17 {
+		t.Fatalf("configured: upload=%d download=%d, want 1/17", s.uploadConcurrency, s.downloadConcurrency)
+	}
+
+	props := s.ToProperties()
+	if props["upload_concurrency"] != "1" || props["download_concurrency"] != "17" {
+		t.Fatalf("ToProperties: %v", props)
+	}
+}

weed/storage/backend/s3_backend/s3_download.go

@@ -0,0 +1,107 @@
+package s3_backend
+
+import (
+	"fmt"
+	"os"
+	"sync/atomic"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string, sourceKey string,
+	fn func(progressed int64, percentage float32) error, concurrency int) (fileSize int64, err error) {
+	if concurrency <= 0 {
+		concurrency = defaultDownloadConcurrency
+	}
+
+	fileSize, err = getFileSize(sess, sourceBucket, sourceKey)
+	if err != nil {
+		return
+	}
+
+	//open the file
+	f, err := os.OpenFile(destFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %q, %v", destFileName, err)
+	}
+	defer f.Close()
+
+	// Create a downloader with the session and custom options
+	downloader := s3manager.NewDownloaderWithClient(sess, func(u *s3manager.Downloader) {
+		u.PartSize = int64(64 * 1024 * 1024)
+		u.Concurrency = concurrency
+	})
+
+	fileWriter := &s3DownloadProgressedWriter{
+		fp:      f,
+		size:    fileSize,
+		written: 0,
+		fn:      fn,
+	}
+
+	// Download the file from S3.
+	fileSize, err = downloader.Download(fileWriter, &s3.GetObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	if err != nil {
+		return fileSize, fmt.Errorf("failed to download /buckets/%s/%s to %s: %v", sourceBucket, sourceKey, destFileName, err)
+	}
+
+	// fsync the downloaded .dat so its content is durable before the caller trims the
+	// remote reference and deletes the shared remote object.
+	if syncErr := f.Sync(); syncErr != nil {
+		return fileSize, fmt.Errorf("failed to fsync %s: %v", destFileName, syncErr)
+	}
+
+	glog.V(1).Infof("downloaded file %s\n", destFileName)
+
+	return
+}
+
+// adapted from https://github.com/aws/aws-sdk-go/pull/1868
+// and https://petersouter.xyz/s3-download-progress-bar-in-golang/
+type s3DownloadProgressedWriter struct {
+	size    int64
+	written int64
+	fn      func(progressed int64, percentage float32) error
+	fp      *os.File
+}
+
+func (w *s3DownloadProgressedWriter) WriteAt(p []byte, off int64) (int, error) {
+	n, err := w.fp.WriteAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	// Got the length have read( or means has uploaded), and you can construct your message
+	atomic.AddInt64(&w.written, int64(n))
+
+	if w.fn != nil {
+		written := w.written
+		if err := w.fn(written, float32(written*100)/float32(w.size)); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+func getFileSize(svc s3iface.S3API, bucket string, key string) (filesize int64, error error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+
+	resp, err := svc.HeadObject(params)
+	if err != nil {
+		return 0, err
+	}
+
+	return *resp.ContentLength, nil
+}

weed/storage/backend/s3_backend/s3_sessions.go

@@ -0,0 +1,63 @@
+package s3_backend
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+)
+
+var (
+	s3Sessions   = make(map[string]s3iface.S3API)
+	sessionsLock sync.Mutex
+)
+
+func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string, forcePathStyle bool) (s3iface.S3API, error) {
+
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	cacheKey := fmt.Sprintf("%s|%s", region, endpoint)
+	if t, found := s3Sessions[cacheKey]; found {
+		return t, nil
+	}
+
+	config := &aws.Config{
+		Region:                        aws.String(region),
+		Endpoint:                      aws.String(endpoint),
+		S3ForcePathStyle:              aws.Bool(forcePathStyle),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create aws session in region %s: %v", region, err)
+	}
+	sess.Handlers.Build.PushBack(func(r *request.Request) {
+		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
+	})
+
+	t := s3.New(sess)
+
+	s3Sessions[cacheKey] = t
+
+	return t, nil
+
+}
+
+func deleteFromS3(sess s3iface.S3API, sourceBucket string, sourceKey string) (err error) {
+	_, err = sess.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	return err
+}

weed/storage/backend/s3_backend/s3_upload.go

@@ -0,0 +1,112 @@
+package s3_backend
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, storageClass string, fn func(progressed int64, percentage float32) error, concurrency int) (fileSize int64, err error) {
+	if concurrency <= 0 {
+		concurrency = defaultUploadConcurrency
+	}
+
+	//open the file
+	f, err := os.Open(filename)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %q, %v", filename, err)
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat file %q, %v", filename, err)
+	}
+
+	fileSize = info.Size()
+
+	partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB
+	for partSize*1000 < fileSize {
+		partSize *= 4
+	}
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) {
+		u.PartSize = partSize
+		u.Concurrency = concurrency
+	})
+
+	fileReader := &s3UploadProgressedReader{
+		fp:      f,
+		size:    fileSize,
+		signMap: map[int64]struct{}{},
+		fn:      fn,
+	}
+
+	// Upload the file to S3.
+	var result *s3manager.UploadOutput
+	result, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket:       aws.String(destBucket),
+		Key:          aws.String(destKey),
+		Body:         fileReader,
+		StorageClass: aws.String(storageClass),
+	})
+
+	//in case it fails to upload
+	if err != nil {
+		return 0, fmt.Errorf("failed to upload file %s: %v", filename, err)
+	}
+	glog.V(1).Infof("file %s uploaded to %s\n", filename, result.Location)
+
+	return
+}
+
+// adapted from https://github.com/aws/aws-sdk-go/pull/1868
+// https://github.com/aws/aws-sdk-go/blob/main/example/service/s3/putObjectWithProcess/putObjWithProcess.go
+type s3UploadProgressedReader struct {
+	fp      *os.File
+	size    int64
+	read    int64
+	signMap map[int64]struct{}
+	mux     sync.Mutex
+	fn      func(progressed int64, percentage float32) error
+}
+
+func (r *s3UploadProgressedReader) Read(p []byte) (int, error) {
+	return r.fp.Read(p)
+}
+
+func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) {
+	n, err := r.fp.ReadAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	r.mux.Lock()
+	// Ignore the first signature call
+	if _, ok := r.signMap[off]; ok {
+		r.read += int64(n)
+	} else {
+		r.signMap[off] = struct{}{}
+	}
+	r.mux.Unlock()
+
+	if r.fn != nil {
+		read := r.read
+		if err := r.fn(read, float32(read*100)/float32(r.size)); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+func (r *s3UploadProgressedReader) Seek(offset int64, whence int) (int64, error) {
+	return r.fp.Seek(offset, whence)
+}