snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/backend/all/all.go

@@ -0,0 +1,10 @@
+// Package all registers every tiered-storage backend factory. Binaries that
+// serve or manage tiered volumes blank-import it from their composition root;
+// keeping the registrations out of weed/storage lets library consumers avoid
+// pulling the backend SDKs into their dependency graph.
+package all
+
+import (
+	_ "github.com/seaweedfs/seaweedfs/weed/storage/backend/rclone_backend"
+	_ "github.com/seaweedfs/seaweedfs/weed/storage/backend/s3_backend"
+)

weed/storage/backend/backend.go

@@ -0,0 +1,143 @@
+package backend
+
+import (
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type BackendStorageFile interface {
+	io.ReaderAt
+	io.WriterAt
+	Truncate(off int64) error
+	io.Closer
+	GetStat() (datSize int64, modTime time.Time, err error)
+	Name() string
+	Sync() error
+}
+
+type BackendStorage interface {
+	ToProperties() map[string]string
+	NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
+	// concurrency > 0 caps concurrent network transfers; <= 0 uses the backend default.
+	CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error)
+	DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error)
+	DeleteFile(key string) (err error)
+}
+
+type StringProperties interface {
+	GetString(key string) string
+}
+type StorageType string
+type BackendStorageFactory interface {
+	StorageType() StorageType
+	BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
+}
+
+var (
+	BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
+	BackendStorages         = make(map[string]BackendStorage)
+)
+
+// used by master to load remote storage configurations
+func LoadConfiguration(config *util.ViperProxy) {
+
+	StorageBackendPrefix := "storage.backend"
+
+	for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
+		backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
+		if !found {
+			glog.Fatalf("backend storage type %s not found", backendTypeName)
+		}
+		for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
+			if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
+				continue
+			}
+			if _, found := BackendStorages[backendTypeName+"."+backendStorageId]; found {
+				continue
+			}
+			backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
+				StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
+			if buildErr != nil {
+				glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
+			}
+			BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
+			if backendStorageId == "default" {
+				BackendStorages[backendTypeName] = backendStorage
+			}
+		}
+	}
+
+}
+
+// used by volume server to receive remote storage configurations from master
+func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
+
+	for _, storageBackend := range storageBackends {
+		backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
+		if !found {
+			glog.Warningf("storage type %s not found", storageBackend.Type)
+			continue
+		}
+		if _, found := BackendStorages[storageBackend.Type+"."+storageBackend.Id]; found {
+			continue
+		}
+		backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
+		if buildErr != nil {
+			glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
+		}
+		BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
+		if storageBackend.Id == "default" {
+			BackendStorages[storageBackend.Type] = backendStorage
+		}
+	}
+}
+
+type Properties struct {
+	m map[string]string
+}
+
+func newProperties(m map[string]string) *Properties {
+	return &Properties{m: m}
+}
+
+func (p *Properties) GetString(key string) string {
+	if v, found := p.m[key]; found {
+		return v
+	}
+	return ""
+}
+
+func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
+	for sName, s := range BackendStorages {
+		sType, sId := BackendNameToTypeId(sName)
+		if sType == "" {
+			continue
+		}
+		backends = append(backends, &master_pb.StorageBackend{
+			Type:       sType,
+			Id:         sId,
+			Properties: s.ToProperties(),
+		})
+	}
+	return
+}
+
+func BackendNameToTypeId(backendName string) (backendType, backendId string) {
+	parts := strings.Split(backendName, ".")
+	if len(parts) == 1 {
+		return backendName, "default"
+	}
+	if len(parts) != 2 {
+		return
+	}
+
+	backendType, backendId = parts[0], parts[1]
+	return
+}

weed/storage/backend/disk_file.go

@@ -0,0 +1,133 @@
+package backend
+
+import (
+	"io"
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	. "github.com/seaweedfs/seaweedfs/weed/storage/types"
+)
+
+var (
+	_ BackendStorageFile = &DiskFile{}
+)
+
+const isMac = runtime.GOOS == "darwin"
+
+type DiskFile struct {
+	File         *os.File
+	fullFilePath string
+	fileSize     int64
+	modTime      time.Time
+}
+
+func NewDiskFile(f *os.File) *DiskFile {
+	stat, err := f.Stat()
+	if err != nil {
+		glog.Fatalf("stat file %s: %v", f.Name(), err)
+	}
+	offset := stat.Size()
+	if offset%NeedlePaddingSize != 0 {
+		offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize)
+	}
+
+	return &DiskFile{
+		fullFilePath: f.Name(),
+		File:         f,
+		fileSize:     offset,
+		modTime:      stat.ModTime(),
+	}
+}
+
+func (df *DiskFile) ReadAt(p []byte, off int64) (n int, err error) {
+	if df.File == nil {
+		return 0, os.ErrClosed
+	}
+	n, err = df.File.ReadAt(p, off)
+	if err == io.EOF && n == len(p) {
+		err = nil
+	}
+	return
+}
+
+func (df *DiskFile) WriteAt(p []byte, off int64) (n int, err error) {
+	if df.File == nil {
+		return 0, os.ErrClosed
+	}
+	n, err = df.File.WriteAt(p, off)
+	if err == nil {
+		waterMark := off + int64(n)
+		if waterMark > df.fileSize {
+			df.fileSize = waterMark
+			df.modTime = time.Now()
+		}
+	}
+	return
+}
+
+func (df *DiskFile) Write(p []byte) (n int, err error) {
+	return df.WriteAt(p, df.fileSize)
+}
+
+func (df *DiskFile) Truncate(off int64) error {
+	if df.File == nil {
+		return os.ErrClosed
+	}
+	err := df.File.Truncate(off)
+	if err == nil {
+		df.fileSize = off
+		df.modTime = time.Now()
+	}
+	return err
+}
+
+func (df *DiskFile) Close() error {
+	if df.File == nil {
+		return nil
+	}
+	err := df.Sync()
+	var err1 error
+	if df.File != nil {
+		// always try to close
+		err1 = df.File.Close()
+	}
+	// assume closed
+	df.File = nil
+	if err != nil {
+		return err
+	}
+	if err1 != nil {
+		return err1
+	}
+	return nil
+}
+
+func (df *DiskFile) GetStat() (datSize int64, modTime time.Time, err error) {
+	if df.File == nil {
+		err = os.ErrClosed
+	}
+	return df.fileSize, df.modTime, err
+}
+
+func (df *DiskFile) Name() string {
+	return df.fullFilePath
+}
+
+func (df *DiskFile) Fd() uintptr {
+	if df.File == nil {
+		return ^uintptr(0)
+	}
+	return df.File.Fd()
+}
+
+func (df *DiskFile) Sync() error {
+	if df.File == nil {
+		return os.ErrClosed
+	}
+	if isMac {
+		return nil
+	}
+	return df.File.Sync()
+}

weed/storage/backend/memory_map/memory_map.go

@@ -0,0 +1,30 @@
+package memory_map
+
+import (
+	"os"
+	"strconv"
+)
+
+type MemoryBuffer struct {
+	aligned_length uint64
+	length         uint64
+	aligned_ptr    uintptr
+	ptr            uintptr
+	Buffer         []byte
+}
+
+type MemoryMap struct {
+	File                   *os.File
+	file_memory_map_handle uintptr
+	write_map_views        []MemoryBuffer
+	max_length             uint64
+	End_of_file            int64
+}
+
+func ReadMemoryMapMaxSizeMb(memoryMapMaxSizeMbString string) (uint32, error) {
+	if memoryMapMaxSizeMbString == "" {
+		return 0, nil
+	}
+	memoryMapMaxSize64, err := strconv.ParseUint(memoryMapMaxSizeMbString, 10, 32)
+	return uint32(memoryMapMaxSize64), err
+}

weed/storage/backend/memory_map/memory_map_backend.go

@@ -0,0 +1,62 @@
+package memory_map
+
+import (
+	"os"
+	"time"
+)
+
+var (
+// _ backend.BackendStorageFile = &MemoryMappedFile{} // remove this to break import cycle
+)
+
+type MemoryMappedFile struct {
+	mm *MemoryMap
+}
+
+func NewMemoryMappedFile(f *os.File, memoryMapSizeMB uint32) *MemoryMappedFile {
+	mmf := &MemoryMappedFile{
+		mm: new(MemoryMap),
+	}
+	mmf.mm.CreateMemoryMap(f, 1024*1024*uint64(memoryMapSizeMB))
+	return mmf
+}
+
+func (mmf *MemoryMappedFile) ReadAt(p []byte, off int64) (n int, err error) {
+	readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p)))
+	if e != nil {
+		return 0, e
+	}
+	// TODO avoid the extra copy
+	copy(p, readBytes)
+	return len(readBytes), nil
+}
+
+func (mmf *MemoryMappedFile) WriteAt(p []byte, off int64) (n int, err error) {
+	mmf.mm.WriteMemory(uint64(off), uint64(len(p)), p)
+	return len(p), nil
+}
+
+func (mmf *MemoryMappedFile) Truncate(off int64) error {
+	return nil
+}
+
+func (mmf *MemoryMappedFile) Close() error {
+	mmf.mm.DeleteFileAndMemoryMap()
+	return nil
+}
+
+func (mmf *MemoryMappedFile) GetStat() (datSize int64, modTime time.Time, err error) {
+	stat, e := mmf.mm.File.Stat()
+	if e == nil {
+		return mmf.mm.End_of_file + 1, stat.ModTime(), nil
+	}
+	return 0, time.Time{}, err
+}
+
+func (mmf *MemoryMappedFile) Name() string {
+	return mmf.mm.File.Name()
+}
+
+func (mm *MemoryMappedFile) Sync() error {
+	return nil
+}

weed/storage/backend/memory_map/memory_map_other.go

@@ -0,0 +1,24 @@
+//go:build !windows
+
+package memory_map
+
+import (
+	"fmt"
+	"os"
+)
+
+func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) {
+}
+
+func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
+
+}
+
+func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) ([]byte, error) {
+	dataSlice := []byte{}
+	return dataSlice, fmt.Errorf("Memory Map not implemented for this platform")
+}
+
+func (mBuffer *MemoryMap) DeleteFileAndMemoryMap() {
+
+}

weed/storage/backend/memory_map/memory_map_test.go

@@ -0,0 +1,10 @@
+package memory_map
+
+import "testing"
+
+func TestMemoryMapMaxSizeReadWrite(t *testing.T) {
+	memoryMapSize, _ := ReadMemoryMapMaxSizeMb("5000")
+	if memoryMapSize != 5000 {
+		t.Errorf("empty memoryMapSize:%v", memoryMapSize)
+	}
+}

weed/storage/backend/memory_map/memory_map_windows.go

@@ -0,0 +1,330 @@
+//go:build windows
+
+package memory_map
+
+import (
+	"os"
+	"reflect"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+type DWORDLONG = uint64
+type DWORD = uint32
+type WORD = uint16
+
+var (
+	modkernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+	procGetSystemInfo            = modkernel32.NewProc("GetSystemInfo")
+	procGlobalMemoryStatusEx     = modkernel32.NewProc("GlobalMemoryStatusEx")
+	procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize")
+	procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize")
+)
+
+var currentProcess, _ = windows.GetCurrentProcess()
+var currentMinWorkingSet uint64 = 0
+var currentMaxWorkingSet uint64 = 0
+var _ = getProcessWorkingSetSize(uintptr(currentProcess), &currentMinWorkingSet, &currentMaxWorkingSet)
+
+var systemInfo, _ = getSystemInfo()
+var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 128
+
+var memoryStatusEx, _ = globalMemoryStatusEx()
+var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8)
+
+func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) {
+
+	chunks := (maxLength / chunkSize)
+	if chunks*chunkSize < maxLength {
+		chunks = chunks + 1
+	}
+
+	alignedMaxLength := chunks * chunkSize
+
+	maxLength_high := uint32(alignedMaxLength >> 32)
+	maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF)
+	file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil)
+
+	if err == nil {
+		mMap.File = file
+		mMap.file_memory_map_handle = uintptr(file_memory_map_handle)
+		mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize)
+		mMap.max_length = alignedMaxLength
+		mMap.End_of_file = -1
+	}
+}
+
+func (mMap *MemoryMap) DeleteFileAndMemoryMap() {
+	//First we close the file handles first to delete the file,
+	//Then we unmap the memory to ensure the unmapping process doesn't write the data to disk
+	windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle))
+	windows.CloseHandle(windows.Handle(mMap.File.Fd()))
+
+	for _, view := range mMap.write_map_views {
+		view.releaseMemory()
+	}
+
+	mMap.write_map_views = nil
+	mMap.max_length = 0
+}
+
+func min(x, y uint64) uint64 {
+	if x < y {
+		return x
+	}
+	return y
+}
+
+func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {
+
+	for {
+		if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) {
+			allocateChunk(mMap)
+		} else {
+			break
+		}
+	}
+
+	remaining_length := length
+	sliceIndex := offset / chunkSize
+	sliceOffset := offset - (sliceIndex * chunkSize)
+	dataOffset := uint64(0)
+
+	for {
+		writeEnd := min((remaining_length + sliceOffset), chunkSize)
+		copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:])
+		remaining_length -= (writeEnd - sliceOffset)
+		dataOffset += (writeEnd - sliceOffset)
+
+		if remaining_length > 0 {
+			sliceIndex += 1
+			sliceOffset = 0
+		} else {
+			break
+		}
+	}
+
+	if mMap.End_of_file < int64(offset+length-1) {
+		mMap.End_of_file = int64(offset + length - 1)
+	}
+}
+
+func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (dataSlice []byte, err error) {
+	dataSlice = make([]byte, length)
+	mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false)
+	copy(dataSlice, mBuffer.Buffer)
+	mBuffer.releaseMemory()
+	return dataSlice, err
+}
+
+func (mBuffer *MemoryBuffer) releaseMemory() {
+
+	windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
+	windows.UnmapViewOfFile(mBuffer.aligned_ptr)
+
+	currentMinWorkingSet -= mBuffer.aligned_length
+	currentMaxWorkingSet -= mBuffer.aligned_length
+
+	if currentMinWorkingSet < maxMemoryLimitBytes {
+		var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
+	}
+
+	mBuffer.ptr = 0
+	mBuffer.aligned_ptr = 0
+	mBuffer.length = 0
+	mBuffer.aligned_length = 0
+	mBuffer.Buffer = nil
+}
+
+func allocateChunk(mMap *MemoryMap) {
+	start := uint64(len(mMap.write_map_views)) * chunkSize
+	mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true)
+
+	if err == nil {
+		mMap.write_map_views = append(mMap.write_map_views, mBuffer)
+	}
+}
+
+func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) {
+
+	mBuffer := MemoryBuffer{}
+
+	//align memory allocations to the minimum virtual memory allocation size
+	dwSysGran := systemInfo.dwAllocationGranularity
+
+	start := (offset / uint64(dwSysGran)) * uint64(dwSysGran)
+	diff := offset - start
+	aligned_length := diff + length
+
+	offset_high := uint32(start >> 32)
+	offset_low := uint32(start & 0xFFFFFFFF)
+
+	access := windows.FILE_MAP_READ
+
+	if write {
+		access = windows.FILE_MAP_WRITE
+	}
+
+	currentMinWorkingSet += aligned_length
+	currentMaxWorkingSet += aligned_length
+
+	if currentMinWorkingSet < maxMemoryLimitBytes {
+		// increase the process working set size to hint to windows memory manager to
+		// prioritise keeping this memory mapped in physical memory over other standby memory
+		var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
+	}
+
+	addr_ptr, errno := windows.MapViewOfFile(hMapFile,
+		uint32(access), // read/write permission
+		offset_high,
+		offset_low,
+		uintptr(aligned_length))
+
+	if addr_ptr == 0 {
+		return mBuffer, errno
+	}
+
+	if currentMinWorkingSet < maxMemoryLimitBytes {
+		windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
+	}
+
+	mBuffer.aligned_ptr = addr_ptr
+	mBuffer.aligned_length = aligned_length
+	mBuffer.ptr = addr_ptr + uintptr(diff)
+	mBuffer.length = length
+
+	slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer))
+	slice_header.Data = addr_ptr + uintptr(diff)
+	slice_header.Len = int(length)
+	slice_header.Cap = int(length)
+
+	return mBuffer, nil
+}
+
+//typedef struct _MEMORYSTATUSEX {
+//	DWORD     dwLength;
+//	DWORD     dwMemoryLoad;
+//	DWORDLONG ullTotalPhys;
+//	DWORDLONG ullAvailPhys;
+//	DWORDLONG ullTotalPageFile;
+//	DWORDLONG ullAvailPageFile;
+//	DWORDLONG ullTotalVirtual;
+//	DWORDLONG ullAvailVirtual;
+//	DWORDLONG ullAvailExtendedVirtual;
+//  } MEMORYSTATUSEX, *LPMEMORYSTATUSEX;
+//https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
+
+type _MEMORYSTATUSEX struct {
+	dwLength                DWORD
+	dwMemoryLoad            DWORD
+	ullTotalPhys            DWORDLONG
+	ullAvailPhys            DWORDLONG
+	ullTotalPageFile        DWORDLONG
+	ullAvailPageFile        DWORDLONG
+	ullTotalVirtual         DWORDLONG
+	ullAvailVirtual         DWORDLONG
+	ullAvailExtendedVirtual DWORDLONG
+}
+
+// BOOL GlobalMemoryStatusEx(
+//
+//	LPMEMORYSTATUSEX lpBuffer
+//
+// );
+// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
+func globalMemoryStatusEx() (_MEMORYSTATUSEX, error) {
+	var mem_status _MEMORYSTATUSEX
+
+	mem_status.dwLength = uint32(unsafe.Sizeof(mem_status))
+	_, _, err := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&mem_status)))
+
+	if err != syscall.Errno(0) {
+		return mem_status, err
+	}
+	return mem_status, nil
+}
+
+//	typedef struct _SYSTEM_INFO {
+//	  union {
+//	    DWORD  dwOemId;
+//	    struct {
+//	      WORD wProcessorArchitecture;
+//	      WORD wReserved;
+//	    };
+//	  };
+//	  DWORD     dwPageSize;
+//	  LPVOID    lpMinimumApplicationAddress;
+//	  LPVOID    lpMaximumApplicationAddress;
+//	  DWORD_PTR dwActiveProcessorMask;
+//	  DWORD     dwNumberOfProcessors;
+//	  DWORD     dwProcessorType;
+//	  DWORD     dwAllocationGranularity;
+//	  WORD      wProcessorLevel;
+//	  WORD      wProcessorRevision;
+//	} SYSTEM_INFO;
+//
+// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info
+type _SYSTEM_INFO struct {
+	dwOemId                     DWORD
+	dwPageSize                  DWORD
+	lpMinimumApplicationAddress uintptr
+	lpMaximumApplicationAddress uintptr
+	dwActiveProcessorMask       uintptr
+	dwNumberOfProcessors        DWORD
+	dwProcessorType             DWORD
+	dwAllocationGranularity     DWORD
+	wProcessorLevel             WORD
+	wProcessorRevision          WORD
+}
+
+// void WINAPI GetSystemInfo(
+//
+//	_Out_ LPSYSTEM_INFO lpSystemInfo
+//
+// );
+// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsysteminfo
+func getSystemInfo() (_SYSTEM_INFO, error) {
+	var si _SYSTEM_INFO
+	_, _, err := procGetSystemInfo.Call(uintptr(unsafe.Pointer(&si)))
+	if err != syscall.Errno(0) {
+		return si, err
+	}
+	return si, nil
+}
+
+// BOOL GetProcessWorkingSetSize(
+//   HANDLE  hProcess,
+//   PSIZE_T lpMinimumWorkingSetSize,
+//   PSIZE_T lpMaximumWorkingSetSize
+// );
+// https://learn.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-getprocessworkingsetsize
+
+func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error {
+	r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet)))
+	if r1 == 0 {
+		if err != syscall.Errno(0) {
+			return err
+		}
+	}
+	return nil
+}
+
+// BOOL SetProcessWorkingSetSize(
+//   HANDLE hProcess,
+//   SIZE_T dwMinimumWorkingSetSize,
+//   SIZE_T dwMaximumWorkingSetSize
+// );
+// https://learn.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-setprocessworkingsetsize
+
+func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uint64, dwMaxWorkingSet uint64) error {
+	r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, uintptr(dwMinWorkingSet), uintptr(dwMaxWorkingSet))
+	if r1 == 0 {
+		if err != syscall.Errno(0) {
+			return err
+		}
+	}
+	return nil
+}

weed/storage/backend/memory_map/os_overloads/file_windows.go

@@ -0,0 +1,168 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os_overloads
+
+import (
+	"os"
+	"syscall"
+
+	"golang.org/x/sys/windows"
+)
+
+func isAbs(path string) (b bool) {
+	v := volumeName(path)
+	if v == "" {
+		return false
+	}
+	path = path[len(v):]
+	if path == "" {
+		return false
+	}
+	return os.IsPathSeparator(path[0])
+}
+
+func volumeName(path string) (v string) {
+	if len(path) < 2 {
+		return ""
+	}
+	// with drive letter
+	c := path[0]
+	if path[1] == ':' &&
+		('0' <= c && c <= '9' || 'a' <= c && c <= 'z' ||
+			'A' <= c && c <= 'Z') {
+		return path[:2]
+	}
+	// is it UNC
+	if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) &&
+		!os.IsPathSeparator(path[2]) && path[2] != '.' {
+		// first, leading `\\` and next shouldn't be `\`. its server name.
+		for n := 3; n < l-1; n++ {
+			// second, next '\' shouldn't be repeated.
+			if os.IsPathSeparator(path[n]) {
+				n++
+				// third, following something characters. its share name.
+				if !os.IsPathSeparator(path[n]) {
+					if path[n] == '.' {
+						break
+					}
+					for ; n < l; n++ {
+						if os.IsPathSeparator(path[n]) {
+							break
+						}
+					}
+					return path[:n]
+				}
+				break
+			}
+		}
+	}
+	return ""
+}
+
+// fixLongPath returns the extended-length (\\?\-prefixed) form of
+// path when needed, in order to avoid the default 260 character file
+// path limit imposed by Windows. If path is not easily converted to
+// the extended-length form (for example, if path is a relative path
+// or contains .. elements), or is short enough, fixLongPath returns
+// path unmodified.
+//
+// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
+func fixLongPath(path string) string {
+	// Do nothing (and don't allocate) if the path is "short".
+	// Empirically (at least on the Windows Server 2013 builder),
+	// the kernel is arbitrarily okay with < 248 bytes. That
+	// matches what the docs above say:
+	// "When using an API to create a directory, the specified
+	// path cannot be so long that you cannot append an 8.3 file
+	// name (that is, the directory name cannot exceed MAX_PATH
+	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
+	//
+	// The MSDN docs appear to say that a normal path that is 248 bytes long
+	// will work; empirically the path must be less then 248 bytes long.
+	if len(path) < 248 {
+		// Don't fix. (This is how Go 1.7 and earlier worked,
+		// not automatically generating the \\?\ form)
+		return path
+	}
+
+	// The extended form begins with \\?\, as in
+	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
+	// The extended form disables evaluation of . and .. path
+	// elements and disables the interpretation of / as equivalent
+	// to \. The conversion here rewrites / to \ and elides
+	// . elements as well as trailing or duplicate separators. For
+	// simplicity it avoids the conversion entirely for relative
+	// paths or paths containing .. elements. For now,
+	// \\server\share paths are not converted to
+	// \\?\UNC\server\share paths because the rules for doing so
+	// are less well-specified.
+	if len(path) >= 2 && path[:2] == `\\` {
+		// Don't canonicalize UNC paths.
+		return path
+	}
+	if !isAbs(path) {
+		// Relative path
+		return path
+	}
+
+	const prefix = `\\?`
+
+	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
+	copy(pathbuf, prefix)
+	n := len(path)
+	r, w := 0, len(prefix)
+	for r < n {
+		switch {
+		case os.IsPathSeparator(path[r]):
+			// empty block
+			r++
+		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
+			// /./
+			r++
+		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
+			// /../ is currently unhandled
+			return path
+		default:
+			pathbuf[w] = '\\'
+			w++
+			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
+				pathbuf[w] = path[r]
+				w++
+			}
+		}
+	}
+	// A drive's root directory needs a trailing \
+	if w == len(`\\?\c:`) {
+		pathbuf[w] = '\\'
+		w++
+	}
+	return string(pathbuf[:w])
+}
+
+// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
+func syscallMode(i os.FileMode) (o uint32) {
+	o |= uint32(i.Perm())
+	if i&os.ModeSetuid != 0 {
+		o |= syscall.S_ISUID
+	}
+	if i&os.ModeSetgid != 0 {
+		o |= syscall.S_ISGID
+	}
+	if i&os.ModeSticky != 0 {
+		o |= syscall.S_ISVTX
+	}
+	// No mapping for Go's ModeTemporary (plan9 only).
+	return
+}
+
+// If the bool is set to true then the file is opened with the parameters FILE_ATTRIBUTE_TEMPORARY and
+// FILE_FLAG_DELETE_ON_CLOSE
+func OpenFile(name string, flag int, perm os.FileMode, setToTempAndDelete bool) (file *os.File, err error) {
+	r, e := Open(fixLongPath(name), flag|windows.O_CLOEXEC, syscallMode(perm), setToTempAndDelete)
+	if e != nil {
+		return nil, e
+	}
+	return os.NewFile(uintptr(r), name), nil
+}

weed/storage/backend/memory_map/os_overloads/syscall_windows.go

@@ -0,0 +1,80 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Windows system calls.
+
+package os_overloads
+
+import (
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+)
+
+// windows api calls
+
+//sys	CreateFile(name *uint16, access uint32, mode uint32, sa *SecurityAttributes, createmode uint32, attrs uint32, templatefile int32) (handle Handle, err error) [failretval==InvalidHandle] = CreateFileW
+
+func makeInheritSa() *syscall.SecurityAttributes {
+	var sa syscall.SecurityAttributes
+	sa.Length = uint32(unsafe.Sizeof(sa))
+	sa.InheritHandle = 1
+	return &sa
+}
+
+// opens the
+func Open(path string, mode int, perm uint32, setToTempAndDelete bool) (fd syscall.Handle, err error) {
+	if len(path) == 0 {
+		return syscall.InvalidHandle, windows.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return syscall.InvalidHandle, err
+	}
+	var access uint32
+	switch mode & (windows.O_RDONLY | windows.O_WRONLY | windows.O_RDWR) {
+	case windows.O_RDONLY:
+		access = windows.GENERIC_READ
+	case windows.O_WRONLY:
+		access = windows.GENERIC_WRITE
+	case windows.O_RDWR:
+		access = windows.GENERIC_READ | windows.GENERIC_WRITE
+	}
+	if mode&windows.O_CREAT != 0 {
+		access |= windows.GENERIC_WRITE
+	}
+	if mode&windows.O_APPEND != 0 {
+		access &^= windows.GENERIC_WRITE
+		access |= windows.FILE_APPEND_DATA
+	}
+	sharemode := uint32(windows.FILE_SHARE_READ | windows.FILE_SHARE_WRITE)
+	var sa *syscall.SecurityAttributes
+	if mode&windows.O_CLOEXEC == 0 {
+		sa = makeInheritSa()
+	}
+	var createmode uint32
+	switch {
+	case mode&(windows.O_CREAT|windows.O_EXCL) == (windows.O_CREAT | windows.O_EXCL):
+		createmode = windows.CREATE_NEW
+	case mode&(windows.O_CREAT|windows.O_TRUNC) == (windows.O_CREAT | windows.O_TRUNC):
+		createmode = windows.CREATE_ALWAYS
+	case mode&windows.O_CREAT == windows.O_CREAT:
+		createmode = windows.OPEN_ALWAYS
+	case mode&windows.O_TRUNC == windows.O_TRUNC:
+		createmode = windows.TRUNCATE_EXISTING
+	default:
+		createmode = windows.OPEN_EXISTING
+	}
+
+	var h syscall.Handle
+	var e error
+
+	if setToTempAndDelete {
+		h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, (windows.FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE), 0)
+	} else {
+		h, e = syscall.CreateFile(pathp, access, sharemode, sa, createmode, windows.FILE_ATTRIBUTE_NORMAL, 0)
+	}
+	return h, e
+}

weed/storage/backend/memory_map/os_overloads/types_windows.go

@@ -0,0 +1,9 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package os_overloads
+
+const (
+	FILE_FLAG_DELETE_ON_CLOSE = 0x04000000
+)

weed/storage/backend/rclone_backend/dummy.go

@@ -0,0 +1 @@
+package rclone_backend

weed/storage/backend/rclone_backend/rclone_backend.go

@@ -0,0 +1,295 @@
+//go:build rclone
+
+package rclone_backend
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/google/uuid"
+	_ "github.com/rclone/rclone/backend/all"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config/configfile"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func init() {
+	backend.BackendStorageFactories["rclone"] = &RcloneBackendFactory{}
+	configfile.Install()
+}
+
+type RcloneBackendFactory struct {
+}
+
+func (factory *RcloneBackendFactory) StorageType() backend.StorageType {
+	return "rclone"
+}
+
+func (factory *RcloneBackendFactory) BuildStorage(configuration backend.StringProperties, configPrefix string, id string) (backend.BackendStorage, error) {
+	return newRcloneBackendStorage(configuration, configPrefix, id)
+}
+
+type RcloneBackendStorage struct {
+	id              string
+	remoteName      string
+	keyTemplate     *template.Template
+	keyTemplateText string
+	fs              fs.Fs
+}
+
+func newRcloneBackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *RcloneBackendStorage, err error) {
+	s = &RcloneBackendStorage{}
+	s.id = id
+	s.remoteName = configuration.GetString(configPrefix + "remote_name")
+	s.keyTemplateText = configuration.GetString(configPrefix + "key_template")
+	s.keyTemplate, err = template.New("keyTemplate").Parse(s.keyTemplateText)
+	if err != nil {
+		return
+	}
+
+	ctx := context.TODO()
+	accounting.Start(ctx)
+
+	fsPath := fmt.Sprintf("%s:", s.remoteName)
+	s.fs, err = fs.NewFs(ctx, fsPath)
+	if err != nil {
+		glog.Errorf("failed to instantiate Rclone filesystem: %s", err)
+		return
+	}
+
+	glog.V(0).Infof("created backend storage rclone.%s for remote name %s", s.id, s.remoteName)
+	return
+}
+
+func (s *RcloneBackendStorage) ToProperties() map[string]string {
+	m := make(map[string]string)
+	m["remote_name"] = s.remoteName
+	if len(s.keyTemplateText) > 0 {
+		m["key_template"] = s.keyTemplateText
+	}
+	return m
+}
+
+func formatKey(key string, storage RcloneBackendStorage) (fKey string, err error) {
+	var b bytes.Buffer
+	if len(storage.keyTemplateText) == 0 {
+		fKey = key
+	} else {
+		err = storage.keyTemplate.Execute(&b, key)
+		if err == nil {
+			fKey = b.String()
+		}
+	}
+	return
+}
+
+func (s *RcloneBackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
+	f := &RcloneBackendStorageFile{
+		backendStorage: s,
+		key:            key,
+		tierInfo:       tierInfo,
+	}
+
+	return f
+}
+
+func (s *RcloneBackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error) {
+	randomUuid, err := uuid.NewRandom()
+	if err != nil {
+		return key, 0, err
+	}
+	key = randomUuid.String()
+
+	key, err = formatKey(key, *s)
+	if err != nil {
+		return key, 0, err
+	}
+
+	glog.V(1).Infof("copy dat file of %s to remote rclone.%s as %s", f.Name(), s.id, key)
+
+	util.Retry("upload via Rclone", func() error {
+		size, err = uploadViaRclone(s.fs, f.Name(), key, fn)
+		return err
+	})
+
+	return
+}
+
+func uploadViaRclone(rfs fs.Fs, filename string, key string, fn func(progressed int64, percentage float32) error) (fileSize int64, err error) {
+	ctx := context.TODO()
+
+	file, err := os.Open(filename)
+	if err != nil {
+		return 0, err
+	}
+	defer file.Close()
+
+	stat, err := file.Stat()
+	if err != nil {
+		return 0, err
+	}
+
+	info := object.NewStaticObjectInfo(key, stat.ModTime(), stat.Size(), true, nil, rfs)
+
+	tr := accounting.NewStats(ctx).NewTransfer(info, rfs)
+	defer tr.Done(ctx, err)
+	acc := tr.Account(ctx, file)
+	pr := ProgressReader{acc: acc, tr: tr, fn: fn}
+
+	obj, err := rfs.Put(ctx, &pr, info)
+	if err != nil {
+		return 0, err
+	}
+
+	return obj.Size(), err
+}
+
+func (s *RcloneBackendStorage) DownloadFile(filename string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error) {
+	glog.V(1).Infof("download dat file of %s from remote rclone.%s as %s", filename, s.id, key)
+
+	util.Retry("download via Rclone", func() error {
+		size, err = downloadViaRclone(s.fs, filename, key, fn)
+		return err
+	})
+
+	return
+}
+
+var createRcloneDownloadFile = func(filename string) (io.WriteCloser, error) {
+	return os.Create(filename)
+}
+
+func downloadViaRclone(fs fs.Fs, filename string, key string, fn func(progressed int64, percentage float32) error) (fileSize int64, err error) {
+	ctx := context.TODO()
+
+	obj, err := fs.NewObject(ctx, key)
+	if err != nil {
+		return 0, err
+	}
+
+	rc, err := obj.Open(ctx)
+	if err != nil {
+		return 0, err
+	}
+	defer rc.Close()
+
+	file, err := createRcloneDownloadFile(filename)
+	if err != nil {
+		return 0, err
+	}
+
+	tr := accounting.NewStats(ctx).NewTransfer(obj, fs)
+	defer func() {
+		// fsync the .dat before closing so its content is durable before the caller
+		// trims the remote reference and deletes the shared remote object.
+		if syncer, ok := file.(interface{ Sync() error }); ok {
+			if syncErr := syncer.Sync(); err == nil && syncErr != nil {
+				err = syncErr
+			}
+		}
+		if closeErr := file.Close(); err == nil && closeErr != nil {
+			err = closeErr
+		}
+		tr.Done(ctx, err)
+	}()
+	acc := tr.Account(ctx, rc)
+	pr := ProgressReader{acc: acc, tr: tr, fn: fn}
+
+	written, err := io.Copy(file, &pr)
+	if err != nil {
+		return 0, err
+	}
+
+	return written, nil
+}
+
+func (s *RcloneBackendStorage) DeleteFile(key string) (err error) {
+	glog.V(1).Infof("delete dat file %s from remote", key)
+
+	util.Retry("delete via Rclone", func() error {
+		err = deleteViaRclone(s.fs, key)
+		return err
+	})
+
+	return
+}
+
+func deleteViaRclone(fs fs.Fs, key string) (err error) {
+	ctx := context.TODO()
+
+	obj, err := fs.NewObject(ctx, key)
+	if err != nil {
+		return err
+	}
+
+	return obj.Remove(ctx)
+}
+
+type RcloneBackendStorageFile struct {
+	backendStorage *RcloneBackendStorage
+	key            string
+	tierInfo       *volume_server_pb.VolumeInfo
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {
+	ctx := context.TODO()
+
+	obj, err := rcloneBackendStorageFile.backendStorage.fs.NewObject(ctx, rcloneBackendStorageFile.key)
+	if err != nil {
+		return 0, err
+	}
+
+	opt := fs.RangeOption{Start: off, End: off + int64(len(p)) - 1}
+
+	rc, err := obj.Open(ctx, &opt)
+	if err != nil {
+		return 0, err
+	}
+	defer rc.Close()
+
+	return io.ReadFull(rc, p)
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
+	panic("not implemented")
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) Truncate(off int64) error {
+	panic("not implemented")
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) Close() error {
+	return nil
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
+	files := rcloneBackendStorageFile.tierInfo.GetFiles()
+
+	if len(files) == 0 {
+		err = fmt.Errorf("remote file info not found")
+		return
+	}
+
+	datSize = int64(files[0].FileSize)
+	modTime = time.Unix(int64(files[0].ModifiedTime), 0)
+
+	return
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) Name() string {
+	return rcloneBackendStorageFile.key
+}
+
+func (rcloneBackendStorageFile RcloneBackendStorageFile) Sync() error {
+	return nil
+}

weed/storage/backend/rclone_backend/rclone_backend_test.go

@@ -0,0 +1,173 @@
+//go:build rclone
+
+package rclone_backend
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fstest/mockfs"
+)
+
+func TestUploadViaRcloneReturnsOpenErrorWithoutPanic(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "missing.dat")
+
+	_, err := uploadViaRclone(nil, missing, "key", nil)
+	if err == nil {
+		t.Fatal("expected open error")
+	}
+	if !os.IsNotExist(err) {
+		t.Fatalf("expected not-exist error, got %v", err)
+	}
+}
+
+func TestDownloadViaRcloneReturnsObjectOpenErrorWithoutPanic(t *testing.T) {
+	openErr := errors.New("open failed")
+	rfs := newTestRcloneFs(t, &testRcloneObject{remote: "key", openErr: openErr})
+
+	_, err := downloadViaRclone(rfs, filepath.Join(t.TempDir(), "out.dat"), "key", nil)
+	if !errors.Is(err, openErr) {
+		t.Fatalf("expected %v, got %v", openErr, err)
+	}
+}
+
+func TestDownloadViaRcloneReturnsCreateErrorWithoutPanic(t *testing.T) {
+	rfs := newTestRcloneFs(t, &testRcloneObject{
+		remote: "key",
+		body:   "data",
+	})
+
+	_, err := downloadViaRclone(rfs, t.TempDir(), "key", nil)
+	if err == nil {
+		t.Fatal("expected create error")
+	}
+}
+
+func TestDownloadViaRcloneReturnsFileCloseError(t *testing.T) {
+	closeErr := errors.New("close failed")
+	rfs := newTestRcloneFs(t, &testRcloneObject{
+		remote: "key",
+		body:   "data",
+	})
+
+	previousCreateFile := createRcloneDownloadFile
+	createRcloneDownloadFile = func(string) (io.WriteCloser, error) {
+		return &closeErrorWriter{closeErr: closeErr}, nil
+	}
+	t.Cleanup(func() {
+		createRcloneDownloadFile = previousCreateFile
+	})
+
+	_, err := downloadViaRclone(rfs, "ignored", "key", func(int64, float32) error {
+		return nil
+	})
+	if !errors.Is(err, closeErr) {
+		t.Fatalf("expected %v, got %v", closeErr, err)
+	}
+}
+
+func TestRcloneBackendStorageFileReadAtReturnsOpenErrorWithoutPanic(t *testing.T) {
+	openErr := errors.New("range open failed")
+	rfs := newTestRcloneFs(t, &testRcloneObject{remote: "key", openErr: openErr})
+	storageFile := RcloneBackendStorageFile{
+		backendStorage: &RcloneBackendStorage{fs: rfs},
+		key:            "key",
+	}
+
+	_, err := storageFile.ReadAt(make([]byte, 4), 0)
+	if !errors.Is(err, openErr) {
+		t.Fatalf("expected %v, got %v", openErr, err)
+	}
+}
+
+func newTestRcloneFs(t *testing.T, objects ...fs.Object) fs.Fs {
+	t.Helper()
+
+	rfs, err := mockfs.NewFs(context.Background(), "mock", "/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	mfs := rfs.(*mockfs.Fs)
+	for _, obj := range objects {
+		mfs.AddObject(obj)
+	}
+	return rfs
+}
+
+type testRcloneObject struct {
+	remote  string
+	body    string
+	openErr error
+	rfs     fs.Info
+}
+
+func (o *testRcloneObject) SetFs(rfs fs.Fs) {
+	o.rfs = rfs
+}
+
+func (o *testRcloneObject) Fs() fs.Info {
+	return o.rfs
+}
+
+func (o *testRcloneObject) String() string {
+	return o.remote
+}
+
+func (o *testRcloneObject) Remote() string {
+	return o.remote
+}
+
+func (o *testRcloneObject) ModTime(context.Context) time.Time {
+	return time.Unix(0, 0)
+}
+
+func (o *testRcloneObject) Size() int64 {
+	return int64(len(o.body))
+}
+
+func (o *testRcloneObject) Hash(context.Context, hash.Type) (string, error) {
+	return "", nil
+}
+
+func (o *testRcloneObject) Storable() bool {
+	return true
+}
+
+func (o *testRcloneObject) SetModTime(context.Context, time.Time) error {
+	return nil
+}
+
+func (o *testRcloneObject) Open(context.Context, ...fs.OpenOption) (io.ReadCloser, error) {
+	if o.openErr != nil {
+		return nil, o.openErr
+	}
+	return io.NopCloser(strings.NewReader(o.body)), nil
+}
+
+func (o *testRcloneObject) Update(context.Context, io.Reader, fs.ObjectInfo, ...fs.OpenOption) error {
+	return nil
+}
+
+func (o *testRcloneObject) Remove(context.Context) error {
+	return nil
+}
+
+type closeErrorWriter struct {
+	closeErr error
+}
+
+func (w *closeErrorWriter) Write(p []byte) (int, error) {
+	return len(p), nil
+}
+
+func (w *closeErrorWriter) Close() error {
+	return w.closeErr
+}

weed/storage/backend/rclone_backend/rclone_progress.go

@@ -0,0 +1,21 @@
+//go:build rclone
+
+package rclone_backend
+
+import "github.com/rclone/rclone/fs/accounting"
+
+type ProgressReader struct {
+	acc *accounting.Account
+	tr  *accounting.Transfer
+	fn  func(progressed int64, percentage float32) error
+}
+
+func (pr *ProgressReader) Read(p []byte) (n int, err error) {
+	n, err = pr.acc.Read(p)
+	if err != nil {
+		return
+	}
+	snap := pr.tr.Snapshot()
+	err = pr.fn(snap.Bytes, 100*float32(snap.Bytes)/float32(snap.Size))
+	return
+}

weed/storage/backend/s3_backend/s3_backend.go

@@ -0,0 +1,247 @@
+package s3_backend
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/google/uuid"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend"
+)
+
+func init() {
+	backend.BackendStorageFactories["s3"] = &S3BackendFactory{}
+}
+
+type S3BackendFactory struct {
+}
+
+func (factory *S3BackendFactory) StorageType() backend.StorageType {
+	return backend.StorageType("s3")
+}
+func (factory *S3BackendFactory) BuildStorage(configuration backend.StringProperties, configPrefix string, id string) (backend.BackendStorage, error) {
+	return newS3BackendStorage(configuration, configPrefix, id)
+}
+
+type S3BackendStorage struct {
+	id                    string
+	aws_access_key_id     string
+	aws_secret_access_key string
+	region                string
+	bucket                string
+	endpoint              string
+	storageClass          string
+	forcePathStyle        bool
+	uploadConcurrency     int
+	downloadConcurrency   int
+	conn                  s3iface.S3API
+}
+
+func newS3BackendStorage(configuration backend.StringProperties, configPrefix string, id string) (s *S3BackendStorage, err error) {
+	s = &S3BackendStorage{}
+	s.id = id
+	s.aws_access_key_id = configuration.GetString(configPrefix + "aws_access_key_id")
+	s.aws_secret_access_key = configuration.GetString(configPrefix + "aws_secret_access_key")
+	s.region = configuration.GetString(configPrefix + "region")
+	s.bucket = configuration.GetString(configPrefix + "bucket")
+	s.endpoint = configuration.GetString(configPrefix + "endpoint")
+	s.storageClass = configuration.GetString(configPrefix + "storage_class")
+	s.forcePathStyle = util.ParseBool(configuration.GetString(configPrefix+"force_path_style"), true)
+	s.uploadConcurrency = parseConcurrency(configuration.GetString(configPrefix+"upload_concurrency"), defaultUploadConcurrency)
+	s.downloadConcurrency = parseConcurrency(configuration.GetString(configPrefix+"download_concurrency"), defaultDownloadConcurrency)
+	if s.storageClass == "" {
+		s.storageClass = "STANDARD_IA"
+	}
+
+	s.conn, err = createSession(s.aws_access_key_id, s.aws_secret_access_key, s.region, s.endpoint, s.forcePathStyle)
+
+	glog.V(0).Infof("created backend storage s3.%s for region %s bucket %s", s.id, s.region, s.bucket)
+	return
+}
+
+func (s *S3BackendStorage) ToProperties() map[string]string {
+	m := make(map[string]string)
+	m["aws_access_key_id"] = s.aws_access_key_id
+	m["aws_secret_access_key"] = s.aws_secret_access_key
+	m["region"] = s.region
+	m["bucket"] = s.bucket
+	m["endpoint"] = s.endpoint
+	m["storage_class"] = s.storageClass
+	m["force_path_style"] = util.BoolToString(s.forcePathStyle)
+	m["upload_concurrency"] = strconv.Itoa(s.uploadConcurrency)
+	m["download_concurrency"] = strconv.Itoa(s.downloadConcurrency)
+	return m
+}
+
+const (
+	defaultUploadConcurrency   = 5
+	defaultDownloadConcurrency = 5
+)
+
+func parseConcurrency(value string, def int) int {
+	if value == "" {
+		return def
+	}
+	if n, err := strconv.Atoi(value); err == nil && n > 0 {
+		return n
+	}
+	glog.Warningf("invalid concurrency value %q, using default %d", value, def)
+	return def
+}
+
+func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) backend.BackendStorageFile {
+	if strings.HasPrefix(key, "/") {
+		key = key[1:]
+	}
+
+	f := &S3BackendStorageFile{
+		backendStorage: s,
+		key:            key,
+		tierInfo:       tierInfo,
+	}
+
+	return f
+}
+
+func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error) {
+	randomUuid, _ := uuid.NewRandom()
+	key = randomUuid.String()
+
+	if concurrency <= 0 {
+		concurrency = s.uploadConcurrency
+	}
+
+	glog.V(1).Infof("copying dat file of %s to remote s3.%s as %s", f.Name(), s.id, key)
+
+	util.Retry("upload to S3", func() error {
+		size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, s.storageClass, fn, concurrency)
+		return err
+	})
+
+	return
+}
+
+func (s *S3BackendStorage) DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error) {
+	if concurrency <= 0 {
+		concurrency = s.downloadConcurrency
+	}
+
+	glog.V(1).Infof("download dat file of %s from remote s3.%s as %s", fileName, s.id, key)
+
+	size, err = downloadFromS3(s.conn, fileName, s.bucket, key, fn, concurrency)
+
+	return
+}
+
+func (s *S3BackendStorage) DeleteFile(key string) (err error) {
+
+	glog.V(1).Infof("delete dat file %s from remote", key)
+
+	err = deleteFromS3(s.conn, s.bucket, key)
+
+	return
+}
+
+type S3BackendStorageFile struct {
+	backendStorage *S3BackendStorage
+	key            string
+	tierInfo       *volume_server_pb.VolumeInfo
+}
+
+func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {
+	if off < 0 {
+		return 0, fmt.Errorf("negative offset %d", off)
+	}
+
+	requested := len(p)
+	if requested == 0 {
+		return 0, nil
+	}
+
+	datSize, _, _ := s3backendStorageFile.GetStat()
+
+	if datSize > 0 && off >= datSize {
+		return 0, io.EOF
+	}
+
+	bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)
+
+	getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
+		Bucket: &s3backendStorageFile.backendStorage.bucket,
+		Key:    &s3backendStorageFile.key,
+		Range:  &bytesRange,
+	})
+
+	if getObjectErr != nil {
+		return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
+	}
+	defer getObjectOutput.Body.Close()
+
+	// glog.V(3).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
+	// glog.V(3).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)
+
+	var readCount int
+	for {
+		p = p[readCount:]
+		if len(p) == 0 {
+			break
+		}
+		readCount, err = getObjectOutput.Body.Read(p)
+		n += readCount
+
+		if err != nil {
+			break
+		}
+	}
+
+	if err == io.EOF && n == requested {
+		err = nil
+	}
+
+	return
+}
+
+func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
+	panic("not implemented")
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Truncate(off int64) error {
+	panic("not implemented")
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Close() error {
+	return nil
+}
+
+func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
+
+	files := s3backendStorageFile.tierInfo.GetFiles()
+
+	if len(files) == 0 {
+		err = fmt.Errorf("remote file info not found")
+		return
+	}
+
+	datSize = int64(files[0].FileSize)
+	modTime = time.Unix(int64(files[0].ModifiedTime), 0)
+
+	return
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Name() string {
+	return s3backendStorageFile.key
+}
+
+func (s3backendStorageFile S3BackendStorageFile) Sync() error {
+	return nil
+}

weed/storage/backend/s3_backend/s3_backend_test.go

@@ -0,0 +1,156 @@
+package s3_backend
+
+import (
+	"errors"
+	"io"
+	"strings"
+	"testing"
+	"testing/iotest"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+)
+
+type testProperties map[string]string
+
+func (m testProperties) GetString(key string) string {
+	return m[key]
+}
+
+type stubS3Client struct {
+	s3iface.S3API
+	getObject func(*s3.GetObjectInput) (*s3.GetObjectOutput, error)
+}
+
+func (s *stubS3Client) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	return s.getObject(input)
+}
+
+func TestReadAtReturnsEOFForShortRead(t *testing.T) {
+	client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+		return &s3.GetObjectOutput{Body: io.NopCloser(strings.NewReader("abc"))}, nil
+	}}
+	file := S3BackendStorageFile{
+		backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+		key:            "key",
+		tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+			{FileSize: 10},
+		}},
+	}
+
+	buffer := make([]byte, 5)
+	n, err := file.ReadAt(buffer, 0)
+	if n != 3 {
+		t.Fatalf("ReadAt() read %d bytes, want 3", n)
+	}
+	if !errors.Is(err, io.EOF) {
+		t.Fatalf("ReadAt() error = %v, want io.EOF", err)
+	}
+}
+
+func TestReadAtClearsEOFOnFullRead(t *testing.T) {
+	client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+		return &s3.GetObjectOutput{Body: io.NopCloser(iotest.DataErrReader(strings.NewReader("abcde")))}, nil
+	}}
+	file := S3BackendStorageFile{
+		backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+		key:            "key",
+		tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+			{FileSize: 10},
+		}},
+	}
+
+	buffer := make([]byte, 5)
+	n, err := file.ReadAt(buffer, 0)
+	if n != 5 || err != nil {
+		t.Fatalf("ReadAt() = (%d, %v), want (5, nil)", n, err)
+	}
+	if string(buffer) != "abcde" {
+		t.Fatalf("ReadAt() buffer = %q, want %q", buffer, "abcde")
+	}
+}
+
+func TestReadAtRejectsInvalidRequestsLocally(t *testing.T) {
+	tests := []struct {
+		name    string
+		buffer  []byte
+		offset  int64
+		wantErr bool
+	}{
+		{name: "empty buffer", buffer: nil},
+		{name: "negative offset", buffer: make([]byte, 1), offset: -1, wantErr: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			called := false
+			client := &stubS3Client{getObject: func(*s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+				called = true
+				return nil, errors.New("unexpected GetObject")
+			}}
+			file := S3BackendStorageFile{
+				backendStorage: &S3BackendStorage{conn: client, bucket: "bucket"},
+				key:            "key",
+				tierInfo: &volume_server_pb.VolumeInfo{Files: []*volume_server_pb.RemoteFile{
+					{FileSize: 10},
+				}},
+			}
+
+			n, err := file.ReadAt(tt.buffer, tt.offset)
+			if n != 0 || (err != nil) != tt.wantErr {
+				t.Fatalf("ReadAt() = (%d, %v), want error %t", n, err, tt.wantErr)
+			}
+			if called {
+				t.Fatal("ReadAt() called S3 for an invalid request")
+			}
+		})
+	}
+}
+
+func TestParseConcurrency(t *testing.T) {
+	cases := []struct {
+		value    string
+		def      int
+		expected int
+	}{
+		{"", 5, 5},    // unset -> default
+		{"0", 5, 5},   // explicit zero means default
+		{"-3", 5, 5},  // invalid -> default
+		{"abc", 5, 5}, // invalid -> default
+		{"1", 5, 1},   // override
+		{"64", 5, 64}, // override
+	}
+	for _, tt := range cases {
+		if got := parseConcurrency(tt.value, tt.def); got != tt.expected {
+			t.Errorf("parseConcurrency(%q, %d) = %d, want %d", tt.value, tt.def, got, tt.expected)
+		}
+	}
+}
+
+func TestS3BackendStorageConcurrencyConfigRoundTrip(t *testing.T) {
+	s, err := newS3BackendStorage(testProperties{}, "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.uploadConcurrency != defaultUploadConcurrency || s.downloadConcurrency != defaultDownloadConcurrency {
+		t.Fatalf("defaults: upload=%d download=%d, want %d/%d",
+			s.uploadConcurrency, s.downloadConcurrency, defaultUploadConcurrency, defaultDownloadConcurrency)
+	}
+
+	s, err = newS3BackendStorage(testProperties{
+		"upload_concurrency":   "1",
+		"download_concurrency": "17",
+	}, "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.uploadConcurrency != 1 || s.downloadConcurrency != 17 {
+		t.Fatalf("configured: upload=%d download=%d, want 1/17", s.uploadConcurrency, s.downloadConcurrency)
+	}
+
+	props := s.ToProperties()
+	if props["upload_concurrency"] != "1" || props["download_concurrency"] != "17" {
+		t.Fatalf("ToProperties: %v", props)
+	}
+}

weed/storage/backend/s3_backend/s3_download.go

@@ -0,0 +1,107 @@
+package s3_backend
+
+import (
+	"fmt"
+	"os"
+	"sync/atomic"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func downloadFromS3(sess s3iface.S3API, destFileName string, sourceBucket string, sourceKey string,
+	fn func(progressed int64, percentage float32) error, concurrency int) (fileSize int64, err error) {
+	if concurrency <= 0 {
+		concurrency = defaultDownloadConcurrency
+	}
+
+	fileSize, err = getFileSize(sess, sourceBucket, sourceKey)
+	if err != nil {
+		return
+	}
+
+	//open the file
+	f, err := os.OpenFile(destFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %q, %v", destFileName, err)
+	}
+	defer f.Close()
+
+	// Create a downloader with the session and custom options
+	downloader := s3manager.NewDownloaderWithClient(sess, func(u *s3manager.Downloader) {
+		u.PartSize = int64(64 * 1024 * 1024)
+		u.Concurrency = concurrency
+	})
+
+	fileWriter := &s3DownloadProgressedWriter{
+		fp:      f,
+		size:    fileSize,
+		written: 0,
+		fn:      fn,
+	}
+
+	// Download the file from S3.
+	fileSize, err = downloader.Download(fileWriter, &s3.GetObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	if err != nil {
+		return fileSize, fmt.Errorf("failed to download /buckets/%s/%s to %s: %v", sourceBucket, sourceKey, destFileName, err)
+	}
+
+	// fsync the downloaded .dat so its content is durable before the caller trims the
+	// remote reference and deletes the shared remote object.
+	if syncErr := f.Sync(); syncErr != nil {
+		return fileSize, fmt.Errorf("failed to fsync %s: %v", destFileName, syncErr)
+	}
+
+	glog.V(1).Infof("downloaded file %s\n", destFileName)
+
+	return
+}
+
+// adapted from https://github.com/aws/aws-sdk-go/pull/1868
+// and https://petersouter.xyz/s3-download-progress-bar-in-golang/
+type s3DownloadProgressedWriter struct {
+	size    int64
+	written int64
+	fn      func(progressed int64, percentage float32) error
+	fp      *os.File
+}
+
+func (w *s3DownloadProgressedWriter) WriteAt(p []byte, off int64) (int, error) {
+	n, err := w.fp.WriteAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	// Got the length have read( or means has uploaded), and you can construct your message
+	atomic.AddInt64(&w.written, int64(n))
+
+	if w.fn != nil {
+		written := w.written
+		if err := w.fn(written, float32(written*100)/float32(w.size)); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+func getFileSize(svc s3iface.S3API, bucket string, key string) (filesize int64, error error) {
+	params := &s3.HeadObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+
+	resp, err := svc.HeadObject(params)
+	if err != nil {
+		return 0, err
+	}
+
+	return *resp.ContentLength, nil
+}

weed/storage/backend/s3_backend/s3_sessions.go

@@ -0,0 +1,63 @@
+package s3_backend
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+)
+
+var (
+	s3Sessions   = make(map[string]s3iface.S3API)
+	sessionsLock sync.Mutex
+)
+
+func createSession(awsAccessKeyId, awsSecretAccessKey, region, endpoint string, forcePathStyle bool) (s3iface.S3API, error) {
+
+	sessionsLock.Lock()
+	defer sessionsLock.Unlock()
+
+	cacheKey := fmt.Sprintf("%s|%s", region, endpoint)
+	if t, found := s3Sessions[cacheKey]; found {
+		return t, nil
+	}
+
+	config := &aws.Config{
+		Region:                        aws.String(region),
+		Endpoint:                      aws.String(endpoint),
+		S3ForcePathStyle:              aws.Bool(forcePathStyle),
+		S3DisableContentMD5Validation: aws.Bool(true),
+	}
+	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
+		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
+	}
+
+	sess, err := session.NewSession(config)
+	if err != nil {
+		return nil, fmt.Errorf("create aws session in region %s: %v", region, err)
+	}
+	sess.Handlers.Build.PushBack(func(r *request.Request) {
+		r.HTTPRequest.Header.Set("User-Agent", "SeaweedFS/"+version.VERSION_NUMBER)
+	})
+
+	t := s3.New(sess)
+
+	s3Sessions[cacheKey] = t
+
+	return t, nil
+
+}
+
+func deleteFromS3(sess s3iface.S3API, sourceBucket string, sourceKey string) (err error) {
+	_, err = sess.DeleteObject(&s3.DeleteObjectInput{
+		Bucket: aws.String(sourceBucket),
+		Key:    aws.String(sourceKey),
+	})
+	return err
+}

weed/storage/backend/s3_backend/s3_upload.go

@@ -0,0 +1,112 @@
+package s3_backend
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string, storageClass string, fn func(progressed int64, percentage float32) error, concurrency int) (fileSize int64, err error) {
+	if concurrency <= 0 {
+		concurrency = defaultUploadConcurrency
+	}
+
+	//open the file
+	f, err := os.Open(filename)
+	if err != nil {
+		return 0, fmt.Errorf("failed to open file %q, %v", filename, err)
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return 0, fmt.Errorf("failed to stat file %q, %v", filename, err)
+	}
+
+	fileSize = info.Size()
+
+	partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB
+	for partSize*1000 < fileSize {
+		partSize *= 4
+	}
+
+	// Create an uploader with the session and custom options
+	uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) {
+		u.PartSize = partSize
+		u.Concurrency = concurrency
+	})
+
+	fileReader := &s3UploadProgressedReader{
+		fp:      f,
+		size:    fileSize,
+		signMap: map[int64]struct{}{},
+		fn:      fn,
+	}
+
+	// Upload the file to S3.
+	var result *s3manager.UploadOutput
+	result, err = uploader.Upload(&s3manager.UploadInput{
+		Bucket:       aws.String(destBucket),
+		Key:          aws.String(destKey),
+		Body:         fileReader,
+		StorageClass: aws.String(storageClass),
+	})
+
+	//in case it fails to upload
+	if err != nil {
+		return 0, fmt.Errorf("failed to upload file %s: %v", filename, err)
+	}
+	glog.V(1).Infof("file %s uploaded to %s\n", filename, result.Location)
+
+	return
+}
+
+// adapted from https://github.com/aws/aws-sdk-go/pull/1868
+// https://github.com/aws/aws-sdk-go/blob/main/example/service/s3/putObjectWithProcess/putObjWithProcess.go
+type s3UploadProgressedReader struct {
+	fp      *os.File
+	size    int64
+	read    int64
+	signMap map[int64]struct{}
+	mux     sync.Mutex
+	fn      func(progressed int64, percentage float32) error
+}
+
+func (r *s3UploadProgressedReader) Read(p []byte) (int, error) {
+	return r.fp.Read(p)
+}
+
+func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) {
+	n, err := r.fp.ReadAt(p, off)
+	if err != nil {
+		return n, err
+	}
+
+	r.mux.Lock()
+	// Ignore the first signature call
+	if _, ok := r.signMap[off]; ok {
+		r.read += int64(n)
+	} else {
+		r.signMap[off] = struct{}{}
+	}
+	r.mux.Unlock()
+
+	if r.fn != nil {
+		read := r.read
+		if err := r.fn(read, float32(read*100)/float32(r.size)); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+func (r *s3UploadProgressedReader) Seek(offset int64, whence int) (int64, error) {
+	return r.fp.Seek(offset, whence)
+}

weed/storage/backend/volume_create.go

@@ -0,0 +1,20 @@
+//go:build !linux && !windows
+
+package backend
+
+import (
+	"os"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
+	file, e := OpenVolumeFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+	if e != nil {
+		return nil, e
+	}
+	if preallocate > 0 {
+		glog.V(2).Infof("Preallocated disk space for %s is not supported", fileName)
+	}
+	return NewDiskFile(file), nil
+}

weed/storage/backend/volume_create_linux.go

@@ -0,0 +1,22 @@
+//go:build linux
+
+package backend
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
+	file, e := OpenVolumeFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
+	if e != nil {
+		return nil, e
+	}
+	if preallocate != 0 {
+		syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)
+		glog.V(1).Infof("Preallocated %d bytes disk space for %s", preallocate, fileName)
+	}
+	return NewDiskFile(file), nil
+}

weed/storage/backend/volume_create_windows.go

@@ -0,0 +1,32 @@
+//go:build windows
+
+package backend
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend/memory_map"
+	"golang.org/x/sys/windows"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/backend/memory_map/os_overloads"
+)
+
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
+	if preallocate > 0 {
+		glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
+	}
+
+	if memoryMapSizeMB > 0 {
+		file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT, 0644, true)
+		if e != nil {
+			return nil, e
+		}
+		return memory_map.NewMemoryMappedFile(file, memoryMapSizeMB), nil
+	} else {
+		file, e := os_overloads.OpenFile(fileName, windows.O_RDWR|windows.O_CREAT|windows.O_TRUNC, 0644, false)
+		if e != nil {
+			return nil, e
+		}
+		return NewDiskFile(file), nil
+	}
+
+}

weed/storage/backend/volume_open.go

@@ -0,0 +1,11 @@
+//go:build !linux
+
+package backend
+
+import "os"
+
+// OpenVolumeFile opens a volume data or index file. Only Linux can suppress
+// atime updates per file descriptor; elsewhere this is a plain open.
+func OpenVolumeFile(fileName string, flag int) (*os.File, error) {
+	return os.OpenFile(fileName, flag, 0644)
+}

weed/storage/backend/volume_open_linux.go

@@ -0,0 +1,22 @@
+//go:build linux
+
+package backend
+
+import (
+	"errors"
+	"os"
+	"syscall"
+)
+
+// OpenVolumeFile opens a volume data or index file with O_NOATIME. Nothing
+// reads these files' atime, but without the flag every needle read dirties
+// the inode — even relatime writes atime on the first read after each write,
+// so an actively written volume pays a metadata write per read/write cycle.
+func OpenVolumeFile(fileName string, flag int) (*os.File, error) {
+	file, err := os.OpenFile(fileName, flag|syscall.O_NOATIME, 0644)
+	if err != nil && errors.Is(err, syscall.EPERM) {
+		// O_NOATIME is refused unless we own the file or hold CAP_FOWNER.
+		return os.OpenFile(fileName, flag, 0644)
+	}
+	return file, err
+}