snapshot of https://github.com/seaweedfs/seaweedfs.git (master, 2026-09-16; blobs > 131072B omitted)

weed/storage/backend/backend.go

@@ -0,0 +1,143 @@
+package backend
+
+import (
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type BackendStorageFile interface {
+	io.ReaderAt
+	io.WriterAt
+	Truncate(off int64) error
+	io.Closer
+	GetStat() (datSize int64, modTime time.Time, err error)
+	Name() string
+	Sync() error
+}
+
+type BackendStorage interface {
+	ToProperties() map[string]string
+	NewStorageFile(key string, tierInfo *volume_server_pb.VolumeInfo) BackendStorageFile
+	// concurrency > 0 caps concurrent network transfers; <= 0 uses the backend default.
+	CopyFile(f *os.File, fn func(progressed int64, percentage float32) error, concurrency int) (key string, size int64, err error)
+	DownloadFile(fileName string, key string, fn func(progressed int64, percentage float32) error, concurrency int) (size int64, err error)
+	DeleteFile(key string) (err error)
+}
+
+type StringProperties interface {
+	GetString(key string) string
+}
+type StorageType string
+type BackendStorageFactory interface {
+	StorageType() StorageType
+	BuildStorage(configuration StringProperties, configPrefix string, id string) (BackendStorage, error)
+}
+
+var (
+	BackendStorageFactories = make(map[StorageType]BackendStorageFactory)
+	BackendStorages         = make(map[string]BackendStorage)
+)
+
+// used by master to load remote storage configurations
+func LoadConfiguration(config *util.ViperProxy) {
+
+	StorageBackendPrefix := "storage.backend"
+
+	for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
+		backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
+		if !found {
+			glog.Fatalf("backend storage type %s not found", backendTypeName)
+		}
+		for backendStorageId := range config.GetStringMap(StorageBackendPrefix + "." + backendTypeName) {
+			if !config.GetBool(StorageBackendPrefix + "." + backendTypeName + "." + backendStorageId + ".enabled") {
+				continue
+			}
+			if _, found := BackendStorages[backendTypeName+"."+backendStorageId]; found {
+				continue
+			}
+			backendStorage, buildErr := backendStorageFactory.BuildStorage(config,
+				StorageBackendPrefix+"."+backendTypeName+"."+backendStorageId+".", backendStorageId)
+			if buildErr != nil {
+				glog.Fatalf("fail to create backend storage %s.%s", backendTypeName, backendStorageId)
+			}
+			BackendStorages[backendTypeName+"."+backendStorageId] = backendStorage
+			if backendStorageId == "default" {
+				BackendStorages[backendTypeName] = backendStorage
+			}
+		}
+	}
+
+}
+
+// used by volume server to receive remote storage configurations from master
+func LoadFromPbStorageBackends(storageBackends []*master_pb.StorageBackend) {
+
+	for _, storageBackend := range storageBackends {
+		backendStorageFactory, found := BackendStorageFactories[StorageType(storageBackend.Type)]
+		if !found {
+			glog.Warningf("storage type %s not found", storageBackend.Type)
+			continue
+		}
+		if _, found := BackendStorages[storageBackend.Type+"."+storageBackend.Id]; found {
+			continue
+		}
+		backendStorage, buildErr := backendStorageFactory.BuildStorage(newProperties(storageBackend.Properties), "", storageBackend.Id)
+		if buildErr != nil {
+			glog.Fatalf("fail to create backend storage %s.%s", storageBackend.Type, storageBackend.Id)
+		}
+		BackendStorages[storageBackend.Type+"."+storageBackend.Id] = backendStorage
+		if storageBackend.Id == "default" {
+			BackendStorages[storageBackend.Type] = backendStorage
+		}
+	}
+}
+
+type Properties struct {
+	m map[string]string
+}
+
+func newProperties(m map[string]string) *Properties {
+	return &Properties{m: m}
+}
+
+func (p *Properties) GetString(key string) string {
+	if v, found := p.m[key]; found {
+		return v
+	}
+	return ""
+}
+
+func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
+	for sName, s := range BackendStorages {
+		sType, sId := BackendNameToTypeId(sName)
+		if sType == "" {
+			continue
+		}
+		backends = append(backends, &master_pb.StorageBackend{
+			Type:       sType,
+			Id:         sId,
+			Properties: s.ToProperties(),
+		})
+	}
+	return
+}
+
+func BackendNameToTypeId(backendName string) (backendType, backendId string) {
+	parts := strings.Split(backendName, ".")
+	if len(parts) == 1 {
+		return backendName, "default"
+	}
+	if len(parts) != 2 {
+		return
+	}
+
+	backendType, backendId = parts[0], parts[1]
+	return
+}