snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/objstore/minio/store.go

@@ -0,0 +1,153 @@
+package minio
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
+)
+
+type (
+	Options struct {
+		Endpoint string
+		Secure   bool
+		Strict   bool
+
+		AccessKeyID     string
+		SecretAccessKey string
+
+		ServerSideEncryptKey []byte
+	}
+
+	minioClient interface {
+		BucketExists(bucketName string) (bool, error)
+		MakeBucket(bucketName string, location string) (err error)
+		PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (n int64, err error)
+		RemoveObject(bucketName, objectName string) error
+		GetObject(bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error)
+	}
+
+	store struct {
+		bucket     string
+		pathPrefix string
+		component  string
+
+		mc  minioClient
+		sse encrypt.ServerSide
+
+		originalFn func(id uint64, ext string) string
+		previewFn  func(id uint64, ext string) string
+	}
+)
+
+var (
+	defPreviewFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d_preview.%s", id, ext)
+	}
+
+	defOriginalFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d.%s", id, ext)
+	}
+)
+
+func New(bucket, pathPrefix, component string, opt Options) (s *store, err error) {
+	client, err := minio.New(opt.Endpoint, opt.AccessKeyID, opt.SecretAccessKey, opt.Secure)
+	if err != nil {
+		return nil, err
+	}
+	return newWithClient(client, bucket, pathPrefix, component, opt)
+}
+
+func newWithClient(mc minioClient, bucket, pathPrefix, component string, opt Options) (s *store, err error) {
+	s = &store{
+		bucket:     bucket,
+		pathPrefix: pathPrefix,
+		component:  component,
+		mc:         mc,
+
+		originalFn: defOriginalFn,
+		previewFn:  defPreviewFn,
+	}
+
+	if err = s3utils.CheckValidBucketName(s.bucket); err != nil {
+		return nil, err
+	}
+
+	if e, err := s.mc.BucketExists(s.bucket); err != nil {
+		return nil, err
+	} else if !e {
+		if opt.Strict {
+			return nil, fmt.Errorf("bucket %q does not exist", s.bucket)
+		}
+
+		err = s.mc.MakeBucket(s.bucket, "us-east-1")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(opt.ServerSideEncryptKey) > 0 {
+		s.sse, err = encrypt.NewSSEC(opt.ServerSideEncryptKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return
+}
+
+func (s *store) check(name string) error {
+	if len(name) == 0 {
+		return fmt.Errorf("Invalid name when trying to store object: '%s' (for %s)", name, s.bucket)
+	}
+
+	return nil
+}
+
+func (s store) Original(id uint64, ext string) string {
+	// @todo presigned URL
+	return s.originalFn(id, ext)
+}
+
+func (s store) Preview(id uint64, ext string) string {
+	// @todo presigned URL
+	return s.previewFn(id, ext)
+
+}
+
+func (s store) Save(name string, f io.Reader) (err error) {
+	_, err = s.mc.PutObject(s.bucket, s.getObjectName(name), f, -1, minio.PutObjectOptions{
+		ServerSideEncryption: s.sse,
+	})
+
+	return err
+}
+
+func (s store) Remove(name string) error {
+	return s.mc.RemoveObject(s.bucket, s.getObjectName(name))
+}
+
+func (s store) Open(name string) (io.ReadSeekCloser, error) {
+	return s.mc.GetObject(s.bucket, s.getObjectName(name), minio.GetObjectOptions{
+		ServerSideEncryption: s.sse,
+	})
+}
+
+func (s *store) Healthcheck(_ context.Context) error {
+	return nil
+}
+
+// getObjectName prefix path to object name
+func (s *store) getObjectName(name string) (out string) {
+	path := strings.Replace(s.pathPrefix, "{component}", s.component, 1)
+	return fmt.Sprintf("%s%s", path, name)
+}
+
+// GetBucket return bucket name based on storage option bucket, separator or bucketName
+func GetBucket(bucket, component string) string {
+	return strings.Replace(bucket, "{component}", component, 1)
+}

server/pkg/objstore/minio/store_test.go

@@ -0,0 +1,235 @@
+package minio
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	testMinio struct{}
+)
+
+func (t testMinio) BucketExists(bucketName string) (out bool, err error) {
+	return
+}
+
+func (t testMinio) MakeBucket(bucketName string, location string) (err error) {
+	return
+}
+
+func (t testMinio) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (n int64, err error) {
+	return
+}
+
+func (t testMinio) RemoveObject(bucketName, objectName string) (err error) {
+	return
+}
+
+func (t testMinio) GetObject(bucketName, objectName string, opts minio.GetObjectOptions) (out *minio.Object, err error) {
+	return
+}
+
+func TestBucketName(t *testing.T) {
+	type (
+		tf struct {
+			// Input
+			bucketName string
+			// Expected result
+			errMsg string
+			// Flag to indicate whether test should Pass
+			valid bool
+		}
+	)
+
+	var (
+		req = require.New(t)
+		tcc = []tf{
+			{
+				bucketName: ".testbucket",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket.",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket-",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket/",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+
+			{
+				bucketName: "te",
+				errMsg:     "Bucket name cannot be shorter than 3 characters",
+				valid:      false,
+			},
+			{
+				bucketName: "",
+				errMsg:     "Bucket name cannot be empty",
+				valid:      false,
+			},
+			{
+				bucketName: "test..bucket",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "test.bucket.com",
+				errMsg:     "",
+				valid:      true,
+			},
+			{
+				bucketName: "test-bucket",
+				errMsg:     "",
+				valid:      true,
+			},
+			{
+				bucketName: "123test-bucket",
+				errMsg:     "",
+				valid:      true,
+			},
+		}
+	)
+
+	for i, tc := range tcc {
+		_ = req
+		_ = i
+
+		err := s3utils.CheckValidBucketName(tc.bucketName)
+		if tc.errMsg != "" {
+			req.Equal(tc.errMsg, err.Error(), tc.errMsg)
+		} else {
+			req.NoError(err)
+		}
+	}
+}
+
+func TestStore(t *testing.T) {
+	type (
+		tf struct {
+			name               string
+			bucket             string
+			pathPrefix         string
+			componentName      string
+			expectedBucketName string
+			expectedError      error
+		}
+	)
+
+	var (
+		mc  testMinio
+		tcc = []tf{
+			{
+				name:               "default bucket",
+				bucket:             "{component}",
+				componentName:      "test",
+				expectedBucketName: "test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:          "bucket has invalid character(/)",
+				bucket:        "corteza/{component}",
+				componentName: "test",
+				expectedError: fmt.Errorf("Bucket name contains invalid characters"),
+			},
+			{
+				name:          "bucket has invalid character(-) at the end of the name",
+				bucket:        "{component}-",
+				componentName: "test",
+				expectedError: fmt.Errorf("Bucket name contains invalid characters"),
+			},
+		}
+	)
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				req    = require.New(t)
+				bucket = GetBucket(tc.bucket, tc.componentName)
+			)
+
+			store, err := newWithClient(mc, bucket, tc.pathPrefix, tc.componentName, Options{})
+			req.Equal(tc.expectedError, err)
+			if tc.expectedError == nil {
+				req.Equal(tc.expectedBucketName, store.bucket, "Unexpected bucket name")
+			}
+
+			if store != nil {
+				{
+					fn := store.Original(123, "txt")
+					expected := "123.txt"
+					req.True(fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+				}
+
+				{
+					fn := store.Preview(123, "txt")
+					expected := "123_preview.txt"
+					req.True(fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+				}
+
+				// @todo extend below test to check expected path, content of the object
+				// 		after write, read and delete
+				// write a file
+				{
+					buf := bytes.NewBuffer([]byte("This is a testing buffer"))
+					err := store.Save("test/123.txt", buf)
+					req.True(err == nil, "Error saving file, %+v", err)
+
+					err = store.Save("test123/123.txt", buf)
+					req.True(err == nil, "Expected error when saving file outside of namespace")
+				}
+
+				// read a file
+				{
+					_, err := store.Open("test/123.txt")
+					req.True(err == nil, "Unexpected error when reading file: %+v", err)
+
+					_, err = store.Open("test/1234.txt")
+					req.True(err == nil, "Expected error when opening non-existent file")
+					_, err = store.Open("test123/123.txt")
+					req.True(err == nil, "Expected error when opening file outside of namespace")
+				}
+
+				// delete a file
+				{
+					err := store.Remove("test/123.txt")
+					req.True(err == nil, "Unexpected error when removing file: %+v", err)
+					err = store.Remove("test/123.txt")
+					req.True(err == nil, "Expected error when removing missing file")
+					err = store.Remove("test123/123.txt")
+					req.True(err == nil, "Expected error when deleting file outside of namespace")
+				}
+			}
+		})
+	}
+}