snapshot of https://github.com/cortezaproject/corteza.git (2024.9.x, 2026-09-18; blobs > 131072B omitted)

server/pkg/objstore/healthcheck.go

@@ -0,0 +1,11 @@
+package objstore
+
+import (
+	"context"
+)
+
+func Healthcheck(s Store) func(ctx context.Context) error {
+	return func(ctx context.Context) error {
+		return s.Healthcheck(ctx)
+	}
+}

server/pkg/objstore/interfaces.go

@@ -0,0 +1,26 @@
+package objstore
+
+import (
+	"context"
+	"io"
+)
+
+type Store interface {
+	// Original returns URL to the original file
+	Original(id uint64, ext string) string
+
+	// Preview returns URL to the preview (of the original) file
+	Preview(id uint64, ext string) string
+
+	// Save stores the file
+	Save(filename string, f io.Reader) error
+
+	// Remove deletes the file
+	Remove(filename string) error
+
+	// Open returns file handle
+	Open(filename string) (io.ReadSeekCloser, error)
+
+	// Healthcheck checks health status of the store
+	Healthcheck(ctx context.Context) error
+}

server/pkg/objstore/minio/store.go

@@ -0,0 +1,153 @@
+package minio
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
+)
+
+type (
+	Options struct {
+		Endpoint string
+		Secure   bool
+		Strict   bool
+
+		AccessKeyID     string
+		SecretAccessKey string
+
+		ServerSideEncryptKey []byte
+	}
+
+	minioClient interface {
+		BucketExists(bucketName string) (bool, error)
+		MakeBucket(bucketName string, location string) (err error)
+		PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (n int64, err error)
+		RemoveObject(bucketName, objectName string) error
+		GetObject(bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error)
+	}
+
+	store struct {
+		bucket     string
+		pathPrefix string
+		component  string
+
+		mc  minioClient
+		sse encrypt.ServerSide
+
+		originalFn func(id uint64, ext string) string
+		previewFn  func(id uint64, ext string) string
+	}
+)
+
+var (
+	defPreviewFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d_preview.%s", id, ext)
+	}
+
+	defOriginalFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d.%s", id, ext)
+	}
+)
+
+func New(bucket, pathPrefix, component string, opt Options) (s *store, err error) {
+	client, err := minio.New(opt.Endpoint, opt.AccessKeyID, opt.SecretAccessKey, opt.Secure)
+	if err != nil {
+		return nil, err
+	}
+	return newWithClient(client, bucket, pathPrefix, component, opt)
+}
+
+func newWithClient(mc minioClient, bucket, pathPrefix, component string, opt Options) (s *store, err error) {
+	s = &store{
+		bucket:     bucket,
+		pathPrefix: pathPrefix,
+		component:  component,
+		mc:         mc,
+
+		originalFn: defOriginalFn,
+		previewFn:  defPreviewFn,
+	}
+
+	if err = s3utils.CheckValidBucketName(s.bucket); err != nil {
+		return nil, err
+	}
+
+	if e, err := s.mc.BucketExists(s.bucket); err != nil {
+		return nil, err
+	} else if !e {
+		if opt.Strict {
+			return nil, fmt.Errorf("bucket %q does not exist", s.bucket)
+		}
+
+		err = s.mc.MakeBucket(s.bucket, "us-east-1")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(opt.ServerSideEncryptKey) > 0 {
+		s.sse, err = encrypt.NewSSEC(opt.ServerSideEncryptKey)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return
+}
+
+func (s *store) check(name string) error {
+	if len(name) == 0 {
+		return fmt.Errorf("Invalid name when trying to store object: '%s' (for %s)", name, s.bucket)
+	}
+
+	return nil
+}
+
+func (s store) Original(id uint64, ext string) string {
+	// @todo presigned URL
+	return s.originalFn(id, ext)
+}
+
+func (s store) Preview(id uint64, ext string) string {
+	// @todo presigned URL
+	return s.previewFn(id, ext)
+
+}
+
+func (s store) Save(name string, f io.Reader) (err error) {
+	_, err = s.mc.PutObject(s.bucket, s.getObjectName(name), f, -1, minio.PutObjectOptions{
+		ServerSideEncryption: s.sse,
+	})
+
+	return err
+}
+
+func (s store) Remove(name string) error {
+	return s.mc.RemoveObject(s.bucket, s.getObjectName(name))
+}
+
+func (s store) Open(name string) (io.ReadSeekCloser, error) {
+	return s.mc.GetObject(s.bucket, s.getObjectName(name), minio.GetObjectOptions{
+		ServerSideEncryption: s.sse,
+	})
+}
+
+func (s *store) Healthcheck(_ context.Context) error {
+	return nil
+}
+
+// getObjectName prefix path to object name
+func (s *store) getObjectName(name string) (out string) {
+	path := strings.Replace(s.pathPrefix, "{component}", s.component, 1)
+	return fmt.Sprintf("%s%s", path, name)
+}
+
+// GetBucket return bucket name based on storage option bucket, separator or bucketName
+func GetBucket(bucket, component string) string {
+	return strings.Replace(bucket, "{component}", component, 1)
+}

server/pkg/objstore/minio/store_test.go

@@ -0,0 +1,235 @@
+package minio
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"testing"
+
+	"github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/stretchr/testify/require"
+)
+
+type (
+	testMinio struct{}
+)
+
+func (t testMinio) BucketExists(bucketName string) (out bool, err error) {
+	return
+}
+
+func (t testMinio) MakeBucket(bucketName string, location string) (err error) {
+	return
+}
+
+func (t testMinio) PutObject(bucketName, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (n int64, err error) {
+	return
+}
+
+func (t testMinio) RemoveObject(bucketName, objectName string) (err error) {
+	return
+}
+
+func (t testMinio) GetObject(bucketName, objectName string, opts minio.GetObjectOptions) (out *minio.Object, err error) {
+	return
+}
+
+func TestBucketName(t *testing.T) {
+	type (
+		tf struct {
+			// Input
+			bucketName string
+			// Expected result
+			errMsg string
+			// Flag to indicate whether test should Pass
+			valid bool
+		}
+	)
+
+	var (
+		req = require.New(t)
+		tcc = []tf{
+			{
+				bucketName: ".testbucket",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket.",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket-",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "testbucket/",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+
+			{
+				bucketName: "te",
+				errMsg:     "Bucket name cannot be shorter than 3 characters",
+				valid:      false,
+			},
+			{
+				bucketName: "",
+				errMsg:     "Bucket name cannot be empty",
+				valid:      false,
+			},
+			{
+				bucketName: "test..bucket",
+				errMsg:     "Bucket name contains invalid characters",
+				valid:      false,
+			},
+			{
+				bucketName: "test.bucket.com",
+				errMsg:     "",
+				valid:      true,
+			},
+			{
+				bucketName: "test-bucket",
+				errMsg:     "",
+				valid:      true,
+			},
+			{
+				bucketName: "123test-bucket",
+				errMsg:     "",
+				valid:      true,
+			},
+		}
+	)
+
+	for i, tc := range tcc {
+		_ = req
+		_ = i
+
+		err := s3utils.CheckValidBucketName(tc.bucketName)
+		if tc.errMsg != "" {
+			req.Equal(tc.errMsg, err.Error(), tc.errMsg)
+		} else {
+			req.NoError(err)
+		}
+	}
+}
+
+func TestStore(t *testing.T) {
+	type (
+		tf struct {
+			name               string
+			bucket             string
+			pathPrefix         string
+			componentName      string
+			expectedBucketName string
+			expectedError      error
+		}
+	)
+
+	var (
+		mc  testMinio
+		tcc = []tf{
+			{
+				name:               "default bucket",
+				bucket:             "{component}",
+				componentName:      "test",
+				expectedBucketName: "test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:               "custom bucket",
+				bucket:             "corteza-{component}",
+				componentName:      "test",
+				expectedBucketName: "corteza-test",
+			},
+			{
+				name:          "bucket has invalid character(/)",
+				bucket:        "corteza/{component}",
+				componentName: "test",
+				expectedError: fmt.Errorf("Bucket name contains invalid characters"),
+			},
+			{
+				name:          "bucket has invalid character(-) at the end of the name",
+				bucket:        "{component}-",
+				componentName: "test",
+				expectedError: fmt.Errorf("Bucket name contains invalid characters"),
+			},
+		}
+	)
+
+	for _, tc := range tcc {
+		t.Run(tc.name, func(t *testing.T) {
+			var (
+				req    = require.New(t)
+				bucket = GetBucket(tc.bucket, tc.componentName)
+			)
+
+			store, err := newWithClient(mc, bucket, tc.pathPrefix, tc.componentName, Options{})
+			req.Equal(tc.expectedError, err)
+			if tc.expectedError == nil {
+				req.Equal(tc.expectedBucketName, store.bucket, "Unexpected bucket name")
+			}
+
+			if store != nil {
+				{
+					fn := store.Original(123, "txt")
+					expected := "123.txt"
+					req.True(fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+				}
+
+				{
+					fn := store.Preview(123, "txt")
+					expected := "123_preview.txt"
+					req.True(fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+				}
+
+				// @todo extend below test to check expected path, content of the object
+				// 		after write, read and delete
+				// write a file
+				{
+					buf := bytes.NewBuffer([]byte("This is a testing buffer"))
+					err := store.Save("test/123.txt", buf)
+					req.True(err == nil, "Error saving file, %+v", err)
+
+					err = store.Save("test123/123.txt", buf)
+					req.True(err == nil, "Expected error when saving file outside of namespace")
+				}
+
+				// read a file
+				{
+					_, err := store.Open("test/123.txt")
+					req.True(err == nil, "Unexpected error when reading file: %+v", err)
+
+					_, err = store.Open("test/1234.txt")
+					req.True(err == nil, "Expected error when opening non-existent file")
+					_, err = store.Open("test123/123.txt")
+					req.True(err == nil, "Expected error when opening file outside of namespace")
+				}
+
+				// delete a file
+				{
+					err := store.Remove("test/123.txt")
+					req.True(err == nil, "Unexpected error when removing file: %+v", err)
+					err = store.Remove("test/123.txt")
+					req.True(err == nil, "Expected error when removing missing file")
+					err = store.Remove("test123/123.txt")
+					req.True(err == nil, "Expected error when deleting file outside of namespace")
+				}
+			}
+		})
+	}
+}

server/pkg/objstore/plain/store.go

@@ -0,0 +1,138 @@
+package plain
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+type (
+	store struct {
+		fs afero.Fs
+
+		namespace string
+
+		originalFn func(id uint64, ext string) string
+		previewFn  func(id uint64, ext string) string
+	}
+)
+
+var (
+	defPreviewFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d_preview.%s", id, ext)
+	}
+
+	defOriginalFn = func(id uint64, ext string) string {
+		return fmt.Sprintf("%d.%s", id, ext)
+	}
+)
+
+func New(namespace string) (*store, error) {
+	return NewWithAfero(afero.NewOsFs(), namespace)
+}
+
+func NewWithAfero(fs afero.Fs, namespace string) (*store, error) {
+	return &store{
+		fs:        fs,
+		namespace: namespace,
+
+		originalFn: defOriginalFn,
+		previewFn:  defPreviewFn,
+	}, nil
+}
+
+func (s *store) check(filename string) error {
+	if len(filename) == 0 {
+		return fmt.Errorf("Invalid filename when trying to store file: '%s' (for %s)", filename, s.namespace)
+	}
+
+	if filename[:len(s.namespace)+1] != s.namespace+"/" {
+		return fmt.Errorf("Invalid namespace when trying to store file: '%s' (for %s)", filename, s.namespace)
+	}
+
+	return nil
+}
+
+func (s *store) Original(id uint64, ext string) string {
+	return path.Join(s.namespace, s.originalFn(id, ext))
+}
+
+func (s *store) Preview(id uint64, ext string) string {
+	return path.Join(s.namespace, s.previewFn(id, ext))
+}
+
+func (s *store) Save(filename string, contents io.Reader) (err error) {
+	// check filename for validity
+	if err = s.check(filename); err != nil {
+		return
+	}
+
+	folder := path.Dir(filename)
+
+	if err = s.fs.MkdirAll(folder, 0755); err != nil {
+		return
+	}
+
+	return afero.WriteReader(s.fs, filename, contents)
+}
+
+func (s *store) Remove(filename string) error {
+	// check filename for validity
+	if err := s.check(filename); err != nil {
+		return err
+	}
+
+	return s.fs.Remove(filename)
+}
+
+func (s *store) Open(filename string) (out io.ReadSeekCloser, err error) {
+	// check filename for validity
+	if err := s.check(filename); err != nil {
+		return nil, err
+	}
+
+	for i := 1; i <= 3; i++ {
+		out, err = s.fs.Open(filename)
+		if err != nil {
+			time.Sleep(time.Millisecond * 500 * time.Duration(i))
+
+			continue
+		}
+
+		return
+	}
+
+	return
+}
+
+func (s *store) Healthcheck(ctx context.Context) error {
+	var (
+		fname = fmt.Sprintf("%s/.healthcheck_%d", s.namespace, time.Now().UnixNano())
+		buf   = &bytes.Buffer{}
+	)
+
+	if s == nil {
+		return fmt.Errorf("uninitialized")
+	}
+
+	buf.Write([]byte("healthcheck"))
+
+	if err := s.Save(fname, buf); err != nil {
+		return err
+	}
+
+	if _, err := s.Open(fname); err != nil {
+		return err
+	}
+
+	if err := s.Remove(fname); err != nil {
+		return err
+	}
+
+	return nil
+}

server/pkg/objstore/plain/store_test.go

@@ -0,0 +1,73 @@
+package plain
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStore(t *testing.T) {
+	readerToString := func(r io.Reader) string {
+		b := new(bytes.Buffer)
+		b.ReadFrom(r)
+		return b.String()
+	}
+
+	store, err := NewWithAfero(afero.NewMemMapFs(), "test")
+
+	require.True(t, err == nil, "Unexpected error when creating store: %+v", err)
+	require.True(t, store != nil, "Expected non-nil return for new store")
+
+	{
+		fn := store.Original(123, "jpg")
+		expected := "test/123.jpg"
+		require.True(t, fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+	}
+
+	{
+		fn := store.Preview(123, "jpg")
+		expected := "test/123_preview.jpg"
+		require.True(t, fn == expected, "Unexpected filename returned: %s != %s", expected, fn)
+	}
+
+	// write a file
+	{
+		buf := bytes.NewBuffer([]byte("This is a testing buffer"))
+		err := store.Save("test/123.jpg", buf)
+		require.True(t, err == nil, "Error saving file, %+v", err)
+
+		err = store.Save("test123/123.jpg", buf)
+		require.True(t, err != nil, "Expected error when saving file outside of namespace")
+	}
+
+	// read a file
+	{
+		buf, err := store.Open("test/123.jpg")
+		require.True(t, err == nil, "Unexpected error when reading file: %+v", err)
+		s := readerToString(buf)
+		require.True(t, s == "This is a testing buffer", "Unexpected response when reading file: %s", s)
+
+		_, err = store.Open("test/1234.jpg")
+		require.True(t, err != nil, "Expected error when opening non-existent file")
+		_, err = store.Open("test123/123.jpg")
+		require.True(t, err != nil, "Expected error when opening file outside of namespace")
+	}
+
+	// delete a file
+	{
+		err := store.Remove("test/123.jpg")
+		require.True(t, err == nil, "Unexpected error when removing file: %+v", err)
+		err = store.Remove("test/123.jpg")
+		require.True(t, err != nil, "Expected error when removing missing file")
+		err = store.Remove("test123/123.jpg")
+		require.True(t, err != nil, "Expected error when deleting file outside of namespace")
+	}
+}
+
+func TestStoreCheckFunc(t *testing.T) {
+	// Should not cause panic
+	require.True(t, (&store{}).check("") != nil, "Expecting an error to be returned on empty filename check")
+}

server/pkg/objstore/util.go

@@ -0,0 +1,53 @@
+package objstore
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
+	"mime/multipart"
+	"net/url"
+
+	"github.com/cortezaproject/corteza/server/pkg/http"
+)
+
+func FromURL(fileURL string) (io.ReadCloser, error) {
+	if u, err := url.ParseRequestURI(fileURL); err != nil {
+		return nil, errors.WithStack(err)
+	} else if u.Scheme != "https" {
+		return nil, fmt.Errorf("Only HTTPS is supported for file uploads")
+	}
+
+	client, err := http.New(&http.Config{
+		Timeout: 10,
+	})
+
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	req, err := client.Get(fileURL)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, nil
+}
+
+func FromMultipartFile(file *multipart.FileHeader) (io.ReadCloser, error) {
+	reader, err := file.Open()
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	return reader, nil
+}
+
+func FromAny(file *multipart.FileHeader, url string) (io.ReadCloser, error) {
+	if file != nil {
+		return FromMultipartFile(file)
+	}
+	return FromURL(url)
+}