snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-16 00:29:55 +05:00
commit 240db4e2a7
15309 changed files with 2173406 additions and 0 deletions

View File

@@ -0,0 +1,10 @@
with-expecter: false
packages:
github.com/mattermost/mattermost/server/v8/platform/shared/filestore:
config:
all: true
dir: "platform/shared/filestore/mocks"
filename: "{{.InterfaceName}}.go"
mockname: "{{.InterfaceName}}"
outpkg: "mocks"
note: "Regenerate this file using `make filestore-mocks`."

View File

@@ -0,0 +1,829 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/google/uuid"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/httpservice"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// azureBlockSize is the chunk size used when staging block blob uploads.
// Matches the Azure SDK's default block size for UploadStream and keeps each
// StageBlock call well under the per-block REST limit (4000 MiB).
const azureBlockSize = 4 * 1024 * 1024
// AzureFileBackend stores files in Azure Blob Storage. Two authentication
// modes are supported: shared key (an account access key configured by the
// admin) and Microsoft Entra ID via DefaultAzureCredential (managed identity,
// service principal, workload identity, or az login - whichever the host
// environment provides).
type AzureFileBackend struct {
client *azblob.Client
// sharedKey holds the account credential when authMode is shared key, and is
// nil under default credential. GeneratePublicLink needs it to sign a Service
// SAS by hand, since GetSASURL does not allow to change the ContentDisposition
// if not at the blob level, and the parsing it does of the URL to retrieve
// container and blob names does not support custom endpoints.
sharedKey *azblob.SharedKeyCredential
authMode string
container string
pathPrefix string
timeout time.Duration
presignExpires time.Duration
ssl bool
}
var _ FileBackendWithLinkGenerator = (*AzureFileBackend)(nil)
func NewAzureFileBackend(settings FileBackendSettings) (*AzureFileBackend, error) {
if err := settings.CheckMandatoryAzureFields(); err != nil {
return nil, err
}
scheme := "https"
if !settings.AzureSSL {
scheme = "http"
}
serviceURL, err := buildAzureServiceURL(settings.AzureCloud, scheme, settings.AzureStorageAccount, settings.AzureEndpoint)
if err != nil {
return nil, err
}
client, sharedKey, err := newAzureClient(settings, serviceURL, buildAzureClientOptions(settings))
if err != nil {
return nil, err
}
// Config.IsValid rejects non-positive timeouts before they reach this
// constructor, but direct callers (tests, library users that build a
// FileBackendSettings by hand) can still slip a zero or negative value
// in. Fall back to a sane default in that case, and log loudly enough
// for the substitution to show up if it ever happens in production.
timeout := time.Duration(settings.AzureRequestTimeoutMilliseconds) * time.Millisecond
if timeout <= 0 {
mlog.Warn("AzureRequestTimeoutMilliseconds is non-positive; falling back to 30s default",
mlog.Int("value", int(settings.AzureRequestTimeoutMilliseconds)))
timeout = 30 * time.Second
}
return &AzureFileBackend{
client: client,
sharedKey: sharedKey,
authMode: settings.AzureAuthMode,
container: settings.AzureContainer,
pathPrefix: settings.AzurePathPrefix,
timeout: timeout,
presignExpires: time.Duration(settings.AzurePresignExpiresSeconds) * time.Second,
ssl: settings.AzureSSL,
}, nil
}
func (b *AzureFileBackend) DriverName() string {
return driverAzure
}
// newAzureClient builds an azblob client for the configured authentication
// mode. Shared key uses NewClientWithSharedKeyCredential; default credential
// uses NewClient with DefaultAzureCredential, which discovers managed
// identity, workload identity, service principal env vars, and az login in
// that order at runtime.
//
// The shared-key credential is returned alongside the client when shared-key
// auth is in use, so callers (GeneratePublicLink in particular) can sign
// Service SAS tokens without round-tripping to Entra ID. It is nil in the
// default-credential path; that path obtains a user-delegation credential
// lazily at link-generation time.
func newAzureClient(settings FileBackendSettings, serviceURL string, clientOptions *azblob.ClientOptions) (*azblob.Client, *azblob.SharedKeyCredential, error) {
switch settings.AzureAuthMode {
case model.AzureAuthModeDefaultCredential:
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure default credential: %w", err)
}
client, err := azblob.NewClient(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, nil, nil
case model.AzureAuthModeSharedKey:
cred, err := azblob.NewSharedKeyCredential(settings.AzureStorageAccount, settings.AzureAccessKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure shared key credential: %w", err)
}
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, cred, nil
default:
return nil, nil, fmt.Errorf("unknown azure auth mode %q", settings.AzureAuthMode)
}
}
// buildAzureClientOptions selects the HTTP transport for the Azure SDK. The
// custom cloud routes requests through httpservice's transport, honoring
// AllowedUntrustedInternalConnections. The commercial and government clouds use
// a fixed Azure host (which may resolve to a private address under Private
// Link), so they keep the SDK default transport and only override it to honor
// SkipVerify, mirroring the S3 backend.
func buildAzureClientOptions(settings FileBackendSettings) *azblob.ClientOptions {
if settings.AzureCloud == model.AzureCloudCustom {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: httpservice.NewTransportForInternalConnections(settings.SkipVerify, settings.AllowedUntrustedInternalConnections),
},
},
}
}
if settings.SkipVerify {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
},
},
}
}
return nil
}
// buildAzureServiceURL renders the Blob service URL that the SDK signs
// requests against. The cloud value selects the topology:
//
// - commercial -> vhost-style against blob.core.windows.net, e.g.
// https://{account}.blob.core.windows.net/.
// - government -> vhost-style against blob.core.usgovcloudapi.net,
// e.g. https://{account}.blob.core.usgovcloudapi.net/.
// - custom -> the admin-provided endpoint is the full service URL,
// including scheme and storage account (vhost-style for production
// Azure, path-style for Azurite or reverse proxies). Mattermost
// does not modify the URL.
//
// Empty cloud is treated as commercial so existing configs that pre-date
// this field keep working. Shared-key auth signs against the URL host,
// so for custom deployments the admin is responsible for ensuring the
// host actually serves the storage account named in the credential.
func buildAzureServiceURL(cloud, scheme, account, endpoint string) (string, error) {
switch cloud {
case model.AzureCloudCommercial, "":
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.windows.net/", scheme, account), nil
case model.AzureCloudGovernment:
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.usgovcloudapi.net/", scheme, account), nil
case model.AzureCloudCustom:
if endpoint == "" {
return "", errors.New("AzureCloud=custom requires AzureEndpoint to be set")
}
// The admin owns this URL end to end, but we still reject inputs
// that the SDK is guaranteed to fail on later (missing scheme,
// missing host, a scheme other than http/https) so the
// failure mode is a clear configuration error at startup rather
// than an opaque SDK error on the first blob request.
parsed, err := url.Parse(endpoint)
if err != nil {
return "", fmt.Errorf("AzureEndpoint is not a valid URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", fmt.Errorf("AzureEndpoint must use http or https, got %q", endpoint)
}
if parsed.Host == "" {
return "", fmt.Errorf("AzureEndpoint must include a host, got %q", endpoint)
}
return endpoint, nil
default:
return "", fmt.Errorf("unknown AzureCloud value %q", cloud)
}
}
// prefix joins the configured pathPrefix and the caller-supplied path.
// Using a plain path.Join, a value like "foo/../../secret" can escape
// the prefix entirely, so we compute the join and verify the result is
// the prefix directory itself or a descendant of it. The descendant check
// requires a path-separator boundary so a prefix of "mattermost" does not
// match a sibling like "mattermost-evil/...". If the joined path escapes,
// we fall back to joining the prefix with path.Base, which may drop any
// intermediate directories the caller intended.
func (b *AzureFileBackend) prefix(p string) string {
joined := path.Join(b.pathPrefix, p)
if b.pathPrefix == "" {
return joined
}
cleanPrefix := strings.TrimSuffix(path.Clean(b.pathPrefix), "/")
if joined == cleanPrefix || strings.HasPrefix(joined, cleanPrefix+"/") {
return joined
}
return path.Join(cleanPrefix, path.Base(p))
}
func (b *AzureFileBackend) newBlobClient(p string) *blob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newBlockBlobClient(p string) *blockblob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlockBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newContainerClient() *container.Client {
return b.client.ServiceClient().NewContainerClient(b.container)
}
// TestConnection probes the configured container and reports the outcome
// using the typed errors shared with the other backends. Container
// creation is deliberately out of scope here - callers (Server.Start)
// decide whether to provision a missing container via MakeContainer.
// That separation keeps a typo in the System Console from silently
// provisioning an unwanted container, and matches the S3 contract where
// TestConnection returns FileBackendNoBucketError and MakeBucket is an
// explicit call.
func (b *AzureFileBackend) TestConnection() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newContainerClient().GetProperties(ctx, nil)
if err == nil {
return nil
}
if bloberror.HasCode(err, bloberror.ContainerNotFound) {
return &FileBackendNoBucketError{Err: fmt.Errorf("azure container %q does not exist: %w", b.container, err)}
}
if isAzureAuthError(err) {
return &FileBackendAuthError{Err: fmt.Errorf("unable to authenticate against azure blob storage: %w", err)}
}
return fmt.Errorf("unable to connect to azure blob storage: %w", err)
}
// MakeContainer creates the configured container. Mirrors S3FileBackend.MakeBucket
// so callers can opt into container provisioning explicitly. An already-existing
// container is treated as success so that concurrent boots (two nodes racing
// through TestConnection plus MakeContainer) both converge cleanly.
func (b *AzureFileBackend) MakeContainer() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if _, err := b.newContainerClient().Create(ctx, nil); err != nil {
if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
return nil
}
return fmt.Errorf("unable to create azure container %q: %w", b.container, err)
}
return nil
}
func (b *AzureFileBackend) Reader(p string) (ReadCloseSeeker, error) {
// Arm the deadline *before* the first network call, then hand the same
// timer to the returned reader on success. The previous code only set up
// the timer on the happy path, which left GetProperties running against a
// no-deadline context.
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(b.timeout, cancel)
blobClient := b.newBlobClient(p)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("unable to read file %q: %w", p, err)
}
if props.ContentLength == nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("missing content length for %q", p)
}
return &azureRangeReader{
ctx: ctx,
cancel: cancel,
timer: timer,
blobClient: blobClient,
size: *props.ContentLength,
}, nil
}
func (b *AzureFileBackend) ReadFile(p string) ([]byte, error) {
r, err := b.Reader(p)
if err != nil {
return nil, err
}
defer r.Close()
return io.ReadAll(r)
}
func (b *AzureFileBackend) FileExists(p string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return false, nil
}
return false, fmt.Errorf("unable to check existence of %q: %w", p, err)
}
return true, nil
}
func (b *AzureFileBackend) FileSize(p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return 0, fmt.Errorf("unable to get size of %q: %w", p, err)
}
return model.SafeDereference(props.ContentLength), nil
}
func (b *AzureFileBackend) FileModTime(p string) (time.Time, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return time.Time{}, fmt.Errorf("unable to get modification time of %q: %w", p, err)
}
return model.SafeDereference(props.LastModified), nil
}
// CopyFile copies via StartCopyFromURL and polls the resulting blob's copy
// status until it succeeds, matching the synchronous semantics that the
// FileBackend interface (and the S3 driver via ComposeObject) provides.
func (b *AzureFileBackend) CopyFile(oldPath, newPath string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
src := b.newBlobClient(oldPath).URL()
dst := b.newBlockBlobClient(newPath)
if _, err := dst.StartCopyFromURL(ctx, src, nil); err != nil {
return fmt.Errorf("unable to copy %q to %q: %w", oldPath, newPath, err)
}
// Poll until the copy reports success. For server-to-server copies within
// the same account this is typically synchronous, but the API is
// asynchronous in general, so we wait.
for {
props, err := dst.GetProperties(ctx, nil)
if err != nil {
return fmt.Errorf("unable to read copy status for %q: %w", newPath, err)
}
if props.CopyStatus == nil {
return nil
}
switch *props.CopyStatus {
case blob.CopyStatusTypeSuccess:
return nil
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
desc := model.SafeDereference(props.CopyStatusDescription)
return fmt.Errorf("azure copy from %q to %q ended in status %q: %q", oldPath, newPath, *props.CopyStatus, desc)
}
select {
case <-ctx.Done():
return fmt.Errorf("azure copy from %q to %q did not complete in time: %w", oldPath, newPath, ctx.Err())
case <-time.After(50 * time.Millisecond):
}
}
}
func (b *AzureFileBackend) MoveFile(oldPath, newPath string) error {
if err := b.CopyFile(oldPath, newPath); err != nil {
return err
}
return b.RemoveFile(oldPath)
}
func (b *AzureFileBackend) WriteFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return b.WriteFileContext(ctx, fr, p)
}
// stageBlocks reads fr in azureBlockSize chunks and stages each chunk as a
// block under a fresh ID. Returns the IDs of the newly staged blocks (in
// order) and the total byte count. The caller is responsible for committing
// the block list.
func (b *AzureFileBackend) stageBlocks(ctx context.Context, bb *blockblob.Client, fr io.Reader, p string) ([]string, int64, error) {
buf := make([]byte, azureBlockSize)
var ids []string
var total int64
for {
n, err := io.ReadFull(fr, buf)
if n > 0 {
id, idErr := newAzureBlockID()
if idErr != nil {
return nil, 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(buf[:n])}, nil); sbErr != nil {
return nil, 0, fmt.Errorf("unable to stage block for %q: %w", p, sbErr)
}
ids = append(ids, id)
total += int64(n)
}
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
}
if err != nil {
return nil, 0, fmt.Errorf("failed to read input: %w", err)
}
}
return ids, total, nil
}
// WriteFileContext stages the body in fixed-size blocks and commits a fresh
// block list. It deliberately does not use the SDK's UploadStream helper:
// UploadStream's small-payload fast path falls back to single-shot PutBlob,
// which leaves the resulting blob with no committed block list. A subsequent
// AppendFile that calls CommitBlockList on that blob would then clobber its
// content. Routing every WriteFile through StageBlock + CommitBlockList keeps
// AppendFile correct regardless of payload size.
//
// The caller's context governs the entire upload - no inner timeout is added.
// TryWriteFileContext (filesstore.go) relies on this to let long-running
// callers like message-export bulk writes opt out of the per-operation
// timeout that WriteFile applies by default.
func (b *AzureFileBackend) WriteFileContext(ctx context.Context, fr io.Reader, p string) (int64, error) {
bb := b.newBlockBlobClient(p)
blockIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if len(blockIDs) == 0 {
// Empty input - still need to materialize an empty blob with a
// committed block list so AppendFile can target it.
id, idErr := newAzureBlockID()
if idErr != nil {
return 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(nil)}, nil); sbErr != nil {
return 0, fmt.Errorf("unable to stage empty block for %q: %w", p, sbErr)
}
blockIDs = append(blockIDs, id)
}
if _, err := bb.CommitBlockList(ctx, blockIDs, nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// AppendFile stages the new chunk as one or more blocks and commits the
// existing committed block list plus the newly staged IDs. Each AppendFile
// call uploads the new bytes exactly once - no re-download, no
// re-concatenate, no re-upload of the prior contents. The S3-style contract
// is preserved: returns an error if the target blob does not yet exist;
// returns the number of bytes appended (not the resulting total size).
//
// Refuses to append to a blob that has content but no committed block list
// (i.e. was uploaded via Put Blob by another tool - Azure portal, azcopy,
// a migration script). Committing a new block list against such a blob
// would replace the existing content with only the appended bytes, so
// failing loud beats silent data loss.
func (b *AzureFileBackend) AppendFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
bb := b.newBlockBlobClient(p)
listResp, err := bb.GetBlockList(ctx, blockblob.BlockListTypeCommitted, nil)
if err != nil {
return 0, fmt.Errorf("unable to find file %q to append data: %w", p, err)
}
var existingIDs []string
if listResp.BlockList.CommittedBlocks != nil {
for _, blk := range listResp.BlockList.CommittedBlocks {
if blk.Name != nil {
existingIDs = append(existingIDs, *blk.Name)
}
}
}
if len(existingIDs) == 0 {
props, propsErr := bb.GetProperties(ctx, nil)
if propsErr != nil {
return 0, fmt.Errorf("unable to inspect %q before append: %w", p, propsErr)
}
if model.SafeDereference(props.ContentLength) > 0 {
return 0, fmt.Errorf("refusing to append to %q: blob has content but no committed block list (likely written via Put Blob by another tool)", p)
}
}
newIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if _, err := bb.CommitBlockList(ctx, append(existingIDs, newIDs...), nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// GeneratePublicLink returns a time-limited, read-only URL to the blob at path
// using a Shared Access Signature. The SAS is auth-mode aware:
//
// - shared-key auth signs a Service SAS in-process with the stored
// SharedKeyCredential.
// - default-credential auth fetches a user-delegation key from Entra ID
// (one round trip per call) and signs a user-delegation SAS with it.
//
// This is intended for the export-download flow (App.GeneratePresignURLForExport
// and the /exportlink slash command). End users never reach this code path.
func (b *AzureFileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
if b.presignExpires <= 0 {
return "", 0, errors.New("azure presign expiration is not configured")
}
prefixed := b.prefix(path)
// Back-date the start by a small fixed amount to absorb minor clock skew
// between this host and Azure, matching the azure-sdk-for-go SAS examples:
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/sas/examples_test.go#L71
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/service/examples_test.go#L315
const clockSkew = 10 * time.Second
start := time.Now().UTC().Add(-clockSkew)
expiry := start.Add(b.presignExpires)
protocol := sas.ProtocolHTTPSandHTTP
if b.ssl {
protocol = sas.ProtocolHTTPS
}
values := sas.BlobSignatureValues{
Protocol: protocol,
StartTime: start,
ExpiryTime: expiry,
Permissions: (&sas.BlobPermissions{Read: true}).String(),
ContainerName: b.container,
BlobName: prefixed,
// Request browsers to download the file instead of rendering it
ContentDisposition: "attachment",
}
var (
qps sas.QueryParameters
err error
)
switch b.authMode {
case model.AzureAuthModeSharedKey:
if b.sharedKey == nil {
return "", 0, errors.New("shared key credential is unavailable")
}
qps, err = values.SignWithSharedKey(b.sharedKey)
if err != nil {
return "", 0, fmt.Errorf("unable to sign service SAS for %q: %w", path, err)
}
case model.AzureAuthModeDefaultCredential:
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
udc, udcErr := b.client.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
Start: new(start.Format(sas.TimeFormat)),
Expiry: new(expiry.Format(sas.TimeFormat)),
}, nil)
if udcErr != nil {
return "", 0, fmt.Errorf("unable to obtain user delegation key for %q: %w", path, udcErr)
}
qps, err = values.SignWithUserDelegation(udc)
if err != nil {
return "", 0, fmt.Errorf("unable to sign user-delegation SAS for %q: %w", path, err)
}
default:
return "", 0, fmt.Errorf("unknown azure auth mode %q", b.authMode)
}
return b.newBlobClient(path).URL() + "?" + qps.Encode(), b.presignExpires, nil
}
func (b *AzureFileBackend) RemoveFile(p string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).Delete(ctx, nil)
if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) {
return fmt.Errorf("unable to remove file %q: %w", p, err)
}
return nil
}
func (b *AzureFileBackend) ListDirectory(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
for _, item := range page.Segment.BlobPrefixes {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
name = strings.TrimSuffix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) ListDirectoryRecursively(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q recursively: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) RemoveDirectory(p string) error {
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
for _, f := range files {
if err := b.RemoveFile(f); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) ZipReader(p string, deflate bool) (io.ReadCloser, error) {
method := zip.Store
if deflate {
method = zip.Deflate
}
pr, pw := io.Pipe()
go func() {
zw := zip.NewWriter(pw)
err := b.writeZip(zw, p, method)
if cerr := zw.Close(); err == nil {
err = cerr
}
pw.CloseWithError(err)
}()
return pr, nil
}
func (b *AzureFileBackend) writeZip(zw *zip.Writer, p string, method uint16) error {
exists, err := b.FileExists(p)
if err != nil {
return err
}
if exists {
return b.writeZipEntry(zw, p, path.Base(p), method)
}
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
prefix := strings.TrimSuffix(p, "/") + "/"
for _, f := range files {
rel := strings.TrimPrefix(f, prefix)
if err := b.writeZipEntry(zw, f, rel, method); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) writeZipEntry(zw *zip.Writer, blobPath, name string, method uint16) error {
r, err := b.Reader(blobPath)
if err != nil {
return err
}
defer r.Close()
header := &zip.FileHeader{Name: name, Method: method}
header.SetMode(0644)
w, err := zw.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(w, r)
return err
}
// readSeekNopCloser adapts a Reader+Seeker into a ReadSeekCloser without
// closing the underlying source. The Azure SDK's StageBlock signature
// requires a ReadSeekCloser.
type readSeekNopCloser struct {
io.Reader
}
func (r *readSeekNopCloser) Seek(offset int64, whence int) (int64, error) {
return r.Reader.(io.Seeker).Seek(offset, whence)
}
func (r *readSeekNopCloser) Close() error { return nil }
// newAzureBlockID returns a fresh base64-encoded 16-byte random block ID,
// generated with github.com/google/uuid - the same library azblob uses
// internally for the block IDs it produces in UploadStream. All committed
// blocks in a single blob must share the same decoded length, so callers
// must use this for both WriteFile and AppendFile staging.
//
// Per https://learn.microsoft.com/en-us/rest/api/storageservices/put-block:
//
// For a given blob, all block IDs must be the same length. If a block is
// uploaded with a block ID of a different length than the block IDs for any
// existing uncommitted blocks, the service returns error response code 400
// (Bad Request).
func newAzureBlockID() (string, error) {
u, err := uuid.NewRandom()
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(u[:]), nil
}
func isAzureAuthError(err error) bool {
if err == nil {
return false
}
return bloberror.HasCode(err, bloberror.AuthenticationFailed) ||
bloberror.HasCode(err, bloberror.AuthorizationFailure) ||
bloberror.HasCode(err, bloberror.InvalidAuthenticationInfo)
}

View File

@@ -0,0 +1,160 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"io"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
pkgerr "github.com/pkg/errors"
)
// blobDownloader is the subset of *blob.Client used by azureRangeReader.
// Defined as an interface so tests can substitute a fake without standing up
// a real Azure client.
type blobDownloader interface {
DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)
}
// azureRangeReader is a seekable reader over an Azure blob, backed by HTTP
// Range requests. A stream is opened lazily on the first Read at the current
// offset; Seek closes any open stream so the next Read re-opens it from the
// new offset. The context is cancelled either by Close or by a timer set to
// the backend's configured timeout, matching the S3 driver's behavior.
//
// Callers constructing this struct directly must set ctx, cancel and timer;
// the methods below assume all three are non-nil.
type azureRangeReader struct {
ctx context.Context
cancel context.CancelFunc
timer *time.Timer
blobClient blobDownloader
size int64
offset int64
body io.ReadCloser
}
// Compile-time guarantees that azureRangeReader satisfies the interfaces the
// app layer relies on. zip.NewReader requires io.ReaderAt for archive
// readers (e.g. the bulk-import worker), and the import worker also
// type-asserts to a CancelTimeout interface for long-running operations.
var (
_ ReadCloseSeeker = (*azureRangeReader)(nil)
_ io.ReaderAt = (*azureRangeReader)(nil)
)
func (r *azureRangeReader) Read(p []byte) (int, error) {
if r.offset >= r.size {
return 0, io.EOF
}
if r.body == nil {
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
Range: blob.HTTPRange{Offset: r.offset, Count: 0},
})
if err != nil {
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
}
r.body = resp.Body
}
n, err := r.body.Read(p)
r.offset += int64(n)
if err == nil {
return n, nil
}
// Close+drop the body so the caller (or a retry) doesn't read more
// from a half-consumed stream, and so Close stays idempotent.
r.body.Close()
r.body = nil
if err == io.EOF && r.offset < r.size {
// The remote stream ended before we reached the blob's content
// length. Surface that as a truncation rather than a clean EOF
// so the caller doesn't accept a partial blob as complete.
return n, io.ErrUnexpectedEOF
}
return n, err
}
func (r *azureRangeReader) Seek(offset int64, whence int) (int64, error) {
var abs int64
switch whence {
case io.SeekStart:
abs = offset
case io.SeekCurrent:
abs = r.offset + offset
case io.SeekEnd:
abs = r.size + offset
default:
return 0, pkgerr.Errorf("invalid whence: %d", whence)
}
if abs < 0 {
return 0, pkgerr.Errorf("negative position: %d", abs)
}
if abs == r.offset {
return abs, nil
}
if r.body != nil {
r.body.Close()
r.body = nil
}
r.offset = abs
return abs, nil
}
// ReadAt reads len(p) bytes starting at offset off. Each call issues a
// dedicated ranged DownloadStream - calls do not affect the cursor that Read
// uses, matching the io.ReaderAt contract. This is what the bulk-import
// worker needs to feed zip.NewReader on Azure-backed deployments.
func (r *azureRangeReader) ReadAt(p []byte, off int64) (int, error) {
if off < 0 {
return 0, pkgerr.Errorf("negative offset: %d", off)
}
if off >= r.size {
return 0, io.EOF
}
count := int64(len(p))
if remaining := r.size - off; count > remaining {
count = remaining
}
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
Range: blob.HTTPRange{Offset: off, Count: count},
})
if err != nil {
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
}
defer resp.Body.Close()
n, err := io.ReadFull(resp.Body, p[:count])
// io.ReadFull returns ErrUnexpectedEOF when the stream terminates
// before count bytes arrive. Only collapse it to io.EOF when we
// actually filled the buffer and consumed the blob to the end -
// otherwise it is a real truncation that needs to surface so
// callers like zip.NewReader do not accept partial content.
if err == io.ErrUnexpectedEOF && int64(n) == count && off+int64(n) == r.size {
return n, io.EOF
}
if err == nil && off+int64(n) == r.size {
return n, io.EOF
}
return n, err
}
// CancelTimeout stops the timer that bounds this reader's lifetime, so
// long-running consumers (e.g. the bulk-import worker, which can run far
// past the default per-operation timeout) can opt out of the automatic
// cancellation. Returns false if the timer has already fired.
func (r *azureRangeReader) CancelTimeout() bool {
return r.timer.Stop()
}
func (r *azureRangeReader) Close() error {
if r.timer != nil {
r.timer.Stop()
}
r.cancel()
if r.body != nil {
return r.body.Close()
}
return nil
}

View File

@@ -0,0 +1,361 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"errors"
"io"
"testing"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/stretchr/testify/require"
)
// trackingReadCloser wraps a Reader and records whether Close was called.
type trackingReadCloser struct {
io.Reader
closed bool
}
func (t *trackingReadCloser) Close() error {
t.closed = true
return nil
}
// fakeDownloader serves bytes from an in-memory blob, records every
// DownloadStream call's Range, and hands out trackingReadClosers so tests
// can assert close-on-Seek behavior. An optional err short-circuits responses.
type fakeDownloader struct {
data []byte
calls []blob.HTTPRange
bodies []*trackingReadCloser
err error
}
func (f *fakeDownloader) DownloadStream(_ context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
if f.err != nil {
return blob.DownloadStreamResponse{}, f.err
}
var rng blob.HTTPRange
if opts != nil {
rng = opts.Range
}
f.calls = append(f.calls, rng)
start := min(max(rng.Offset, 0), int64(len(f.data)))
end := int64(len(f.data))
if rng.Count > 0 && start+rng.Count < end {
end = start + rng.Count
}
body := &trackingReadCloser{Reader: bytes.NewReader(f.data[start:end])}
f.bodies = append(f.bodies, body)
return blob.DownloadStreamResponse{
DownloadResponse: blob.DownloadResponse{Body: body},
}, nil
}
// newTestReader returns an azureRangeReader wired to the given fake, with a
// long-lived timer so it never fires during the test. Caller must Close it.
func newTestReader(t *testing.T, fake *fakeDownloader, size int64) *azureRangeReader {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(time.Hour, cancel)
return &azureRangeReader{
ctx: ctx,
cancel: cancel,
timer: timer,
blobClient: fake,
size: size,
}
}
func TestRead(t *testing.T) {
t.Run("returns EOF at end of blob without downloading", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(0, io.SeekEnd)
require.NoError(t, err)
n, err := r.Read(make([]byte, 4))
require.Equal(t, 0, n)
require.Equal(t, io.EOF, err)
require.Empty(t, fake.calls, "no download should be issued past end of blob")
})
t.Run("opens stream at current offset", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("hello world")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(6, io.SeekStart)
require.NoError(t, err)
buf := make([]byte, 5)
n, err := io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, 5, n)
require.Equal(t, "world", string(buf))
require.Len(t, fake.calls, 1)
require.Equal(t, blob.HTTPRange{Offset: 6, Count: 0}, fake.calls[0])
})
t.Run("sequential reads reuse the open stream", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
buf := make([]byte, 4)
_, err := io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "abcd", string(buf))
_, err = io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "efgh", string(buf))
require.Len(t, fake.calls, 1, "sequential reads must reuse the open stream")
})
t.Run("propagates download errors", func(t *testing.T) {
wantErr := errors.New("boom")
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Read(make([]byte, 1))
require.ErrorIs(t, err, wantErr)
})
t.Run("surfaces truncation when stream EOFs before the blob ends", func(t *testing.T) {
// Promised size is larger than what the fake actually serves,
// so the body eventually returns io.EOF while r.offset < r.size.
// bytes.Reader returns its content + nil first, then 0 + EOF on
// the next call, so we drain the bytes before the truncation
// is observable.
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data))+10)
defer r.Close()
buf := make([]byte, 16)
n, err := r.Read(buf)
require.NoError(t, err)
require.Equal(t, 5, n)
// Second call hits EOF from the body before we've reached r.size,
// so the reader must surface that as a truncation.
n, err = r.Read(buf)
require.Equal(t, 0, n)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
require.Nil(t, r.body, "body must be released after a truncation error")
})
}
func TestReadAt(t *testing.T) {
t.Run("reads at the given offset without disturbing the cursor", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
// Advance the streaming cursor first.
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Equal(t, int64(3), r.offset)
buf := make([]byte, 4)
n, err := r.ReadAt(buf, 5)
require.NoError(t, err)
require.Equal(t, 4, n)
require.Equal(t, "fghi", string(buf))
require.Equal(t, int64(3), r.offset, "ReadAt must not touch the streaming offset")
})
t.Run("returns io.EOF when the read lands exactly at the end of the blob", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
buf := make([]byte, 3)
n, err := r.ReadAt(buf, 7)
require.Equal(t, io.EOF, err)
require.Equal(t, 3, n)
require.Equal(t, "hij", string(buf))
})
t.Run("returns io.EOF when off is past the size", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
n, err := r.ReadAt(make([]byte, 4), 100)
require.Equal(t, 0, n)
require.Equal(t, io.EOF, err)
require.Empty(t, fake.calls, "no download should be issued past end of blob")
})
t.Run("rejects negative offsets", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
defer r.Close()
_, err := r.ReadAt(make([]byte, 1), -1)
require.Error(t, err)
})
t.Run("propagates download errors", func(t *testing.T) {
wantErr := errors.New("boom")
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.ReadAt(make([]byte, 1), 0)
require.ErrorIs(t, err, wantErr)
})
t.Run("surfaces truncation when stream falls short of the requested count", func(t *testing.T) {
// Promised size exceeds the fake's actual data so ReadFull
// sees the body terminate before count bytes arrived. That
// must surface as ErrUnexpectedEOF, not a clean EOF.
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data))+5)
defer r.Close()
buf := make([]byte, 10)
n, err := r.ReadAt(buf, 0)
require.Equal(t, 5, n)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
})
}
func TestCancelTimeout(t *testing.T) {
fake := &fakeDownloader{data: []byte("abc")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
require.True(t, r.CancelTimeout(), "first stop should succeed")
require.False(t, r.CancelTimeout(), "second stop must report the timer was already stopped")
}
func TestSeek(t *testing.T) {
t.Run("absolute from start", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
pos, err := r.Seek(10, io.SeekStart)
require.NoError(t, err)
require.Equal(t, int64(10), pos)
})
t.Run("relative to current position", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(10, io.SeekStart)
require.NoError(t, err)
pos, err := r.Seek(5, io.SeekCurrent)
require.NoError(t, err)
require.Equal(t, int64(15), pos)
})
t.Run("relative to end", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
pos, err := r.Seek(-4, io.SeekEnd)
require.NoError(t, err)
require.Equal(t, int64(28), pos)
})
t.Run("rejects invalid whence", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 0)
defer r.Close()
_, err := r.Seek(0, 99)
require.Error(t, err)
})
t.Run("rejects negative absolute position", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
defer r.Close()
_, err := r.Seek(-1, io.SeekStart)
require.Error(t, err)
_, err = r.Seek(-20, io.SeekEnd)
require.Error(t, err)
})
t.Run("same offset leaves the open stream untouched", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefgh")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
openBody := fake.bodies[0]
pos, err := r.Seek(3, io.SeekStart)
require.NoError(t, err)
require.Equal(t, int64(3), pos)
require.False(t, openBody.closed, "same-offset seek must not close the open stream")
_, err = io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.calls, 1, "same-offset seek must not trigger a new download")
})
t.Run("different offset closes the open stream and the next read reopens", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := io.ReadFull(r, make([]byte, 2))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
firstBody := fake.bodies[0]
_, err = r.Seek(7, io.SeekStart)
require.NoError(t, err)
require.True(t, firstBody.closed, "seek to a new offset must close the open stream")
buf := make([]byte, 3)
_, err = io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "hij", string(buf))
require.Len(t, fake.calls, 2)
require.Equal(t, int64(7), fake.calls[1].Offset)
})
}
func TestClose(t *testing.T) {
t.Run("cancels context and closes the open body", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdef")}
r := newTestReader(t, fake, int64(len(fake.data)))
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
require.NoError(t, r.Close())
require.True(t, fake.bodies[0].closed)
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
})
t.Run("works when no stream was opened", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
require.NoError(t, r.Close())
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
})
}

View File

@@ -0,0 +1,466 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"net/url"
"os"
"testing"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
func TestBuildAzureServiceURL(t *testing.T) {
const account = "acmemattermost"
tests := []struct {
name string
cloud string
scheme string
account string // defaults to the package-level account const when empty
endpoint string
expected string
wantErr bool
}{
{
name: "commercial cloud, default scheme",
cloud: model.AzureCloudCommercial,
scheme: "https",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "commercial cloud rejects an account name with a hash character",
cloud: model.AzureCloudCommercial,
scheme: "https",
account: "account#",
wantErr: true,
},
{
name: "commercial cloud rejects an account name with a slash character",
cloud: model.AzureCloudCommercial,
scheme: "https",
account: "account/",
wantErr: true,
},
{
name: "government cloud rejects a malformed account name",
cloud: model.AzureCloudGovernment,
scheme: "https",
account: "account#",
wantErr: true,
},
{
name: "empty cloud falls back to commercial for legacy configs",
cloud: "",
scheme: "https",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "government cloud uses the usgovcloudapi suffix",
cloud: model.AzureCloudGovernment,
scheme: "https",
expected: "https://acmemattermost.blob.core.usgovcloudapi.net/",
},
{
name: "custom cloud returns the endpoint verbatim (vhost-style)",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https://acmemattermost.blob.core.windows.net/",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "custom cloud returns the endpoint verbatim (Azurite path-style)",
cloud: model.AzureCloudCustom,
scheme: "http",
endpoint: "http://localhost:10000/devstoreaccount1/",
expected: "http://localhost:10000/devstoreaccount1/",
},
{
name: "custom cloud preserves arbitrary paths the admin provides",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https://blob-proxy.internal.example.com/some/prefix/account/",
expected: "https://blob-proxy.internal.example.com/some/prefix/account/",
},
{
name: "custom cloud rejects an empty endpoint",
cloud: model.AzureCloudCustom,
scheme: "https",
wantErr: true,
},
{
name: "custom cloud rejects an endpoint missing the scheme",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "acmemattermost.blob.core.windows.net/",
wantErr: true,
},
{
name: "custom cloud rejects an endpoint with no host",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https:///acmemattermost/",
wantErr: true,
},
{
name: "custom cloud rejects a non-HTTP scheme",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "ftp://blob.example.com/",
wantErr: true,
},
{
name: "unknown cloud value is rejected",
cloud: "azuregermany",
scheme: "https",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
acct := account
if tt.account != "" {
acct = tt.account
}
got, err := buildAzureServiceURL(tt.cloud, tt.scheme, acct, tt.endpoint)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tt.expected, got)
})
}
}
func TestAzureFileBackendPrefix(t *testing.T) {
tests := []struct {
name string
prefix string
input string
expected string
}{
{name: "no prefix, plain path", prefix: "", input: "team/channel/file", expected: "team/channel/file"},
{name: "no prefix, with dot-dot", prefix: "", input: "../escape", expected: "../escape"},
{name: "prefix, plain path", prefix: "mattermost", input: "team/channel/file", expected: "mattermost/team/channel/file"},
{name: "prefix, exact root", prefix: "mattermost", input: "", expected: "mattermost"},
{name: "prefix, dot-dot escapes", prefix: "mattermost", input: "../escape", expected: "mattermost/escape"},
{name: "prefix, nested dot-dot escapes", prefix: "mattermost", input: "sub/../../escape", expected: "mattermost/escape"},
{name: "prefix, dot-dot in middle stays inside", prefix: "mattermost", input: "a/../b", expected: "mattermost/b"},
{name: "prefix with trailing slash, dot-dot escapes", prefix: "mattermost/", input: "../escape", expected: "mattermost/escape"},
{name: "prefix boundary collision must not escape", prefix: "mattermost", input: "../mattermost-evil/file", expected: "mattermost/file"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &AzureFileBackend{pathPrefix: tt.prefix}
require.Equal(t, tt.expected, b.prefix(tt.input))
})
}
}
// azuriteWellKnownAccount and azuriteWellKnownKey are Azurite's published
// development credentials. They are not secrets - they are documented in the
// Azurite README and ship hardcoded in every Azurite distribution.
const (
azuriteWellKnownAccount = "devstoreaccount1"
azuriteWellKnownKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
)
// TestAzureFileBackendAppendRefusesNonBlockBlob exercises the safety
// check in AppendFile: when a blob exists with content but no committed
// block list (i.e. it was uploaded via Put Blob by another tool), the
// backend must refuse the append rather than silently destroy the
// existing content.
func TestAzureFileBackendAppendRefusesNonBlockBlob(t *testing.T) {
be := newAzuriteBackend(t)
path := "append-refusal-test.bin"
t.Cleanup(func() { _ = be.RemoveFile(path) })
// Write the blob via the high-level Upload helper, which calls the
// Put Blob REST endpoint and leaves the committed-block list empty.
original := []byte("planted-by-another-tool")
bb := be.newBlockBlobClient(path)
_, err := bb.Upload(context.Background(), nopReadSeekCloser{bytes.NewReader(original)}, nil)
require.NoError(t, err)
_, err = be.AppendFile(bytes.NewReader([]byte("would-overwrite")), path)
require.Error(t, err)
require.Contains(t, err.Error(), "no committed block list")
// The original content must still be intact.
got, err := be.ReadFile(path)
require.NoError(t, err)
require.Equal(t, original, got)
}
// TestAzureFileBackendMakeContainerIdempotent ensures that calling
// MakeContainer twice on the same backend is a no-op the second time.
// Two nodes can race through TestConnection plus MakeContainer at boot;
// the loser must converge instead of returning an error.
func TestAzureFileBackendMakeContainerIdempotent(t *testing.T) {
be := newAzuriteBackend(t)
require.NoError(t, be.MakeContainer())
require.NoError(t, be.MakeContainer())
}
type nopReadSeekCloser struct {
*bytes.Reader
}
func (nopReadSeekCloser) Close() error { return nil }
// newAzuriteBackend builds an Azure backend pointed at the Azurite emulator
// and ensures the container exists. Standalone Azure tests should use this
// instead of calling NewAzureFileBackend + TestConnection directly; the
// shared FileBackendTestSuite handles provisioning itself in SetupTest.
func newAzuriteBackend(t *testing.T) *AzureFileBackend {
t.Helper()
be, err := NewAzureFileBackend(azuriteSettings(t))
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if err := be.TestConnection(); errors.As(err, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, err)
}
return be
}
func azuriteSettings(t *testing.T) FileBackendSettings {
t.Helper()
host := os.Getenv("CI_AZURITE_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("CI_AZURITE_PORT")
if port == "" {
port = "10000"
}
return FileBackendSettings{
DriverName: driverAzure,
AzureStorageAccount: azuriteWellKnownAccount,
AzureAuthMode: model.AzureAuthModeSharedKey,
AzureAccessKey: azuriteWellKnownKey,
AzureContainer: "mattermost-test",
AzureCloud: model.AzureCloudCustom,
AzureEndpoint: "http://" + net.JoinHostPort(host, port) + "/" + azuriteWellKnownAccount + "/",
// The emulator runs on an internal address, so its host must be allowed.
AllowedUntrustedInternalConnections: host,
AzureRequestTimeoutMilliseconds: 30000,
}
}
func TestNewAzureFileBackendAuthMode(t *testing.T) {
base := FileBackendSettings{
DriverName: driverAzure,
AzureStorageAccount: "anaccount",
AzureContainer: "acontainer",
AzureEndpoint: "localhost:10000",
AzureSSL: false,
AzureRequestTimeoutMilliseconds: 30000,
}
t.Run("shared_key constructs a client", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeSharedKey
s.AzureAccessKey = azuriteWellKnownKey
be, err := NewAzureFileBackend(s)
require.NoError(t, err)
require.NotNil(t, be.client)
})
t.Run("default_credential constructs a client without an access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
// Intentionally no AzureAccessKey - default credential reads
// identity from the host environment, not config.
be, err := NewAzureFileBackend(s)
require.NoError(t, err)
require.NotNil(t, be.client)
})
t.Run("empty AuthMode is rejected", func(t *testing.T) {
s := base
s.AzureAuthMode = ""
s.AzureAccessKey = azuriteWellKnownKey
_, err := NewAzureFileBackend(s)
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
t.Run("unknown AuthMode is rejected", func(t *testing.T) {
s := base
s.AzureAuthMode = "oauth2"
s.AzureAccessKey = azuriteWellKnownKey
_, err := NewAzureFileBackend(s)
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
}
func TestCheckMandatoryAzureFieldsAuthMode(t *testing.T) {
base := FileBackendSettings{
AzureStorageAccount: "anaccount",
AzureContainer: "acontainer",
}
t.Run("shared_key requires access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeSharedKey
s.AzureAccessKey = ""
require.Error(t, s.CheckMandatoryAzureFields())
s.AzureAccessKey = "somekey"
require.NoError(t, s.CheckMandatoryAzureFields())
})
t.Run("default_credential does not require access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
s.AzureAccessKey = ""
require.NoError(t, s.CheckMandatoryAzureFields())
})
}
func TestAzureFileBackendTestSuite(t *testing.T) {
suite.Run(t, &FileBackendTestSuite{settings: azuriteSettings(t)})
}
// TestAzureGeneratePublicLink exercises the Service SAS path against Azurite.
// The user-delegation SAS path requires Entra ID and cannot run in CI; it is
// covered by the manual verification recipe in the PR description.
func TestAzureGeneratePublicLink(t *testing.T) {
t.Run("refuses to issue a link when presign expiration is unset", func(t *testing.T) {
settings := azuriteSettings(t)
// Deliberately leave AzurePresignExpiresSeconds at zero - the
// shape an admin would hit if they enabled the export-direct-
// download feature without ever configuring the field.
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
_, _, err = be.GeneratePublicLink("any/path")
require.Error(t, err)
require.Contains(t, err.Error(), "presign expiration is not configured")
})
t.Run("shared-key SAS round-trips through Azurite", func(t *testing.T) {
settings := azuriteSettings(t)
settings.AzurePresignExpiresSeconds = 300
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, connErr)
}
path := "tests/presign-" + model.NewId() + ".bin"
payload := bytes.Repeat([]byte("mattermost"), 4096) // 40 KiB
_, writeErr := be.WriteFile(bytes.NewReader(payload), path)
require.NoError(t, writeErr)
t.Cleanup(func() { _ = be.RemoveFile(path) })
link, expiry, err := be.GeneratePublicLink(path)
require.NoError(t, err)
require.NotEmpty(t, link)
require.Equal(t, 300*time.Second, expiry)
// SAS query parameters Azurite expects to see.
u, err := url.Parse(link)
require.NoError(t, err)
require.NotEmpty(t, u.Query().Get("sig"))
require.Equal(t, "r", u.Query().Get("sp"))
require.Equal(t, "attachment", u.Query().Get("rscd"))
resp, err := http.Get(link)
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
require.Equal(t, http.StatusOK, resp.StatusCode)
require.Equal(t, "attachment", resp.Header.Get("Content-Disposition"))
got, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, payload, got)
})
t.Run("tampered SAS is rejected by the storage service", func(t *testing.T) {
settings := azuriteSettings(t)
settings.AzurePresignExpiresSeconds = 300
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, connErr)
}
path := "tests/tamper-" + model.NewId() + ".bin"
_, writeErr := be.WriteFile(bytes.NewReader([]byte("contents")), path)
require.NoError(t, writeErr)
t.Cleanup(func() { _ = be.RemoveFile(path) })
link, _, err := be.GeneratePublicLink(path)
require.NoError(t, err)
// Flip the last character of the signature so the link no longer
// matches what Azurite signed.
u, err := url.Parse(link)
require.NoError(t, err)
q := u.Query()
sig := q.Get("sig")
require.NotEmpty(t, sig)
swapped := sig[:len(sig)-1] + flipBase64Char(sig[len(sig)-1])
q.Set("sig", swapped)
u.RawQuery = q.Encode()
resp, err := http.Get(u.String())
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
require.Equal(t, http.StatusForbidden, resp.StatusCode)
})
t.Run("unknown auth mode is rejected", func(t *testing.T) {
// Build a backend by hand so we can exercise the default-branch
// guard. Going through NewAzureFileBackend would be rejected by
// CheckMandatoryAzureFields first.
be := &AzureFileBackend{
authMode: "totally-not-real",
presignExpires: time.Minute,
}
_, _, err := be.GeneratePublicLink("any/path")
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
}
// flipBase64Char swaps a base64 character for another in the same alphabet
// to produce a still-valid-but-wrong signature character. Used only by the
// tamper test above.
func flipBase64Char(c byte) string {
if c == 'A' {
return "B"
}
return "A"
}

View File

@@ -0,0 +1,44 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
// FileBackendAuthError is returned when testing a connection and authentication
// against the file storage backend fails. Backends should wrap the underlying
// auth failure in this type so the admin Test Connection flow can surface a
// useful message regardless of which driver is configured.
type FileBackendAuthError struct {
// Err is the underlying driver error, if any.
Err error
// DetailedError is a human-readable message describing the failure.
// Kept for compatibility with the previous S3-specific type.
DetailedError string
}
func (e *FileBackendAuthError) Error() string {
if e.DetailedError != "" {
return e.DetailedError
}
if e.Err != nil {
return e.Err.Error()
}
return "authentication failed"
}
func (e *FileBackendAuthError) Unwrap() error { return e.Err }
// FileBackendNoBucketError is returned when testing a connection and the
// configured bucket / container does not exist.
type FileBackendNoBucketError struct {
// Err is the underlying driver error, if any.
Err error
}
func (e *FileBackendNoBucketError) Error() string {
if e.Err != nil {
return e.Err.Error()
}
return "no such bucket or container"
}
func (e *FileBackendNoBucketError) Unwrap() error { return e.Err }

View File

@@ -0,0 +1,250 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"io"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/pkg/errors"
)
const (
driverS3 = "amazons3"
driverLocal = "local"
driverAzure = "azureblob"
)
type ReadCloseSeeker interface {
io.ReadCloser
io.Seeker
}
type FileBackend interface {
DriverName() string
TestConnection() error
Reader(path string) (ReadCloseSeeker, error)
ReadFile(path string) ([]byte, error)
FileExists(path string) (bool, error)
FileSize(path string) (int64, error)
CopyFile(oldPath, newPath string) error
MoveFile(oldPath, newPath string) error
WriteFile(fr io.Reader, path string) (int64, error)
AppendFile(fr io.Reader, path string) (int64, error)
RemoveFile(path string) error
FileModTime(path string) (time.Time, error)
ListDirectory(path string) ([]string, error)
ListDirectoryRecursively(path string) ([]string, error)
RemoveDirectory(path string) error
ZipReader(path string, deflate bool) (io.ReadCloser, error)
}
type FileBackendWithLinkGenerator interface {
GeneratePublicLink(path string) (string, time.Duration, error)
}
type FileBackendSettings struct {
DriverName string
Directory string
AmazonS3AccessKeyId string
AmazonS3SecretAccessKey string
AmazonS3Bucket string
AmazonS3PathPrefix string
AmazonS3Region string
AmazonS3Endpoint string
AmazonS3SSL bool
AmazonS3SignV2 bool
AmazonS3SSE bool
AmazonS3Trace bool
SkipVerify bool
AmazonS3RequestTimeoutMilliseconds int64
AmazonS3PresignExpiresSeconds int64
AmazonS3UploadPartSizeBytes int64
AmazonS3StorageClass string
AzureStorageAccount string
AzureAuthMode string
AzureAccessKey string
AzureContainer string
AzurePathPrefix string
AzureCloud string
AzureEndpoint string
AzureSSL bool
AzureRequestTimeoutMilliseconds int64
AzurePresignExpiresSeconds int64
// AllowedUntrustedInternalConnections carries the value of the
// ServiceSettings.AllowedUntrustedInternalConnections config setting, used
// by the Azure custom-cloud backend.
AllowedUntrustedInternalConnections string
}
func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
if *fileSettings.DriverName == model.ImageDriverLocal {
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
Directory: *fileSettings.Directory,
}
}
if *fileSettings.DriverName == model.ImageDriverAzure {
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
AzureStorageAccount: *fileSettings.AzureStorageAccount,
AzureAuthMode: *fileSettings.AzureAuthMode,
AzureAccessKey: *fileSettings.AzureAccessKey,
AzureContainer: *fileSettings.AzureContainer,
AzurePathPrefix: *fileSettings.AzurePathPrefix,
AzureCloud: *fileSettings.AzureCloud,
AzureEndpoint: *fileSettings.AzureEndpoint,
AzureSSL: fileSettings.AzureSSL == nil || *fileSettings.AzureSSL,
AzureRequestTimeoutMilliseconds: *fileSettings.AzureRequestTimeoutMilliseconds,
SkipVerify: skipVerify,
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
}
}
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
AmazonS3AccessKeyId: *fileSettings.AmazonS3AccessKeyId,
AmazonS3SecretAccessKey: *fileSettings.AmazonS3SecretAccessKey,
AmazonS3Bucket: *fileSettings.AmazonS3Bucket,
AmazonS3PathPrefix: *fileSettings.AmazonS3PathPrefix,
AmazonS3Region: *fileSettings.AmazonS3Region,
AmazonS3Endpoint: *fileSettings.AmazonS3Endpoint,
AmazonS3SSL: fileSettings.AmazonS3SSL == nil || *fileSettings.AmazonS3SSL,
AmazonS3SignV2: fileSettings.AmazonS3SignV2 != nil && *fileSettings.AmazonS3SignV2,
AmazonS3SSE: fileSettings.AmazonS3SSE != nil && *fileSettings.AmazonS3SSE && enableComplianceFeature,
AmazonS3Trace: fileSettings.AmazonS3Trace != nil && *fileSettings.AmazonS3Trace,
AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
SkipVerify: skipVerify,
AmazonS3UploadPartSizeBytes: *fileSettings.AmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.AmazonS3StorageClass,
}
}
func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
if *fileSettings.ExportDriverName == model.ImageDriverLocal {
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
Directory: *fileSettings.ExportDirectory,
}
}
if *fileSettings.ExportDriverName == model.ImageDriverAzure {
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
AzureStorageAccount: *fileSettings.ExportAzureStorageAccount,
AzureAuthMode: *fileSettings.ExportAzureAuthMode,
AzureAccessKey: *fileSettings.ExportAzureAccessKey,
AzureContainer: *fileSettings.ExportAzureContainer,
AzurePathPrefix: *fileSettings.ExportAzurePathPrefix,
AzureCloud: *fileSettings.ExportAzureCloud,
AzureEndpoint: *fileSettings.ExportAzureEndpoint,
AzureSSL: fileSettings.ExportAzureSSL == nil || *fileSettings.ExportAzureSSL,
AzureRequestTimeoutMilliseconds: *fileSettings.ExportAzureRequestTimeoutMilliseconds,
AzurePresignExpiresSeconds: *fileSettings.ExportAzurePresignExpiresSeconds,
SkipVerify: skipVerify,
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
}
}
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
AmazonS3AccessKeyId: *fileSettings.ExportAmazonS3AccessKeyId,
AmazonS3SecretAccessKey: *fileSettings.ExportAmazonS3SecretAccessKey,
AmazonS3Bucket: *fileSettings.ExportAmazonS3Bucket,
AmazonS3PathPrefix: *fileSettings.ExportAmazonS3PathPrefix,
AmazonS3Region: *fileSettings.ExportAmazonS3Region,
AmazonS3Endpoint: *fileSettings.ExportAmazonS3Endpoint,
AmazonS3SSL: fileSettings.ExportAmazonS3SSL == nil || *fileSettings.ExportAmazonS3SSL,
AmazonS3SignV2: fileSettings.ExportAmazonS3SignV2 != nil && *fileSettings.ExportAmazonS3SignV2,
AmazonS3SSE: fileSettings.ExportAmazonS3SSE != nil && *fileSettings.ExportAmazonS3SSE && enableComplianceFeature,
AmazonS3Trace: fileSettings.ExportAmazonS3Trace != nil && *fileSettings.ExportAmazonS3Trace,
AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
AmazonS3PresignExpiresSeconds: *fileSettings.ExportAmazonS3PresignExpiresSeconds,
AmazonS3UploadPartSizeBytes: *fileSettings.ExportAmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.ExportAmazonS3StorageClass,
SkipVerify: skipVerify,
}
}
func (settings *FileBackendSettings) CheckMandatoryS3Fields() error {
if settings.AmazonS3Bucket == "" {
return errors.New("missing s3 bucket settings")
}
// if S3 endpoint is not set call the set defaults to set that
if settings.AmazonS3Endpoint == "" {
settings.AmazonS3Endpoint = "s3.amazonaws.com"
}
return nil
}
func (settings *FileBackendSettings) CheckMandatoryAzureFields() error {
if settings.AzureStorageAccount == "" {
return errors.New("missing azure storage account setting")
}
if settings.AzureContainer == "" {
return errors.New("missing azure container setting")
}
// AzureAccessKey is only meaningful for shared-key auth. Default credential
// reads identity from the host environment (managed identity / workload
// identity / service principal env vars / az login), so an empty access key
// is the expected configuration in that mode.
if settings.AzureAuthMode == model.AzureAuthModeSharedKey && settings.AzureAccessKey == "" {
return errors.New("missing azure access key setting")
}
return nil
}
// NewFileBackend creates a new file backend
func NewFileBackend(settings FileBackendSettings) (FileBackend, error) {
return newFileBackend(settings, true)
}
// NewExportFileBackend creates a new file backend for exports, that will not attempt to use bifrost.
func NewExportFileBackend(settings FileBackendSettings) (FileBackend, error) {
return newFileBackend(settings, false)
}
func newFileBackend(settings FileBackendSettings, canBeCloud bool) (FileBackend, error) {
switch settings.DriverName {
case driverS3:
newBackendFn := NewS3FileBackend
if !canBeCloud {
newBackendFn = NewS3FileBackendWithoutBifrost
}
backend, err := newBackendFn(settings)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the s3 backend")
}
return backend, nil
case driverLocal:
return &LocalFileBackend{
directory: settings.Directory,
}, nil
case driverAzure:
backend, err := NewAzureFileBackend(settings)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the azure backend")
}
return backend, nil
}
return nil, errors.New("no valid filestorage driver found")
}
// TryWriteFileContext checks if the file backend supports context writes and passes the context in that case.
// Should the file backend not support contexts, it just calls WriteFile instead. This can be used to disable
// the timeouts for long writes (like exports).
func TryWriteFileContext(ctx context.Context, fb FileBackend, fr io.Reader, path string) (int64, error) {
type ContextWriter interface {
WriteFileContext(context.Context, io.Reader, string) (int64, error)
}
if cw, ok := fb.(ContextWriter); ok {
return cw.WriteFileContext(ctx, fr, path)
}
return fb.WriteFile(fr, path)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,347 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
const (
TestFilePath = "/testfile"
MaxRecursionDepth = 50
)
type LocalFileBackend struct {
directory string
}
// copyFile will copy a file from src path to dst path.
// Overwrites any existing files at dst.
// Permissions are copied from file at src to the new file at dst.
func copyFile(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
return
}
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
if e := out.Close(); e != nil {
err = e
}
}()
_, err = io.Copy(out, in)
if err != nil {
return
}
err = out.Sync()
if err != nil {
return
}
stat, err := os.Stat(src)
if err != nil {
return
}
err = os.Chmod(dst, stat.Mode())
if err != nil {
return
}
return
}
func (b *LocalFileBackend) DriverName() string {
return driverLocal
}
func (b *LocalFileBackend) TestConnection() error {
f := bytes.NewReader([]byte("testingwrite"))
if _, err := writeFileLocally(f, filepath.Join(b.directory, TestFilePath)); err != nil {
return errors.Wrap(err, "unable to write to the local filesystem storage")
}
os.Remove(filepath.Join(b.directory, TestFilePath))
mlog.Debug("Able to write files to local storage.")
return nil
}
func (b *LocalFileBackend) Reader(path string) (ReadCloseSeeker, error) {
f, err := os.Open(filepath.Join(b.directory, path))
if err != nil {
return nil, errors.Wrapf(err, "unable to open file %s", path)
}
return f, nil
}
func (b *LocalFileBackend) ReadFile(path string) ([]byte, error) {
f, err := os.ReadFile(filepath.Join(b.directory, path))
if err != nil {
return nil, errors.Wrapf(err, "unable to read file %s", path)
}
return f, nil
}
func (b *LocalFileBackend) FileExists(path string) (bool, error) {
_, err := os.Stat(filepath.Join(b.directory, path))
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
}
return true, nil
}
func (b *LocalFileBackend) FileSize(path string) (int64, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
}
return info.Size(), nil
}
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.ModTime(), nil
}
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
if err := copyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
}
return nil
}
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) error {
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0750); err != nil {
return errors.Wrapf(err, "unable to create the new destination directory %s", filepath.Dir(newPath))
}
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to move the file to %s to the destination directory", newPath)
}
return nil
}
func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
return writeFileLocally(fr, filepath.Join(b.directory, path))
}
func writeFileLocally(fr io.Reader, path string) (int64, error) {
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
directory, _ := filepath.Abs(filepath.Dir(path))
return 0, errors.Wrapf(err, "unable to create the directory %s for the file %s", directory, path)
}
fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return 0, errors.Wrapf(err, "unable to open the file %s to write the data", path)
}
defer fw.Close()
written, err := io.Copy(fw, fr)
if err != nil {
return written, errors.Wrapf(err, "unable write the data in the file %s", path)
}
return written, nil
}
func (b *LocalFileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
fp := filepath.Join(b.directory, path)
if _, err := os.Stat(fp); err != nil {
return 0, errors.Wrapf(err, "unable to find the file %s to append the data", path)
}
fw, err := os.OpenFile(fp, os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return 0, errors.Wrapf(err, "unable to open the file %s to append the data", path)
}
defer fw.Close()
written, err := io.Copy(fw, fr)
if err != nil {
return written, errors.Wrapf(err, "unable append the data in the file %s", path)
}
return written, nil
}
func (b *LocalFileBackend) RemoveFile(path string) error {
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
return errors.Wrapf(err, "unable to remove the file %s", path)
}
return nil
}
// basePath: path to get to the file but won't be added to the end result
// path: basePath+path current directory we are looking at
// maxDepth: parameter to prevent infinite recursion, once this is reached we won't look any further
func appendRecursively(basePath, path string, maxDepth int) ([]string, error) {
results := []string{}
dirEntries, err := os.ReadDir(filepath.Join(basePath, path))
if err != nil {
if os.IsNotExist(err) {
return results, nil
}
return results, errors.Wrapf(err, "unable to list the directory %s", path)
}
for _, dirEntry := range dirEntries {
entryName := dirEntry.Name()
entryPath := filepath.Join(path, entryName)
if entryName == "." || entryName == ".." || entryPath == path {
continue
}
if dirEntry.IsDir() {
if maxDepth <= 0 {
mlog.Warn("Max depth reached, skipping any further directories", mlog.Int("depth", maxDepth), mlog.String("path", entryPath))
results = append(results, entryPath)
continue // we'll ignore it if max depth is reached.
}
nestedResults, err := appendRecursively(basePath, entryPath, maxDepth-1)
if err != nil {
return results, err
}
results = append(results, nestedResults...)
} else {
results = append(results, entryPath)
}
}
return results, nil
}
func (b *LocalFileBackend) ListDirectory(path string) ([]string, error) {
results := []string{}
dirEntries, err := os.ReadDir(filepath.Join(b.directory, path))
if err != nil {
if os.IsNotExist(err) {
// ideally os.ErrNotExist should've been returned but to keep the
// consistency, leaving it as is before.
return results, nil
}
// same here, ideally we shouldn't return the empty slice
return results, errors.Wrapf(err, "unable to list the directory %s", path)
}
for _, dirEntry := range dirEntries {
results = append(results, filepath.Join(path, dirEntry.Name()))
}
return results, nil
}
func (b *LocalFileBackend) ListDirectoryRecursively(path string) ([]string, error) {
return appendRecursively(b.directory, path, MaxRecursionDepth)
}
func (b *LocalFileBackend) RemoveDirectory(path string) error {
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
return errors.Wrapf(err, "unable to remove the directory %s", path)
}
return nil
}
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
func (b *LocalFileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
deflateMethod := zip.Store
if deflate {
deflateMethod = zip.Deflate
}
fullPath := filepath.Join(b.directory, path)
baseInfo, err := os.Stat(fullPath)
if err != nil {
return nil, errors.Wrapf(err, "unable to stat path %s", path)
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
zipWriter := zip.NewWriter(pw)
defer zipWriter.Close()
err = filepath.Walk(fullPath, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Handle single file case
baseDir := fullPath
if !baseInfo.IsDir() {
baseDir = filepath.Dir(baseDir)
}
// Get the relative path from the base directory
relPath, err := filepath.Rel(baseDir, filePath)
if err != nil {
return errors.Wrapf(err, "unable to get relative path for %s", filePath)
}
// Skip the root directory itself
if relPath == "." {
return nil
}
// Create zip header
header, err := zip.FileInfoHeader(info)
if err != nil {
return errors.Wrapf(err, "unable to create zip header for %s", relPath)
}
// Ensure consistent forward slashes in paths
header.Name = filepath.ToSlash(relPath)
// Skip directories - we don't need to create entries for them
if info.IsDir() {
return nil
}
// Create file entry
header.Method = deflateMethod
header.SetMode(0644) // rw-r--r-- permissions
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return errors.Wrapf(err, "unable to create zip entry for %s", relPath)
}
file, err := os.Open(filePath)
if err != nil {
return errors.Wrapf(err, "unable to open file %s", filePath)
}
defer file.Close()
if _, err := io.Copy(writer, file); err != nil {
return errors.Wrapf(err, "unable to copy file content for %s", relPath)
}
return nil
})
if err != nil {
pw.CloseWithError(errors.Wrap(err, "error walking directory"))
}
}()
return pr, nil
}

View File

@@ -0,0 +1,430 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
io "io"
time "time"
filestore "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
mock "github.com/stretchr/testify/mock"
)
// FileBackend is an autogenerated mock type for the FileBackend type
type FileBackend struct {
mock.Mock
}
// AppendFile provides a mock function with given fields: fr, path
func (_m *FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
ret := _m.Called(fr, path)
if len(ret) == 0 {
panic("no return value specified for AppendFile")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
return rf(fr, path)
}
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
r0 = rf(fr, path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
r1 = rf(fr, path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CopyFile provides a mock function with given fields: oldPath, newPath
func (_m *FileBackend) CopyFile(oldPath string, newPath string) error {
ret := _m.Called(oldPath, newPath)
if len(ret) == 0 {
panic("no return value specified for CopyFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(oldPath, newPath)
} else {
r0 = ret.Error(0)
}
return r0
}
// DriverName provides a mock function with no fields
func (_m *FileBackend) DriverName() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for DriverName")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// FileExists provides a mock function with given fields: path
func (_m *FileBackend) FileExists(path string) (bool, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileExists")
}
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func(string) (bool, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) bool); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileModTime provides a mock function with given fields: path
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileModTime")
}
var r0 time.Time
var r1 error
if rf, ok := ret.Get(0).(func(string) (time.Time, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(time.Time)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileSize provides a mock function with given fields: path
func (_m *FileBackend) FileSize(path string) (int64, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileSize")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (int64, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) int64); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListDirectory provides a mock function with given fields: path
func (_m *FileBackend) ListDirectory(path string) ([]string, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ListDirectory")
}
var r0 []string
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []string); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListDirectoryRecursively provides a mock function with given fields: path
func (_m *FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ListDirectoryRecursively")
}
var r0 []string
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []string); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MoveFile provides a mock function with given fields: oldPath, newPath
func (_m *FileBackend) MoveFile(oldPath string, newPath string) error {
ret := _m.Called(oldPath, newPath)
if len(ret) == 0 {
panic("no return value specified for MoveFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(oldPath, newPath)
} else {
r0 = ret.Error(0)
}
return r0
}
// ReadFile provides a mock function with given fields: path
func (_m *FileBackend) ReadFile(path string) ([]byte, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ReadFile")
}
var r0 []byte
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]byte, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []byte); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Reader provides a mock function with given fields: path
func (_m *FileBackend) Reader(path string) (filestore.ReadCloseSeeker, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for Reader")
}
var r0 filestore.ReadCloseSeeker
var r1 error
if rf, ok := ret.Get(0).(func(string) (filestore.ReadCloseSeeker, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(filestore.ReadCloseSeeker)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RemoveDirectory provides a mock function with given fields: path
func (_m *FileBackend) RemoveDirectory(path string) error {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for RemoveDirectory")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(path)
} else {
r0 = ret.Error(0)
}
return r0
}
// RemoveFile provides a mock function with given fields: path
func (_m *FileBackend) RemoveFile(path string) error {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for RemoveFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(path)
} else {
r0 = ret.Error(0)
}
return r0
}
// TestConnection provides a mock function with no fields
func (_m *FileBackend) TestConnection() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for TestConnection")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// WriteFile provides a mock function with given fields: fr, path
func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
ret := _m.Called(fr, path)
if len(ret) == 0 {
panic("no return value specified for WriteFile")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
return rf(fr, path)
}
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
r0 = rf(fr, path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
r1 = rf(fr, path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ZipReader provides a mock function with given fields: path, deflate
func (_m *FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
ret := _m.Called(path, deflate)
if len(ret) == 0 {
panic("no return value specified for ZipReader")
}
var r0 io.ReadCloser
var r1 error
if rf, ok := ret.Get(0).(func(string, bool) (io.ReadCloser, error)); ok {
return rf(path, deflate)
}
if rf, ok := ret.Get(0).(func(string, bool) io.ReadCloser); ok {
r0 = rf(path, deflate)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(io.ReadCloser)
}
}
if rf, ok := ret.Get(1).(func(string, bool) error); ok {
r1 = rf(path, deflate)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewFileBackend creates a new instance of FileBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewFileBackend(t interface {
mock.TestingT
Cleanup(func())
}) *FileBackend {
mock := &FileBackend{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,65 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
time "time"
mock "github.com/stretchr/testify/mock"
)
// FileBackendWithLinkGenerator is an autogenerated mock type for the FileBackendWithLinkGenerator type
type FileBackendWithLinkGenerator struct {
mock.Mock
}
// GeneratePublicLink provides a mock function with given fields: path
func (_m *FileBackendWithLinkGenerator) GeneratePublicLink(path string) (string, time.Duration, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for GeneratePublicLink")
}
var r0 string
var r1 time.Duration
var r2 error
if rf, ok := ret.Get(0).(func(string) (string, time.Duration, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) string); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(string) time.Duration); ok {
r1 = rf(path)
} else {
r1 = ret.Get(1).(time.Duration)
}
if rf, ok := ret.Get(2).(func(string) error); ok {
r2 = rf(path)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// NewFileBackendWithLinkGenerator creates a new instance of FileBackendWithLinkGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewFileBackendWithLinkGenerator(t interface {
mock.TestingT
Cleanup(func())
}) *FileBackendWithLinkGenerator {
mock := &FileBackendWithLinkGenerator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,100 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import mock "github.com/stretchr/testify/mock"
// ReadCloseSeeker is an autogenerated mock type for the ReadCloseSeeker type
type ReadCloseSeeker struct {
mock.Mock
}
// Close provides a mock function with no fields
func (_m *ReadCloseSeeker) Close() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for Close")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Read provides a mock function with given fields: p
func (_m *ReadCloseSeeker) Read(p []byte) (int, error) {
ret := _m.Called(p)
if len(ret) == 0 {
panic("no return value specified for Read")
}
var r0 int
var r1 error
if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok {
return rf(p)
}
if rf, ok := ret.Get(0).(func([]byte) int); ok {
r0 = rf(p)
} else {
r0 = ret.Get(0).(int)
}
if rf, ok := ret.Get(1).(func([]byte) error); ok {
r1 = rf(p)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Seek provides a mock function with given fields: offset, whence
func (_m *ReadCloseSeeker) Seek(offset int64, whence int) (int64, error) {
ret := _m.Called(offset, whence)
if len(ret) == 0 {
panic("no return value specified for Seek")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok {
return rf(offset, whence)
}
if rf, ok := ret.Get(0).(func(int64, int) int64); ok {
r0 = rf(offset, whence)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(int64, int) error); ok {
r1 = rf(offset, whence)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewReadCloseSeeker creates a new instance of ReadCloseSeeker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewReadCloseSeeker(t interface {
mock.TestingT
Cleanup(func())
}) *ReadCloseSeeker {
mock := &ReadCloseSeeker{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,60 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
context "context"
blob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
mock "github.com/stretchr/testify/mock"
)
// blobDownloader is an autogenerated mock type for the blobDownloader type
type blobDownloader struct {
mock.Mock
}
// DownloadStream provides a mock function with given fields: ctx, opts
func (_m *blobDownloader) DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
ret := _m.Called(ctx, opts)
if len(ret) == 0 {
panic("no return value specified for DownloadStream")
}
var r0 blob.DownloadStreamResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)); ok {
return rf(ctx, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) blob.DownloadStreamResponse); ok {
r0 = rf(ctx, opts)
} else {
r0 = ret.Get(0).(blob.DownloadStreamResponse)
}
if rf, ok := ret.Get(1).(func(context.Context, *blob.DownloadStreamOptions) error); ok {
r1 = rf(ctx, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// newBlobDownloader creates a new instance of blobDownloader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func newBlobDownloader(t interface {
mock.TestingT
Cleanup(func())
}) *blobDownloader {
mock := &blobDownloader{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,122 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
context "context"
minio "github.com/minio/minio-go/v7"
mock "github.com/stretchr/testify/mock"
)
// s3CopyClient is an autogenerated mock type for the s3CopyClient type
type s3CopyClient struct {
mock.Mock
}
// ComposeObject provides a mock function with given fields: ctx, dst, srcs
func (_m *s3CopyClient) ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (minio.UploadInfo, error) {
_va := make([]interface{}, len(srcs))
for _i := range srcs {
_va[_i] = srcs[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, dst)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for ComposeObject")
}
var r0 minio.UploadInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
return rf(ctx, dst, srcs...)
}
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) minio.UploadInfo); ok {
r0 = rf(ctx, dst, srcs...)
} else {
r0 = ret.Get(0).(minio.UploadInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) error); ok {
r1 = rf(ctx, dst, srcs...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CopyObject provides a mock function with given fields: ctx, dst, src
func (_m *s3CopyClient) CopyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error) {
ret := _m.Called(ctx, dst, src)
if len(ret) == 0 {
panic("no return value specified for CopyObject")
}
var r0 minio.UploadInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
return rf(ctx, dst, src)
}
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) minio.UploadInfo); ok {
r0 = rf(ctx, dst, src)
} else {
r0 = ret.Get(0).(minio.UploadInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) error); ok {
r1 = rf(ctx, dst, src)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// StatObject provides a mock function with given fields: ctx, bucketName, objectName, opts
func (_m *s3CopyClient) StatObject(ctx context.Context, bucketName string, objectName string, opts minio.GetObjectOptions) (minio.ObjectInfo, error) {
ret := _m.Called(ctx, bucketName, objectName, opts)
if len(ret) == 0 {
panic("no return value specified for StatObject")
}
var r0 minio.ObjectInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) (minio.ObjectInfo, error)); ok {
return rf(ctx, bucketName, objectName, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) minio.ObjectInfo); ok {
r0 = rf(ctx, bucketName, objectName, opts)
} else {
r0 = ret.Get(0).(minio.ObjectInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string, minio.GetObjectOptions) error); ok {
r1 = rf(ctx, bucketName, objectName, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// newS3CopyClient creates a new instance of s3CopyClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func newS3CopyClient(t interface {
mock.TestingT
Cleanup(func())
}) *s3CopyClient {
mock := &s3CopyClient{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,61 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"net/http"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// customTransport is used to point the request to a different server.
// This is helpful in situations where a different service is handling AWS S3 requests
// from multiple Mattermost applications, and the Mattermost service itself does not
// have any S3 credentials.
type customTransport struct {
host string
scheme string
client http.Client
}
// RoundTrip implements the http.Roundtripper interface.
func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Roundtrippers should not modify the original request.
newReq := req.Clone(context.Background())
*newReq.URL = *req.URL
req.URL.Scheme = t.scheme
req.URL.Host = t.host
return t.client.Do(req)
}
// customProvider is a dummy credentials provider for the minio client to work
// without actually providing credentials. This is needed with a custom transport
// in cases where the minio client does not actually have credentials with itself,
// rather needs responses from another entity.
//
// It satisfies the credentials.Provider interface.
type customProvider struct {
isSignV2 bool
}
// RetrieveWithCredContext simply calls Retrieve, because we don't need to use
// the param.
func (cp customProvider) RetrieveWithCredContext(_ *credentials.CredContext) (credentials.Value, error) {
return cp.Retrieve()
}
// Retrieve just returns empty credentials.
func (cp customProvider) Retrieve() (credentials.Value, error) {
sign := credentials.SignatureV4
if cp.isSignV2 {
sign = credentials.SignatureV2
}
return credentials.Value{
SignerType: sign,
}, nil
}
// IsExpired always returns false.
func (cp customProvider) IsExpired() bool { return false }

View File

@@ -0,0 +1,877 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"io/fs"
"mime"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// S3FileBackend contains all necessary information to communicate with
// an AWS S3 compatible API backend.
type S3FileBackend struct {
endpoint string
accessKey string
secretKey string
secure bool
signV2 bool
region string
bucket string
pathPrefix string
encrypt bool
trace bool
client *s3.Client
skipVerify bool
timeout time.Duration
presignExpires time.Duration
isCloud bool // field to indicate whether this is running under Mattermost cloud or not.
uploadPartSize int64
storageClass string
}
// S3FileBackendAuthError and S3FileBackendNoBucketError are aliases for the
// generic backend errors. They are kept so external code (plugins,
// historically-typed consumers) continues to compile.
type (
S3FileBackendAuthError = FileBackendAuthError
S3FileBackendNoBucketError = FileBackendNoBucketError
)
const (
// This is not exported by minio. See: https://github.com/minio/minio-go/issues/1339
bucketNotFound = "NoSuchBucket"
invalidBucket = "InvalidBucketName"
)
var (
// Ensure that the ReaderAt interface is implemented.
_ io.ReaderAt = (*s3WithCancel)(nil)
_ FileBackendWithLinkGenerator = (*S3FileBackend)(nil)
)
func getContentType(ext string) string {
mimeType := mime.TypeByExtension(strings.ToLower(ext))
if mimeType == "" {
mimeType = "application/octet-stream"
}
return mimeType
}
// NewS3FileBackend returns an instance of an S3FileBackend and determine if we are in Mattermost cloud or not.
func NewS3FileBackend(settings FileBackendSettings) (*S3FileBackend, error) {
return newS3FileBackend(settings, os.Getenv("MM_CLOUD_FILESTORE_BIFROST") != "")
}
// NewS3FileBackendWithoutBifrost returns an instance of an S3FileBackend that will not use bifrost.
func NewS3FileBackendWithoutBifrost(settings FileBackendSettings) (*S3FileBackend, error) {
return newS3FileBackend(settings, false)
}
func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBackend, error) {
timeout := time.Duration(settings.AmazonS3RequestTimeoutMilliseconds) * time.Millisecond
backend := &S3FileBackend{
endpoint: settings.AmazonS3Endpoint,
accessKey: settings.AmazonS3AccessKeyId,
secretKey: settings.AmazonS3SecretAccessKey,
secure: settings.AmazonS3SSL,
signV2: settings.AmazonS3SignV2,
region: settings.AmazonS3Region,
bucket: settings.AmazonS3Bucket,
pathPrefix: settings.AmazonS3PathPrefix,
encrypt: settings.AmazonS3SSE,
trace: settings.AmazonS3Trace,
skipVerify: settings.SkipVerify,
timeout: timeout,
presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
storageClass: settings.AmazonS3StorageClass,
}
cli, err := backend.s3New(isCloud)
if err != nil {
return nil, err
}
backend.client = cli
backend.isCloud = isCloud
return backend, nil
}
type s3Trace struct {
}
func (*s3Trace) Write(in []byte) (int, error) {
mlog.Debug(string(in))
return len(in), nil
}
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func (b *S3FileBackend) s3New(isCloud bool) (*s3.Client, error) {
var creds *credentials.Credentials
if isCloud {
creds = credentials.New(customProvider{isSignV2: b.signV2})
} else if b.accessKey == "" && b.secretKey == "" {
creds = credentials.NewIAM("")
} else if b.signV2 {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
}
opts := s3.Options{
Creds: creds,
Secure: b.secure,
Region: b.region,
}
tr, err := s3.DefaultTransport(b.secure)
if err != nil {
return nil, err
}
if b.skipVerify {
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
opts.Transport = tr
// If this is a cloud installation, we override the default transport.
if isCloud {
scheme := "http"
if b.secure {
scheme = "https"
}
newTransport := http.DefaultTransport.(*http.Transport).Clone()
newTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: b.skipVerify}
opts.Transport = &customTransport{
host: b.endpoint,
scheme: scheme,
client: http.Client{Transport: newTransport},
}
}
s3Clnt, err := s3.New(b.endpoint, &opts)
if err != nil {
return nil, err
}
if b.trace {
s3Clnt.TraceOn(&s3Trace{})
}
if tr, ok := opts.Transport.(*http.Transport); ok {
if tr.TLSClientConfig == nil {
tr.TLSClientConfig = &tls.Config{}
}
tr.TLSClientConfig.MinVersion = tls.VersionTLS12
}
return s3Clnt, nil
}
func (b *S3FileBackend) DriverName() string {
return driverS3
}
func (b *S3FileBackend) TestConnection() error {
exists := true
var err error
// If a path prefix is present, we attempt to test the bucket by listing objects under the path
// and just checking the first response. This is because the BucketExists call is only at a bucket level
// and sometimes the user might only be allowed access to the specified path prefix.
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if b.pathPrefix != "" {
obj := <-b.client.ListObjects(ctx, b.bucket, s3.ListObjectsOptions{Prefix: b.pathPrefix})
if obj.Err != nil {
typedErr := s3.ToErrorResponse(obj.Err)
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to list objects in the S3 bucket: %v", typedErr)}
}
exists = false
}
} else {
exists, err = b.client.BucketExists(ctx, b.bucket)
if err != nil {
typedErr := s3.ToErrorResponse(err)
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to check if the S3 bucket exists: %v", typedErr)}
}
}
}
if !exists {
return &S3FileBackendNoBucketError{}
}
mlog.Debug("Connection to S3 or minio is good. Bucket exists.")
return nil
}
func (b *S3FileBackend) MakeBucket() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
err := b.client.MakeBucket(ctx, b.bucket, s3.MakeBucketOptions{Region: b.region})
if err != nil {
return errors.Wrap(err, "unable to create the s3 bucket")
}
return nil
}
// s3WithCancel is a wrapper struct which cancels the context
// when the object is closed.
type s3WithCancel struct {
*s3.Object
timer *time.Timer
cancel context.CancelFunc
}
func (sc *s3WithCancel) Close() error {
sc.timer.Stop()
sc.cancel()
return sc.Object.Close()
}
// CancelTimeout attempts to cancel the timeout for this reader. It allows calling
// code to ignore the timeout in case of longer running operations. The methods returns
// false if the timeout has already fired.
func (sc *s3WithCancel) CancelTimeout() bool {
return sc.timer.Stop()
}
// Caller must close the first return value
func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, error) {
path, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithCancel(context.Background())
minioObject, err := b.client.GetObject(ctx, b.bucket, path, s3.GetObjectOptions{})
if err != nil {
cancel()
return nil, errors.Wrapf(err, "unable to open file %s", path)
}
sc := &s3WithCancel{
Object: minioObject,
timer: time.AfterFunc(b.timeout, cancel),
cancel: cancel,
}
return sc, nil
}
func (b *S3FileBackend) ReadFile(path string) ([]byte, error) {
encodedPath, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
minioObject, err := b.client.GetObject(ctx, b.bucket, encodedPath, s3.GetObjectOptions{})
if err != nil {
return nil, errors.Wrapf(err, "unable to open file %s", encodedPath)
}
defer minioObject.Close()
f, err := io.ReadAll(minioObject)
if err != nil {
return nil, errors.Wrapf(err, "unable to read file %s", encodedPath)
}
return f, nil
}
func (b *S3FileBackend) FileExists(path string) (bool, error) {
path, err := b.prefixedPath(path)
if err != nil {
return false, errors.Wrapf(err, "unable to prefix path %s", path)
}
return b._fileExists(path)
}
func (b *S3FileBackend) _fileExists(path string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err == nil {
return true, nil
}
var s3Err s3.ErrorResponse
if errors.As(err, &s3Err); s3Err.Code == "NoSuchKey" {
return false, nil
}
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
}
func (b *S3FileBackend) FileSize(path string) (int64, error) {
path, err := b.prefixedPath(path)
if err != nil {
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
path, err := b.prefixedPath(path)
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.LastModified, nil
}
// maxS3SingleCopySize is the largest source object (5GiB) that S3 allows for a
// single server-side copy (CopyObject). Sources larger than this must be copied
// with a server-side multipart copy (UploadPartCopy).
const maxS3SingleCopySize = 5 * 1024 * 1024 * 1024
// s3CopyClient is the subset of the S3 client used to route a server-side copy.
// It exists so copyObjectWithClient can be unit tested with a mock client.
type s3CopyClient interface {
StatObject(ctx context.Context, bucketName, objectName string, opts s3.StatObjectOptions) (s3.ObjectInfo, error)
CopyObject(ctx context.Context, dst s3.CopyDestOptions, src s3.CopySrcOptions) (s3.UploadInfo, error)
ComposeObject(ctx context.Context, dst s3.CopyDestOptions, srcs ...s3.CopySrcOptions) (s3.UploadInfo, error)
}
// copyObject performs a server-side copy of a single object from srcOpts to
// dstOpts, using the backend's request timeout.
func (b *S3FileBackend) copyObject(srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return copyObjectWithClient(ctx, b.client, srcOpts, dstOpts)
}
// copyObjectWithClient uses CopyObject for sources up to 5GiB, and falls back to
// ComposeObject — which performs a server-side multipart copy — for larger
// sources, which S3's single-operation CopyObject rejects. The size is checked
// explicitly rather than relying on ComposeObject's own single-copy fast path,
// which is only taken when the source range Start is -1 (a value its input
// validation rejects, so it is unreachable here).
func copyObjectWithClient(ctx context.Context, client s3CopyClient, srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
stat, err := client.StatObject(ctx, srcOpts.Bucket, srcOpts.Object, s3.StatObjectOptions{})
if err != nil {
return errors.Wrapf(err, "unable to stat source file %s", srcOpts.Object)
}
if stat.Size > maxS3SingleCopySize {
_, err = client.ComposeObject(ctx, dstOpts, srcOpts)
} else {
_, err = client.CopyObject(ctx, dstOpts, srcOpts)
}
return err
}
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
oldPath, err := b.prefixedPath(oldPath)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
}
newPath = filepath.Join(b.pathPrefix, newPath)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
}
return nil
}
// DecodeFilePathIfNeeded is a special method to URL decode all older
// file paths. It is only needed for the migration, and will be removed
// as soon as the migration is complete.
func (b *S3FileBackend) DecodeFilePathIfNeeded(path string) error {
// Encode and check if file path changes.
// If there is no change, then there is no need to do anything.
if path == s3utils.EncodePath(path) {
return nil
}
// Check if encoded path exists.
exists, err := b.lookupOriginalPath(s3utils.EncodePath(path))
if err != nil {
return err
}
if !exists {
return nil
}
// If yes, then it needs to be migrated.
// This is basically a copy of MoveFile without the path encoding.
// We avoid any further refactoring because this method will be removed anyways.
oldPath := filepath.Join(b.pathPrefix, s3utils.EncodePath(path))
newPath := filepath.Join(b.pathPrefix, path)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
}
return nil
}
func (b *S3FileBackend) MoveFile(oldPath, newPath string) error {
oldPath, err := b.prefixedPath(oldPath)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
}
newPath = filepath.Join(b.pathPrefix, newPath)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
}
return nil
}
func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return b.WriteFileContext(ctx, fr, path)
}
func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, error) {
var contentType string
path = filepath.Join(b.pathPrefix, path)
ext := filepath.Ext(path)
contentType = getContentType(ext)
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
objSize := int64(-1)
if b.isCloud {
options.DisableContentSha256 = true
} else {
// We pass an object size only in situations where bifrost is not
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
switch t := fr.(type) {
case *bytes.Buffer:
objSize = int64(t.Len())
case *os.File:
if s, err2 := t.Stat(); err2 == nil {
objSize = s.Size()
}
}
}
info, err := b.client.PutObject(ctx, b.bucket, path, fr, objSize, options)
if err != nil {
return info.Size, errors.Wrapf(err, "unable write the data in the file %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
fp, err := b.prefixedPath(path)
if err != nil {
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if _, err2 := b.client.StatObject(ctx, b.bucket, fp, s3.StatObjectOptions{}); err2 != nil {
return 0, errors.Wrapf(err2, "unable to find the file %s to append the data", path)
}
contentType := getContentType(filepath.Ext(fp))
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
sse := options.ServerSideEncryption
partName := fp + ".part"
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
objSize := -1
if b.isCloud {
options.DisableContentSha256 = true
}
// We pass an object size only in situations where bifrost is not
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
if buf, ok := fr.(*bytes.Buffer); ok && !b.isCloud {
objSize = buf.Len()
}
info, err := b.client.PutObject(ctx2, b.bucket, partName, fr, int64(objSize), options)
if err != nil {
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
}
defer func() {
ctx4, cancel4 := context.WithTimeout(context.Background(), b.timeout)
defer cancel4()
b.client.RemoveObject(ctx4, b.bucket, partName, s3.RemoveObjectOptions{})
}()
src1Opts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: fp,
}
src2Opts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: partName,
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: fp,
Encryption: sse,
}
ctx3, cancel3 := context.WithTimeout(context.Background(), b.timeout)
defer cancel3()
_, err = b.client.ComposeObject(ctx3, dstOpts, src1Opts, src2Opts)
if err != nil {
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) RemoveFile(path string) error {
path, err := b.prefixedPath(path)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if err := b.client.RemoveObject(ctx, b.bucket, path, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file %s", path)
}
return nil
}
func (b *S3FileBackend) listDirectory(path string, recursion bool) ([]string, error) {
path, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
if !strings.HasSuffix(path, "/") && path != "" {
// s3Clnt returns only the path itself when "/" is not present
// appending "/" to make it consistent across all filestores
path = path + "/"
}
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: recursion,
}
var paths []string
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
var count int
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
if object.Err != nil {
return nil, errors.Wrapf(object.Err, "unable to list the directory %s", path)
}
// We strip the path prefix that gets applied,
// so that it remains transparent to the application.
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
trimmed := strings.Trim(object.Key, "/")
if trimmed != "" {
paths = append(paths, trimmed)
}
count++
}
// Check if only one item was returned and it matches the path prefix
if count == 1 && len(paths) > 0 && strings.TrimRight(path, "/") == paths[0] {
// Return a fs.PathError to maintain consistency
return nil, &fs.PathError{Op: "readdir", Path: path, Err: fs.ErrNotExist}
}
return paths, nil
}
func (b *S3FileBackend) ListDirectory(path string) ([]string, error) {
return b.listDirectory(path, false)
}
func (b *S3FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
return b.listDirectory(path, true)
}
func (b *S3FileBackend) RemoveDirectory(path string) error {
path, err := b.prefixedPath(path)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", path)
}
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: true,
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
// List all objects in the directory
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
if object.Err != nil {
return errors.Wrapf(object.Err, "unable to list the directory %s", path)
}
// Remove each object individually to avoid MD5 usage
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
err := b.client.RemoveObject(ctx2, b.bucket, object.Key, s3.RemoveObjectOptions{})
if err != nil {
return errors.Wrapf(err, "unable to remove object %s from directory %s", object.Key, path)
}
}
return nil
}
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
func (b *S3FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
deflateMethod := zip.Store
if deflate {
deflateMethod = zip.Deflate
}
path, err := b.prefixedPath(path)
if err != nil {
return nil, err
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
zipWriter := zip.NewWriter(pw)
defer zipWriter.Close()
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
// Is path a single file?
object, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err == nil {
// We want the zipped file to be at the root of the zip. E.g., given a path of
// "path/to/file.sh" we want the zip to have one file: "file.sh", not "path/to/file.sh".
stripPath := filepath.Dir(path)
if stripPath != "" {
stripPath += "/"
}
if err = b._copyObjectToZipWriter(zipWriter, object, stripPath, deflateMethod); err != nil {
pw.CloseWithError(err)
}
return
}
// Is path a directory?
path = path + "/"
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: true,
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
for object := range b.client.ListObjects(ctx2, b.bucket, opts) {
if object.Err != nil {
pw.CloseWithError(errors.Wrapf(object.Err, "unable to list the directory %s", path))
return
}
if err = b._copyObjectToZipWriter(zipWriter, object, path, deflateMethod); err != nil {
pw.CloseWithError(err)
return
}
}
}()
return pr, nil
}
func (b *S3FileBackend) _copyObjectToZipWriter(zipWriter *zip.Writer, object s3.ObjectInfo, stripPath string, deflateMethod uint16) error {
// We strip the path prefix that gets applied,
// so that it remains transparent to the application.
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
// We strip the path prefix + path so the zip file is relative to the root of the requested path
relPath := strings.TrimPrefix(object.Key, stripPath)
header := &zip.FileHeader{
Name: relPath,
Method: deflateMethod,
Modified: object.LastModified,
}
header.SetMode(0644) // rw-r--r-- permissions
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return errors.Wrapf(err, "unable to create zip entry for %s", object.Key)
}
reader, err := b.Reader(object.Key)
if err != nil {
return errors.Wrapf(err, "unable to create reader for %s", object.Key)
}
defer reader.Close()
_, err = io.Copy(writer, reader)
if err != nil {
return errors.Wrapf(err, "unable to copy content for %s", object.Key)
}
return nil
}
func (b *S3FileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
path, err := b.prefixedPath(path)
if err != nil {
return "", 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
reqParams := make(url.Values)
reqParams.Set("response-content-disposition", "attachment")
req, err := b.client.PresignedGetObject(ctx, b.bucket, path, b.presignExpires, reqParams)
if err != nil {
return "", 0, errors.Wrapf(err, "unable to generate public link for %s", path)
}
return req.String(), b.presignExpires, nil
}
func (b *S3FileBackend) lookupOriginalPath(s string) (bool, error) {
exists, err := b._fileExists(filepath.Join(b.pathPrefix, s))
if err != nil {
var s3Err s3.ErrorResponse
// Sometimes S3 will not allow to access other paths.
// In that case, we consider them as not exists.
if errors.As(err, &s3Err); s3Err.Code == "AccessDenied" {
return false, nil
}
return false, errors.Wrapf(err, "unable to check for file path %s", s)
}
return exists, nil
}
func (b *S3FileBackend) prefixedPath(s string) (string, error) {
if b.isCloud {
// We do a lookup of the original path for compatibility purposes.
exists, err := b.lookupOriginalPath(s)
if err != nil {
return "", err
}
// If it exists, then we don't want to encode it
// because it's an old path.
if !exists {
// If the request is routed via bifrost, then we need to encode the path
// to avoid signature validation errors.
// This happens because in bifrost, we are signing the URL outside the SDK
// and therefore the signature sent from the bifrost client
// will contain the encoded path, whereas the original path is sent
// un-encoded.
// More info at: https://github.com/aws/aws-sdk-go-v2/blob/1e4148ac334a4ea7abe31bd984a31dc761bb289d/aws/signer/v4/v4.go#L20
s = s3utils.EncodePath(s)
}
}
return filepath.Join(b.pathPrefix, s), nil
}
func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
options := s3.PutObjectOptions{}
if encrypted {
options.ServerSideEncryption = encrypt.NewSSE()
}
options.ContentType = contentType
options.PartSize = uint64(uploadPartSize)
options.StorageClass = storageClass
return options
}

View File

@@ -0,0 +1,79 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"errors"
"testing"
s3 "github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ensure the real S3 client satisfies the interface used by copyObjectWithClient.
var _ s3CopyClient = (*s3.Client)(nil)
// mockS3CopyClient is a minimal s3CopyClient that records which copy operation
// was invoked and lets StatObject's result be controlled per test.
type mockS3CopyClient struct {
statInfo s3.ObjectInfo
statErr error
copyCalled bool
composeCalled bool
}
func (m *mockS3CopyClient) StatObject(_ context.Context, _, _ string, _ s3.StatObjectOptions) (s3.ObjectInfo, error) {
return m.statInfo, m.statErr
}
func (m *mockS3CopyClient) CopyObject(_ context.Context, _ s3.CopyDestOptions, _ s3.CopySrcOptions) (s3.UploadInfo, error) {
m.copyCalled = true
return s3.UploadInfo{}, nil
}
func (m *mockS3CopyClient) ComposeObject(_ context.Context, _ s3.CopyDestOptions, _ ...s3.CopySrcOptions) (s3.UploadInfo, error) {
m.composeCalled = true
return s3.UploadInfo{}, nil
}
var (
testCopySrc = s3.CopySrcOptions{Bucket: "bucket", Object: "src"}
testCopyDst = s3.CopyDestOptions{Bucket: "bucket", Object: "dst"}
)
func Test_copyObject_UsesCompose_ForLarge(t *testing.T) {
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize + 1}}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.NoError(t, err)
assert.True(t, mock.composeCalled, "expected ComposeObject (multipart copy) for sources larger than 5GiB")
assert.False(t, mock.copyCalled, "expected CopyObject not to be used for sources larger than 5GiB")
}
func Test_copyObject_UsesCopy_ForSmall(t *testing.T) {
// A source exactly at the 5GiB limit must still use the single CopyObject.
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize}}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.NoError(t, err)
assert.True(t, mock.copyCalled, "expected CopyObject for sources up to 5GiB")
assert.False(t, mock.composeCalled, "expected ComposeObject not to be used for sources up to 5GiB")
}
func Test_copyObject_PropagatesStatError(t *testing.T) {
statErr := errors.New("stat failed")
mock := &mockS3CopyClient{statErr: statErr}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.Error(t, err)
assert.ErrorIs(t, err, statErr, "StatObject error must be propagated")
assert.False(t, mock.copyCalled, "no copy should be attempted when StatObject fails")
assert.False(t, mock.composeCalled, "no compose should be attempted when StatObject fails")
}

View File

@@ -0,0 +1,456 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"io"
"io/fs"
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/mattermost/mattermost/server/public/model"
s3 "github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckMandatoryS3Fields(t *testing.T) {
cfg := FileBackendSettings{}
err := cfg.CheckMandatoryS3Fields()
require.Error(t, err)
require.Equal(t, err.Error(), "missing s3 bucket settings", "should've failed with missing s3 bucket")
cfg.AmazonS3Bucket = "test-mm"
err = cfg.CheckMandatoryS3Fields()
require.NoError(t, err)
cfg.AmazonS3Endpoint = ""
err = cfg.CheckMandatoryS3Fields()
require.NoError(t, err)
require.Equal(t, "s3.amazonaws.com", cfg.AmazonS3Endpoint, "should've set the endpoint to the default")
}
func TestMakeBucket(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
SkipVerify: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
}
func TestTimeout(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
SkipVerify: false,
AmazonS3RequestTimeoutMilliseconds: 0,
}
t.Run("MakeBucket", func(t *testing.T) {
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.True(t, errors.Is(err, context.DeadlineExceeded))
})
t.Run("WriteFile", func(t *testing.T) {
cfg.AmazonS3RequestTimeoutMilliseconds = 1000
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
r, w := io.Pipe()
go func() {
defer w.Close()
for range 10 {
_, writeErr := w.Write([]byte("data"))
require.NoError(t, writeErr)
time.Sleep(time.Millisecond * 200)
}
}()
_, err = fileBackend.WriteFile(r, "tests/"+randomString()+".png")
require.True(t, errors.Is(err, context.DeadlineExceeded))
})
}
func TestInsecureMakeBucket(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
proxySelfSignedHTTPS := newTLSProxyServer(&url.URL{Scheme: "http", Host: s3Endpoint})
defer proxySelfSignedHTTPS.Close()
enableInsecure, secure := true, false
testCases := []struct {
description string
skipVerify bool
expectedAllowed bool
}{
{"allow self-signed HTTPS when insecure enabled", enableInsecure, true},
{"reject self-signed HTTPS when secured", secure, false},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: proxySelfSignedHTTPS.URL[8:],
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: true,
SkipVerify: testCase.skipVerify,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
if testCase.expectedAllowed {
require.NoError(t, err)
} else {
require.Error(t, err)
}
})
}
}
func newTLSProxyServer(backend *url.URL) *httptest.Server {
return httptest.NewTLSServer(httputil.NewSingleHostReverseProxy(backend))
}
func TestS3WithCancel(t *testing.T) {
// Some of these tests use time.Sleep to wait for the timeout to expire.
// They are run in parallel to reduce wait times.
t.Run("zero timeout", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(0, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
t.Run("timeout", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(100 * time.Millisecond) // give the context time to cancel
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
t.Run("timeout cancel", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
require.False(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("timeout closed", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
require.NoError(t, r.Close())
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("close cancel close", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, r.Close())
require.Error(t, ctx.Err())
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("close error", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, errors.New("test error"))
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.NoError(t, ctx.Err())
require.Error(t, r.Close())
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
}
func newMockS3WithCancel(timeout time.Duration, closeErr error) (*fauxCloser, context.Context) {
ctx, cancel := context.WithCancel(context.Background())
return &fauxCloser{
s3WithCancel: &s3WithCancel{
Object: &s3.Object{},
timer: time.AfterFunc(timeout, cancel),
cancel: cancel,
},
closeErr: closeErr,
}, ctx
}
type fauxCloser struct {
*s3WithCancel
closeErr error
}
func (fc fauxCloser) Close() error {
fc.s3WithCancel.timer.Stop()
fc.s3WithCancel.cancel()
return fc.closeErr
}
func TestListDirectory(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
cfg := FileBackendSettings{
DriverName: driverS3,
AmazonS3AccessKeyId: "minioaccesskey",
AmazonS3SecretAccessKey: "miniosecretkey",
AmazonS3Bucket: "mattermost-test-1",
AmazonS3Region: "",
AmazonS3Endpoint: s3Endpoint,
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
AmazonS3SSE: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
found, err := fileBackend.client.BucketExists(context.Background(), cfg.AmazonS3Bucket)
require.NoError(t, err)
if !found {
err = fileBackend.MakeBucket()
require.NoError(t, err)
}
fileBackend.pathPrefix = "19700101/"
require.NoError(t, err)
b := []byte("test")
path1 := "19700101/" + randomString() + ".txt"
_, err = fileBackend.WriteFile(bytes.NewReader(b), path1)
require.NoError(t, err, "Failed to write file1 to S3")
_, err = fileBackend.ListDirectory("")
var pErr *fs.PathError
assert.True(t, errors.As(err, &pErr), "error is not of type fs.PathError")
err = fileBackend.RemoveFile(path1)
require.NoError(t, err, "Failed to remove file1 from S3")
err = fileBackend.RemoveDirectory("19700101")
require.NoError(t, err)
}
func TestWriteFileVideoMimeTypes(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3SSL: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
// Test video types
// Test video types with multiple valid MIME types
testContent := []byte("test-video-content")
videoTypes := map[string][]string{
".avi": {"video/vnd.avi", "video/x-msvideo"},
".mpeg": {"video/mpeg"},
".mp4": {"video/mp4"},
}
for ext, validMimeTypes := range videoTypes {
t.Run(strings.TrimPrefix(ext, "."), func(t *testing.T) {
path := "test" + ext
reader := bytes.NewReader(testContent)
written, err := fileBackend.WriteFile(reader, path)
require.NoError(t, err)
require.Equal(t, int64(len(testContent)), written)
// Verify the file exists with correct mime type
props, err := fileBackend.client.StatObject(
context.Background(),
bucketName,
path,
s3.StatObjectOptions{},
)
require.NoError(t, err)
// Ensure the MIME type is one of the expected values
assert.Contains(t, validMimeTypes, props.ContentType, "Unexpected MIME type: %s", props.ContentType)
defer func() {
err = fileBackend.RemoveFile(path)
require.NoError(t, err)
}()
})
}
}