snapshot of https://github.com/mattermost/mattermost.git (master, 2026-09-16; blobs > 131072B omitted)
Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s

This commit is contained in:
upstream-mirror
2026-09-16 00:29:55 +05:00
commit 240db4e2a7
15309 changed files with 2173406 additions and 0 deletions

View File

@@ -0,0 +1,10 @@
with-expecter: false
packages:
github.com/mattermost/mattermost/server/v8/platform/shared/filestore:
config:
all: true
dir: "platform/shared/filestore/mocks"
filename: "{{.InterfaceName}}.go"
mockname: "{{.InterfaceName}}"
outpkg: "mocks"
note: "Regenerate this file using `make filestore-mocks`."

View File

@@ -0,0 +1,829 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"context"
"crypto/tls"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
"github.com/google/uuid"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/httpservice"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// azureBlockSize is the chunk size used when staging block blob uploads.
// Matches the Azure SDK's default block size for UploadStream and keeps each
// StageBlock call well under the per-block REST limit (4000 MiB).
const azureBlockSize = 4 * 1024 * 1024
// AzureFileBackend stores files in Azure Blob Storage. Two authentication
// modes are supported: shared key (an account access key configured by the
// admin) and Microsoft Entra ID via DefaultAzureCredential (managed identity,
// service principal, workload identity, or az login - whichever the host
// environment provides).
type AzureFileBackend struct {
client *azblob.Client
// sharedKey holds the account credential when authMode is shared key, and is
// nil under default credential. GeneratePublicLink needs it to sign a Service
// SAS by hand, since GetSASURL does not allow to change the ContentDisposition
// if not at the blob level, and the parsing it does of the URL to retrieve
// container and blob names does not support custom endpoints.
sharedKey *azblob.SharedKeyCredential
authMode string
container string
pathPrefix string
timeout time.Duration
presignExpires time.Duration
ssl bool
}
var _ FileBackendWithLinkGenerator = (*AzureFileBackend)(nil)
func NewAzureFileBackend(settings FileBackendSettings) (*AzureFileBackend, error) {
if err := settings.CheckMandatoryAzureFields(); err != nil {
return nil, err
}
scheme := "https"
if !settings.AzureSSL {
scheme = "http"
}
serviceURL, err := buildAzureServiceURL(settings.AzureCloud, scheme, settings.AzureStorageAccount, settings.AzureEndpoint)
if err != nil {
return nil, err
}
client, sharedKey, err := newAzureClient(settings, serviceURL, buildAzureClientOptions(settings))
if err != nil {
return nil, err
}
// Config.IsValid rejects non-positive timeouts before they reach this
// constructor, but direct callers (tests, library users that build a
// FileBackendSettings by hand) can still slip a zero or negative value
// in. Fall back to a sane default in that case, and log loudly enough
// for the substitution to show up if it ever happens in production.
timeout := time.Duration(settings.AzureRequestTimeoutMilliseconds) * time.Millisecond
if timeout <= 0 {
mlog.Warn("AzureRequestTimeoutMilliseconds is non-positive; falling back to 30s default",
mlog.Int("value", int(settings.AzureRequestTimeoutMilliseconds)))
timeout = 30 * time.Second
}
return &AzureFileBackend{
client: client,
sharedKey: sharedKey,
authMode: settings.AzureAuthMode,
container: settings.AzureContainer,
pathPrefix: settings.AzurePathPrefix,
timeout: timeout,
presignExpires: time.Duration(settings.AzurePresignExpiresSeconds) * time.Second,
ssl: settings.AzureSSL,
}, nil
}
func (b *AzureFileBackend) DriverName() string {
return driverAzure
}
// newAzureClient builds an azblob client for the configured authentication
// mode. Shared key uses NewClientWithSharedKeyCredential; default credential
// uses NewClient with DefaultAzureCredential, which discovers managed
// identity, workload identity, service principal env vars, and az login in
// that order at runtime.
//
// The shared-key credential is returned alongside the client when shared-key
// auth is in use, so callers (GeneratePublicLink in particular) can sign
// Service SAS tokens without round-tripping to Entra ID. It is nil in the
// default-credential path; that path obtains a user-delegation credential
// lazily at link-generation time.
func newAzureClient(settings FileBackendSettings, serviceURL string, clientOptions *azblob.ClientOptions) (*azblob.Client, *azblob.SharedKeyCredential, error) {
switch settings.AzureAuthMode {
case model.AzureAuthModeDefaultCredential:
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure default credential: %w", err)
}
client, err := azblob.NewClient(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, nil, nil
case model.AzureAuthModeSharedKey:
cred, err := azblob.NewSharedKeyCredential(settings.AzureStorageAccount, settings.AzureAccessKey)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure shared key credential: %w", err)
}
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, clientOptions)
if err != nil {
return nil, nil, fmt.Errorf("failed to create azure blob client: %w", err)
}
return client, cred, nil
default:
return nil, nil, fmt.Errorf("unknown azure auth mode %q", settings.AzureAuthMode)
}
}
// buildAzureClientOptions selects the HTTP transport for the Azure SDK. The
// custom cloud routes requests through httpservice's transport, honoring
// AllowedUntrustedInternalConnections. The commercial and government clouds use
// a fixed Azure host (which may resolve to a private address under Private
// Link), so they keep the SDK default transport and only override it to honor
// SkipVerify, mirroring the S3 backend.
func buildAzureClientOptions(settings FileBackendSettings) *azblob.ClientOptions {
if settings.AzureCloud == model.AzureCloudCustom {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: httpservice.NewTransportForInternalConnections(settings.SkipVerify, settings.AllowedUntrustedInternalConnections),
},
},
}
}
if settings.SkipVerify {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
},
},
}
}
return nil
}
// buildAzureServiceURL renders the Blob service URL that the SDK signs
// requests against. The cloud value selects the topology:
//
// - commercial -> vhost-style against blob.core.windows.net, e.g.
// https://{account}.blob.core.windows.net/.
// - government -> vhost-style against blob.core.usgovcloudapi.net,
// e.g. https://{account}.blob.core.usgovcloudapi.net/.
// - custom -> the admin-provided endpoint is the full service URL,
// including scheme and storage account (vhost-style for production
// Azure, path-style for Azurite or reverse proxies). Mattermost
// does not modify the URL.
//
// Empty cloud is treated as commercial so existing configs that pre-date
// this field keep working. Shared-key auth signs against the URL host,
// so for custom deployments the admin is responsible for ensuring the
// host actually serves the storage account named in the credential.
func buildAzureServiceURL(cloud, scheme, account, endpoint string) (string, error) {
switch cloud {
case model.AzureCloudCommercial, "":
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.windows.net/", scheme, account), nil
case model.AzureCloudGovernment:
if !model.IsValidAzureStorageAccountName(account) {
return "", fmt.Errorf("invalid azure storage account name %q", account)
}
return fmt.Sprintf("%s://%s.blob.core.usgovcloudapi.net/", scheme, account), nil
case model.AzureCloudCustom:
if endpoint == "" {
return "", errors.New("AzureCloud=custom requires AzureEndpoint to be set")
}
// The admin owns this URL end to end, but we still reject inputs
// that the SDK is guaranteed to fail on later (missing scheme,
// missing host, a scheme other than http/https) so the
// failure mode is a clear configuration error at startup rather
// than an opaque SDK error on the first blob request.
parsed, err := url.Parse(endpoint)
if err != nil {
return "", fmt.Errorf("AzureEndpoint is not a valid URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", fmt.Errorf("AzureEndpoint must use http or https, got %q", endpoint)
}
if parsed.Host == "" {
return "", fmt.Errorf("AzureEndpoint must include a host, got %q", endpoint)
}
return endpoint, nil
default:
return "", fmt.Errorf("unknown AzureCloud value %q", cloud)
}
}
// prefix joins the configured pathPrefix and the caller-supplied path.
// Using a plain path.Join, a value like "foo/../../secret" can escape
// the prefix entirely, so we compute the join and verify the result is
// the prefix directory itself or a descendant of it. The descendant check
// requires a path-separator boundary so a prefix of "mattermost" does not
// match a sibling like "mattermost-evil/...". If the joined path escapes,
// we fall back to joining the prefix with path.Base, which may drop any
// intermediate directories the caller intended.
func (b *AzureFileBackend) prefix(p string) string {
joined := path.Join(b.pathPrefix, p)
if b.pathPrefix == "" {
return joined
}
cleanPrefix := strings.TrimSuffix(path.Clean(b.pathPrefix), "/")
if joined == cleanPrefix || strings.HasPrefix(joined, cleanPrefix+"/") {
return joined
}
return path.Join(cleanPrefix, path.Base(p))
}
func (b *AzureFileBackend) newBlobClient(p string) *blob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newBlockBlobClient(p string) *blockblob.Client {
return b.client.ServiceClient().NewContainerClient(b.container).NewBlockBlobClient(b.prefix(p))
}
func (b *AzureFileBackend) newContainerClient() *container.Client {
return b.client.ServiceClient().NewContainerClient(b.container)
}
// TestConnection probes the configured container and reports the outcome
// using the typed errors shared with the other backends. Container
// creation is deliberately out of scope here - callers (Server.Start)
// decide whether to provision a missing container via MakeContainer.
// That separation keeps a typo in the System Console from silently
// provisioning an unwanted container, and matches the S3 contract where
// TestConnection returns FileBackendNoBucketError and MakeBucket is an
// explicit call.
func (b *AzureFileBackend) TestConnection() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newContainerClient().GetProperties(ctx, nil)
if err == nil {
return nil
}
if bloberror.HasCode(err, bloberror.ContainerNotFound) {
return &FileBackendNoBucketError{Err: fmt.Errorf("azure container %q does not exist: %w", b.container, err)}
}
if isAzureAuthError(err) {
return &FileBackendAuthError{Err: fmt.Errorf("unable to authenticate against azure blob storage: %w", err)}
}
return fmt.Errorf("unable to connect to azure blob storage: %w", err)
}
// MakeContainer creates the configured container. Mirrors S3FileBackend.MakeBucket
// so callers can opt into container provisioning explicitly. An already-existing
// container is treated as success so that concurrent boots (two nodes racing
// through TestConnection plus MakeContainer) both converge cleanly.
func (b *AzureFileBackend) MakeContainer() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if _, err := b.newContainerClient().Create(ctx, nil); err != nil {
if bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
return nil
}
return fmt.Errorf("unable to create azure container %q: %w", b.container, err)
}
return nil
}
func (b *AzureFileBackend) Reader(p string) (ReadCloseSeeker, error) {
// Arm the deadline *before* the first network call, then hand the same
// timer to the returned reader on success. The previous code only set up
// the timer on the happy path, which left GetProperties running against a
// no-deadline context.
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(b.timeout, cancel)
blobClient := b.newBlobClient(p)
props, err := blobClient.GetProperties(ctx, nil)
if err != nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("unable to read file %q: %w", p, err)
}
if props.ContentLength == nil {
timer.Stop()
cancel()
return nil, fmt.Errorf("missing content length for %q", p)
}
return &azureRangeReader{
ctx: ctx,
cancel: cancel,
timer: timer,
blobClient: blobClient,
size: *props.ContentLength,
}, nil
}
func (b *AzureFileBackend) ReadFile(p string) ([]byte, error) {
r, err := b.Reader(p)
if err != nil {
return nil, err
}
defer r.Close()
return io.ReadAll(r)
}
func (b *AzureFileBackend) FileExists(p string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return false, nil
}
return false, fmt.Errorf("unable to check existence of %q: %w", p, err)
}
return true, nil
}
func (b *AzureFileBackend) FileSize(p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return 0, fmt.Errorf("unable to get size of %q: %w", p, err)
}
return model.SafeDereference(props.ContentLength), nil
}
func (b *AzureFileBackend) FileModTime(p string) (time.Time, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
props, err := b.newBlobClient(p).GetProperties(ctx, nil)
if err != nil {
return time.Time{}, fmt.Errorf("unable to get modification time of %q: %w", p, err)
}
return model.SafeDereference(props.LastModified), nil
}
// CopyFile copies via StartCopyFromURL and polls the resulting blob's copy
// status until it succeeds, matching the synchronous semantics that the
// FileBackend interface (and the S3 driver via ComposeObject) provides.
func (b *AzureFileBackend) CopyFile(oldPath, newPath string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
src := b.newBlobClient(oldPath).URL()
dst := b.newBlockBlobClient(newPath)
if _, err := dst.StartCopyFromURL(ctx, src, nil); err != nil {
return fmt.Errorf("unable to copy %q to %q: %w", oldPath, newPath, err)
}
// Poll until the copy reports success. For server-to-server copies within
// the same account this is typically synchronous, but the API is
// asynchronous in general, so we wait.
for {
props, err := dst.GetProperties(ctx, nil)
if err != nil {
return fmt.Errorf("unable to read copy status for %q: %w", newPath, err)
}
if props.CopyStatus == nil {
return nil
}
switch *props.CopyStatus {
case blob.CopyStatusTypeSuccess:
return nil
case blob.CopyStatusTypeFailed, blob.CopyStatusTypeAborted:
desc := model.SafeDereference(props.CopyStatusDescription)
return fmt.Errorf("azure copy from %q to %q ended in status %q: %q", oldPath, newPath, *props.CopyStatus, desc)
}
select {
case <-ctx.Done():
return fmt.Errorf("azure copy from %q to %q did not complete in time: %w", oldPath, newPath, ctx.Err())
case <-time.After(50 * time.Millisecond):
}
}
}
func (b *AzureFileBackend) MoveFile(oldPath, newPath string) error {
if err := b.CopyFile(oldPath, newPath); err != nil {
return err
}
return b.RemoveFile(oldPath)
}
func (b *AzureFileBackend) WriteFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return b.WriteFileContext(ctx, fr, p)
}
// stageBlocks reads fr in azureBlockSize chunks and stages each chunk as a
// block under a fresh ID. Returns the IDs of the newly staged blocks (in
// order) and the total byte count. The caller is responsible for committing
// the block list.
func (b *AzureFileBackend) stageBlocks(ctx context.Context, bb *blockblob.Client, fr io.Reader, p string) ([]string, int64, error) {
buf := make([]byte, azureBlockSize)
var ids []string
var total int64
for {
n, err := io.ReadFull(fr, buf)
if n > 0 {
id, idErr := newAzureBlockID()
if idErr != nil {
return nil, 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(buf[:n])}, nil); sbErr != nil {
return nil, 0, fmt.Errorf("unable to stage block for %q: %w", p, sbErr)
}
ids = append(ids, id)
total += int64(n)
}
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
}
if err != nil {
return nil, 0, fmt.Errorf("failed to read input: %w", err)
}
}
return ids, total, nil
}
// WriteFileContext stages the body in fixed-size blocks and commits a fresh
// block list. It deliberately does not use the SDK's UploadStream helper:
// UploadStream's small-payload fast path falls back to single-shot PutBlob,
// which leaves the resulting blob with no committed block list. A subsequent
// AppendFile that calls CommitBlockList on that blob would then clobber its
// content. Routing every WriteFile through StageBlock + CommitBlockList keeps
// AppendFile correct regardless of payload size.
//
// The caller's context governs the entire upload - no inner timeout is added.
// TryWriteFileContext (filesstore.go) relies on this to let long-running
// callers like message-export bulk writes opt out of the per-operation
// timeout that WriteFile applies by default.
func (b *AzureFileBackend) WriteFileContext(ctx context.Context, fr io.Reader, p string) (int64, error) {
bb := b.newBlockBlobClient(p)
blockIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if len(blockIDs) == 0 {
// Empty input - still need to materialize an empty blob with a
// committed block list so AppendFile can target it.
id, idErr := newAzureBlockID()
if idErr != nil {
return 0, fmt.Errorf("failed to generate azure block id: %w", idErr)
}
if _, sbErr := bb.StageBlock(ctx, id, &readSeekNopCloser{Reader: bytes.NewReader(nil)}, nil); sbErr != nil {
return 0, fmt.Errorf("unable to stage empty block for %q: %w", p, sbErr)
}
blockIDs = append(blockIDs, id)
}
if _, err := bb.CommitBlockList(ctx, blockIDs, nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// AppendFile stages the new chunk as one or more blocks and commits the
// existing committed block list plus the newly staged IDs. Each AppendFile
// call uploads the new bytes exactly once - no re-download, no
// re-concatenate, no re-upload of the prior contents. The S3-style contract
// is preserved: returns an error if the target blob does not yet exist;
// returns the number of bytes appended (not the resulting total size).
//
// Refuses to append to a blob that has content but no committed block list
// (i.e. was uploaded via Put Blob by another tool - Azure portal, azcopy,
// a migration script). Committing a new block list against such a blob
// would replace the existing content with only the appended bytes, so
// failing loud beats silent data loss.
func (b *AzureFileBackend) AppendFile(fr io.Reader, p string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
bb := b.newBlockBlobClient(p)
listResp, err := bb.GetBlockList(ctx, blockblob.BlockListTypeCommitted, nil)
if err != nil {
return 0, fmt.Errorf("unable to find file %q to append data: %w", p, err)
}
var existingIDs []string
if listResp.BlockList.CommittedBlocks != nil {
for _, blk := range listResp.BlockList.CommittedBlocks {
if blk.Name != nil {
existingIDs = append(existingIDs, *blk.Name)
}
}
}
if len(existingIDs) == 0 {
props, propsErr := bb.GetProperties(ctx, nil)
if propsErr != nil {
return 0, fmt.Errorf("unable to inspect %q before append: %w", p, propsErr)
}
if model.SafeDereference(props.ContentLength) > 0 {
return 0, fmt.Errorf("refusing to append to %q: blob has content but no committed block list (likely written via Put Blob by another tool)", p)
}
}
newIDs, total, err := b.stageBlocks(ctx, bb, fr, p)
if err != nil {
return 0, err
}
if _, err := bb.CommitBlockList(ctx, append(existingIDs, newIDs...), nil); err != nil {
return 0, fmt.Errorf("unable to commit block list for %q: %w", p, err)
}
return total, nil
}
// GeneratePublicLink returns a time-limited, read-only URL to the blob at path
// using a Shared Access Signature. The SAS is auth-mode aware:
//
// - shared-key auth signs a Service SAS in-process with the stored
// SharedKeyCredential.
// - default-credential auth fetches a user-delegation key from Entra ID
// (one round trip per call) and signs a user-delegation SAS with it.
//
// This is intended for the export-download flow (App.GeneratePresignURLForExport
// and the /exportlink slash command). End users never reach this code path.
func (b *AzureFileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
if b.presignExpires <= 0 {
return "", 0, errors.New("azure presign expiration is not configured")
}
prefixed := b.prefix(path)
// Back-date the start by a small fixed amount to absorb minor clock skew
// between this host and Azure, matching the azure-sdk-for-go SAS examples:
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/sas/examples_test.go#L71
// https://github.com/Azure/azure-sdk-for-go/blob/65c3b792856d9ad7ce0b59c127ce299358e41a01/sdk/storage/azblob/service/examples_test.go#L315
const clockSkew = 10 * time.Second
start := time.Now().UTC().Add(-clockSkew)
expiry := start.Add(b.presignExpires)
protocol := sas.ProtocolHTTPSandHTTP
if b.ssl {
protocol = sas.ProtocolHTTPS
}
values := sas.BlobSignatureValues{
Protocol: protocol,
StartTime: start,
ExpiryTime: expiry,
Permissions: (&sas.BlobPermissions{Read: true}).String(),
ContainerName: b.container,
BlobName: prefixed,
// Request browsers to download the file instead of rendering it
ContentDisposition: "attachment",
}
var (
qps sas.QueryParameters
err error
)
switch b.authMode {
case model.AzureAuthModeSharedKey:
if b.sharedKey == nil {
return "", 0, errors.New("shared key credential is unavailable")
}
qps, err = values.SignWithSharedKey(b.sharedKey)
if err != nil {
return "", 0, fmt.Errorf("unable to sign service SAS for %q: %w", path, err)
}
case model.AzureAuthModeDefaultCredential:
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
udc, udcErr := b.client.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
Start: new(start.Format(sas.TimeFormat)),
Expiry: new(expiry.Format(sas.TimeFormat)),
}, nil)
if udcErr != nil {
return "", 0, fmt.Errorf("unable to obtain user delegation key for %q: %w", path, udcErr)
}
qps, err = values.SignWithUserDelegation(udc)
if err != nil {
return "", 0, fmt.Errorf("unable to sign user-delegation SAS for %q: %w", path, err)
}
default:
return "", 0, fmt.Errorf("unknown azure auth mode %q", b.authMode)
}
return b.newBlobClient(path).URL() + "?" + qps.Encode(), b.presignExpires, nil
}
func (b *AzureFileBackend) RemoveFile(p string) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.newBlobClient(p).Delete(ctx, nil)
if err != nil && !bloberror.HasCode(err, bloberror.BlobNotFound) {
return fmt.Errorf("unable to remove file %q: %w", p, err)
}
return nil
}
func (b *AzureFileBackend) ListDirectory(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
for _, item := range page.Segment.BlobPrefixes {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
name = strings.TrimSuffix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) ListDirectoryRecursively(p string) ([]string, error) {
prefix := b.prefix(p)
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
pager := b.newContainerClient().NewListBlobsFlatPager(&container.ListBlobsFlatOptions{
Prefix: &prefix,
})
var entries []string
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("unable to list directory %q recursively: %w", p, err)
}
for _, item := range page.Segment.BlobItems {
if item.Name == nil {
continue
}
name := strings.TrimPrefix(*item.Name, b.pathPrefix)
name = strings.TrimPrefix(name, "/")
entries = append(entries, name)
}
}
return entries, nil
}
func (b *AzureFileBackend) RemoveDirectory(p string) error {
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
for _, f := range files {
if err := b.RemoveFile(f); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) ZipReader(p string, deflate bool) (io.ReadCloser, error) {
method := zip.Store
if deflate {
method = zip.Deflate
}
pr, pw := io.Pipe()
go func() {
zw := zip.NewWriter(pw)
err := b.writeZip(zw, p, method)
if cerr := zw.Close(); err == nil {
err = cerr
}
pw.CloseWithError(err)
}()
return pr, nil
}
func (b *AzureFileBackend) writeZip(zw *zip.Writer, p string, method uint16) error {
exists, err := b.FileExists(p)
if err != nil {
return err
}
if exists {
return b.writeZipEntry(zw, p, path.Base(p), method)
}
files, err := b.ListDirectoryRecursively(p)
if err != nil {
return err
}
prefix := strings.TrimSuffix(p, "/") + "/"
for _, f := range files {
rel := strings.TrimPrefix(f, prefix)
if err := b.writeZipEntry(zw, f, rel, method); err != nil {
return err
}
}
return nil
}
func (b *AzureFileBackend) writeZipEntry(zw *zip.Writer, blobPath, name string, method uint16) error {
r, err := b.Reader(blobPath)
if err != nil {
return err
}
defer r.Close()
header := &zip.FileHeader{Name: name, Method: method}
header.SetMode(0644)
w, err := zw.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(w, r)
return err
}
// readSeekNopCloser adapts a Reader+Seeker into a ReadSeekCloser without
// closing the underlying source. The Azure SDK's StageBlock signature
// requires a ReadSeekCloser.
type readSeekNopCloser struct {
io.Reader
}
func (r *readSeekNopCloser) Seek(offset int64, whence int) (int64, error) {
return r.Reader.(io.Seeker).Seek(offset, whence)
}
func (r *readSeekNopCloser) Close() error { return nil }
// newAzureBlockID returns a fresh base64-encoded 16-byte random block ID,
// generated with github.com/google/uuid - the same library azblob uses
// internally for the block IDs it produces in UploadStream. All committed
// blocks in a single blob must share the same decoded length, so callers
// must use this for both WriteFile and AppendFile staging.
//
// Per https://learn.microsoft.com/en-us/rest/api/storageservices/put-block:
//
// For a given blob, all block IDs must be the same length. If a block is
// uploaded with a block ID of a different length than the block IDs for any
// existing uncommitted blocks, the service returns error response code 400
// (Bad Request).
func newAzureBlockID() (string, error) {
u, err := uuid.NewRandom()
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(u[:]), nil
}
func isAzureAuthError(err error) bool {
if err == nil {
return false
}
return bloberror.HasCode(err, bloberror.AuthenticationFailed) ||
bloberror.HasCode(err, bloberror.AuthorizationFailure) ||
bloberror.HasCode(err, bloberror.InvalidAuthenticationInfo)
}

View File

@@ -0,0 +1,160 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"io"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
pkgerr "github.com/pkg/errors"
)
// blobDownloader is the subset of *blob.Client used by azureRangeReader.
// Defined as an interface so tests can substitute a fake without standing up
// a real Azure client.
type blobDownloader interface {
DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)
}
// azureRangeReader is a seekable reader over an Azure blob, backed by HTTP
// Range requests. A stream is opened lazily on the first Read at the current
// offset; Seek closes any open stream so the next Read re-opens it from the
// new offset. The context is cancelled either by Close or by a timer set to
// the backend's configured timeout, matching the S3 driver's behavior.
//
// Callers constructing this struct directly must set ctx, cancel and timer;
// the methods below assume all three are non-nil.
type azureRangeReader struct {
ctx context.Context
cancel context.CancelFunc
timer *time.Timer
blobClient blobDownloader
size int64
offset int64
body io.ReadCloser
}
// Compile-time guarantees that azureRangeReader satisfies the interfaces the
// app layer relies on. zip.NewReader requires io.ReaderAt for archive
// readers (e.g. the bulk-import worker), and the import worker also
// type-asserts to a CancelTimeout interface for long-running operations.
var (
_ ReadCloseSeeker = (*azureRangeReader)(nil)
_ io.ReaderAt = (*azureRangeReader)(nil)
)
func (r *azureRangeReader) Read(p []byte) (int, error) {
if r.offset >= r.size {
return 0, io.EOF
}
if r.body == nil {
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
Range: blob.HTTPRange{Offset: r.offset, Count: 0},
})
if err != nil {
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
}
r.body = resp.Body
}
n, err := r.body.Read(p)
r.offset += int64(n)
if err == nil {
return n, nil
}
// Close+drop the body so the caller (or a retry) doesn't read more
// from a half-consumed stream, and so Close stays idempotent.
r.body.Close()
r.body = nil
if err == io.EOF && r.offset < r.size {
// The remote stream ended before we reached the blob's content
// length. Surface that as a truncation rather than a clean EOF
// so the caller doesn't accept a partial blob as complete.
return n, io.ErrUnexpectedEOF
}
return n, err
}
func (r *azureRangeReader) Seek(offset int64, whence int) (int64, error) {
var abs int64
switch whence {
case io.SeekStart:
abs = offset
case io.SeekCurrent:
abs = r.offset + offset
case io.SeekEnd:
abs = r.size + offset
default:
return 0, pkgerr.Errorf("invalid whence: %d", whence)
}
if abs < 0 {
return 0, pkgerr.Errorf("negative position: %d", abs)
}
if abs == r.offset {
return abs, nil
}
if r.body != nil {
r.body.Close()
r.body = nil
}
r.offset = abs
return abs, nil
}
// ReadAt reads len(p) bytes starting at offset off. Each call issues a
// dedicated ranged DownloadStream - calls do not affect the cursor that Read
// uses, matching the io.ReaderAt contract. This is what the bulk-import
// worker needs to feed zip.NewReader on Azure-backed deployments.
func (r *azureRangeReader) ReadAt(p []byte, off int64) (int, error) {
if off < 0 {
return 0, pkgerr.Errorf("negative offset: %d", off)
}
if off >= r.size {
return 0, io.EOF
}
count := int64(len(p))
if remaining := r.size - off; count > remaining {
count = remaining
}
resp, err := r.blobClient.DownloadStream(r.ctx, &blob.DownloadStreamOptions{
Range: blob.HTTPRange{Offset: off, Count: count},
})
if err != nil {
return 0, pkgerr.Wrap(err, "failed to open azure range stream")
}
defer resp.Body.Close()
n, err := io.ReadFull(resp.Body, p[:count])
// io.ReadFull returns ErrUnexpectedEOF when the stream terminates
// before count bytes arrive. Only collapse it to io.EOF when we
// actually filled the buffer and consumed the blob to the end -
// otherwise it is a real truncation that needs to surface so
// callers like zip.NewReader do not accept partial content.
if err == io.ErrUnexpectedEOF && int64(n) == count && off+int64(n) == r.size {
return n, io.EOF
}
if err == nil && off+int64(n) == r.size {
return n, io.EOF
}
return n, err
}
// CancelTimeout stops the timer that bounds this reader's lifetime, so
// long-running consumers (e.g. the bulk-import worker, which can run far
// past the default per-operation timeout) can opt out of the automatic
// cancellation. Returns false if the timer has already fired.
func (r *azureRangeReader) CancelTimeout() bool {
return r.timer.Stop()
}
func (r *azureRangeReader) Close() error {
if r.timer != nil {
r.timer.Stop()
}
r.cancel()
if r.body != nil {
return r.body.Close()
}
return nil
}

View File

@@ -0,0 +1,361 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"errors"
"io"
"testing"
"time"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/stretchr/testify/require"
)
// trackingReadCloser wraps a Reader and records whether Close was called.
type trackingReadCloser struct {
io.Reader
closed bool
}
func (t *trackingReadCloser) Close() error {
t.closed = true
return nil
}
// fakeDownloader serves bytes from an in-memory blob, records every
// DownloadStream call's Range, and hands out trackingReadClosers so tests
// can assert close-on-Seek behavior. An optional err short-circuits responses.
type fakeDownloader struct {
data []byte
calls []blob.HTTPRange
bodies []*trackingReadCloser
err error
}
func (f *fakeDownloader) DownloadStream(_ context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
if f.err != nil {
return blob.DownloadStreamResponse{}, f.err
}
var rng blob.HTTPRange
if opts != nil {
rng = opts.Range
}
f.calls = append(f.calls, rng)
start := min(max(rng.Offset, 0), int64(len(f.data)))
end := int64(len(f.data))
if rng.Count > 0 && start+rng.Count < end {
end = start + rng.Count
}
body := &trackingReadCloser{Reader: bytes.NewReader(f.data[start:end])}
f.bodies = append(f.bodies, body)
return blob.DownloadStreamResponse{
DownloadResponse: blob.DownloadResponse{Body: body},
}, nil
}
// newTestReader returns an azureRangeReader wired to the given fake, with a
// long-lived timer so it never fires during the test. Caller must Close it.
func newTestReader(t *testing.T, fake *fakeDownloader, size int64) *azureRangeReader {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
timer := time.AfterFunc(time.Hour, cancel)
return &azureRangeReader{
ctx: ctx,
cancel: cancel,
timer: timer,
blobClient: fake,
size: size,
}
}
func TestRead(t *testing.T) {
t.Run("returns EOF at end of blob without downloading", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(0, io.SeekEnd)
require.NoError(t, err)
n, err := r.Read(make([]byte, 4))
require.Equal(t, 0, n)
require.Equal(t, io.EOF, err)
require.Empty(t, fake.calls, "no download should be issued past end of blob")
})
t.Run("opens stream at current offset", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("hello world")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(6, io.SeekStart)
require.NoError(t, err)
buf := make([]byte, 5)
n, err := io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, 5, n)
require.Equal(t, "world", string(buf))
require.Len(t, fake.calls, 1)
require.Equal(t, blob.HTTPRange{Offset: 6, Count: 0}, fake.calls[0])
})
t.Run("sequential reads reuse the open stream", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
buf := make([]byte, 4)
_, err := io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "abcd", string(buf))
_, err = io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "efgh", string(buf))
require.Len(t, fake.calls, 1, "sequential reads must reuse the open stream")
})
t.Run("propagates download errors", func(t *testing.T) {
wantErr := errors.New("boom")
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Read(make([]byte, 1))
require.ErrorIs(t, err, wantErr)
})
t.Run("surfaces truncation when stream EOFs before the blob ends", func(t *testing.T) {
// Promised size is larger than what the fake actually serves,
// so the body eventually returns io.EOF while r.offset < r.size.
// bytes.Reader returns its content + nil first, then 0 + EOF on
// the next call, so we drain the bytes before the truncation
// is observable.
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data))+10)
defer r.Close()
buf := make([]byte, 16)
n, err := r.Read(buf)
require.NoError(t, err)
require.Equal(t, 5, n)
// Second call hits EOF from the body before we've reached r.size,
// so the reader must surface that as a truncation.
n, err = r.Read(buf)
require.Equal(t, 0, n)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
require.Nil(t, r.body, "body must be released after a truncation error")
})
}
func TestReadAt(t *testing.T) {
t.Run("reads at the given offset without disturbing the cursor", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
// Advance the streaming cursor first.
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Equal(t, int64(3), r.offset)
buf := make([]byte, 4)
n, err := r.ReadAt(buf, 5)
require.NoError(t, err)
require.Equal(t, 4, n)
require.Equal(t, "fghi", string(buf))
require.Equal(t, int64(3), r.offset, "ReadAt must not touch the streaming offset")
})
t.Run("returns io.EOF when the read lands exactly at the end of the blob", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
buf := make([]byte, 3)
n, err := r.ReadAt(buf, 7)
require.Equal(t, io.EOF, err)
require.Equal(t, 3, n)
require.Equal(t, "hij", string(buf))
})
t.Run("returns io.EOF when off is past the size", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
n, err := r.ReadAt(make([]byte, 4), 100)
require.Equal(t, 0, n)
require.Equal(t, io.EOF, err)
require.Empty(t, fake.calls, "no download should be issued past end of blob")
})
t.Run("rejects negative offsets", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
defer r.Close()
_, err := r.ReadAt(make([]byte, 1), -1)
require.Error(t, err)
})
t.Run("propagates download errors", func(t *testing.T) {
wantErr := errors.New("boom")
fake := &fakeDownloader{data: []byte("xyz"), err: wantErr}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.ReadAt(make([]byte, 1), 0)
require.ErrorIs(t, err, wantErr)
})
t.Run("surfaces truncation when stream falls short of the requested count", func(t *testing.T) {
// Promised size exceeds the fake's actual data so ReadFull
// sees the body terminate before count bytes arrived. That
// must surface as ErrUnexpectedEOF, not a clean EOF.
fake := &fakeDownloader{data: []byte("hello")}
r := newTestReader(t, fake, int64(len(fake.data))+5)
defer r.Close()
buf := make([]byte, 10)
n, err := r.ReadAt(buf, 0)
require.Equal(t, 5, n)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
})
}
func TestCancelTimeout(t *testing.T) {
fake := &fakeDownloader{data: []byte("abc")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
require.True(t, r.CancelTimeout(), "first stop should succeed")
require.False(t, r.CancelTimeout(), "second stop must report the timer was already stopped")
}
func TestSeek(t *testing.T) {
t.Run("absolute from start", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
pos, err := r.Seek(10, io.SeekStart)
require.NoError(t, err)
require.Equal(t, int64(10), pos)
})
t.Run("relative to current position", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := r.Seek(10, io.SeekStart)
require.NoError(t, err)
pos, err := r.Seek(5, io.SeekCurrent)
require.NoError(t, err)
require.Equal(t, int64(15), pos)
})
t.Run("relative to end", func(t *testing.T) {
fake := &fakeDownloader{data: bytes.Repeat([]byte("x"), 32)}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
pos, err := r.Seek(-4, io.SeekEnd)
require.NoError(t, err)
require.Equal(t, int64(28), pos)
})
t.Run("rejects invalid whence", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 0)
defer r.Close()
_, err := r.Seek(0, 99)
require.Error(t, err)
})
t.Run("rejects negative absolute position", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
defer r.Close()
_, err := r.Seek(-1, io.SeekStart)
require.Error(t, err)
_, err = r.Seek(-20, io.SeekEnd)
require.Error(t, err)
})
t.Run("same offset leaves the open stream untouched", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefgh")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
openBody := fake.bodies[0]
pos, err := r.Seek(3, io.SeekStart)
require.NoError(t, err)
require.Equal(t, int64(3), pos)
require.False(t, openBody.closed, "same-offset seek must not close the open stream")
_, err = io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.calls, 1, "same-offset seek must not trigger a new download")
})
t.Run("different offset closes the open stream and the next read reopens", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdefghij")}
r := newTestReader(t, fake, int64(len(fake.data)))
defer r.Close()
_, err := io.ReadFull(r, make([]byte, 2))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
firstBody := fake.bodies[0]
_, err = r.Seek(7, io.SeekStart)
require.NoError(t, err)
require.True(t, firstBody.closed, "seek to a new offset must close the open stream")
buf := make([]byte, 3)
_, err = io.ReadFull(r, buf)
require.NoError(t, err)
require.Equal(t, "hij", string(buf))
require.Len(t, fake.calls, 2)
require.Equal(t, int64(7), fake.calls[1].Offset)
})
}
func TestClose(t *testing.T) {
t.Run("cancels context and closes the open body", func(t *testing.T) {
fake := &fakeDownloader{data: []byte("abcdef")}
r := newTestReader(t, fake, int64(len(fake.data)))
_, err := io.ReadFull(r, make([]byte, 3))
require.NoError(t, err)
require.Len(t, fake.bodies, 1)
require.NoError(t, r.Close())
require.True(t, fake.bodies[0].closed)
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
})
t.Run("works when no stream was opened", func(t *testing.T) {
r := newTestReader(t, &fakeDownloader{}, 10)
require.NoError(t, r.Close())
require.ErrorIs(t, r.ctx.Err(), context.Canceled)
})
}

View File

@@ -0,0 +1,466 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"net/url"
"os"
"testing"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
func TestBuildAzureServiceURL(t *testing.T) {
const account = "acmemattermost"
tests := []struct {
name string
cloud string
scheme string
account string // defaults to the package-level account const when empty
endpoint string
expected string
wantErr bool
}{
{
name: "commercial cloud, default scheme",
cloud: model.AzureCloudCommercial,
scheme: "https",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "commercial cloud rejects an account name with a hash character",
cloud: model.AzureCloudCommercial,
scheme: "https",
account: "account#",
wantErr: true,
},
{
name: "commercial cloud rejects an account name with a slash character",
cloud: model.AzureCloudCommercial,
scheme: "https",
account: "account/",
wantErr: true,
},
{
name: "government cloud rejects a malformed account name",
cloud: model.AzureCloudGovernment,
scheme: "https",
account: "account#",
wantErr: true,
},
{
name: "empty cloud falls back to commercial for legacy configs",
cloud: "",
scheme: "https",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "government cloud uses the usgovcloudapi suffix",
cloud: model.AzureCloudGovernment,
scheme: "https",
expected: "https://acmemattermost.blob.core.usgovcloudapi.net/",
},
{
name: "custom cloud returns the endpoint verbatim (vhost-style)",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https://acmemattermost.blob.core.windows.net/",
expected: "https://acmemattermost.blob.core.windows.net/",
},
{
name: "custom cloud returns the endpoint verbatim (Azurite path-style)",
cloud: model.AzureCloudCustom,
scheme: "http",
endpoint: "http://localhost:10000/devstoreaccount1/",
expected: "http://localhost:10000/devstoreaccount1/",
},
{
name: "custom cloud preserves arbitrary paths the admin provides",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https://blob-proxy.internal.example.com/some/prefix/account/",
expected: "https://blob-proxy.internal.example.com/some/prefix/account/",
},
{
name: "custom cloud rejects an empty endpoint",
cloud: model.AzureCloudCustom,
scheme: "https",
wantErr: true,
},
{
name: "custom cloud rejects an endpoint missing the scheme",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "acmemattermost.blob.core.windows.net/",
wantErr: true,
},
{
name: "custom cloud rejects an endpoint with no host",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "https:///acmemattermost/",
wantErr: true,
},
{
name: "custom cloud rejects a non-HTTP scheme",
cloud: model.AzureCloudCustom,
scheme: "https",
endpoint: "ftp://blob.example.com/",
wantErr: true,
},
{
name: "unknown cloud value is rejected",
cloud: "azuregermany",
scheme: "https",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
acct := account
if tt.account != "" {
acct = tt.account
}
got, err := buildAzureServiceURL(tt.cloud, tt.scheme, acct, tt.endpoint)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
require.Equal(t, tt.expected, got)
})
}
}
func TestAzureFileBackendPrefix(t *testing.T) {
tests := []struct {
name string
prefix string
input string
expected string
}{
{name: "no prefix, plain path", prefix: "", input: "team/channel/file", expected: "team/channel/file"},
{name: "no prefix, with dot-dot", prefix: "", input: "../escape", expected: "../escape"},
{name: "prefix, plain path", prefix: "mattermost", input: "team/channel/file", expected: "mattermost/team/channel/file"},
{name: "prefix, exact root", prefix: "mattermost", input: "", expected: "mattermost"},
{name: "prefix, dot-dot escapes", prefix: "mattermost", input: "../escape", expected: "mattermost/escape"},
{name: "prefix, nested dot-dot escapes", prefix: "mattermost", input: "sub/../../escape", expected: "mattermost/escape"},
{name: "prefix, dot-dot in middle stays inside", prefix: "mattermost", input: "a/../b", expected: "mattermost/b"},
{name: "prefix with trailing slash, dot-dot escapes", prefix: "mattermost/", input: "../escape", expected: "mattermost/escape"},
{name: "prefix boundary collision must not escape", prefix: "mattermost", input: "../mattermost-evil/file", expected: "mattermost/file"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &AzureFileBackend{pathPrefix: tt.prefix}
require.Equal(t, tt.expected, b.prefix(tt.input))
})
}
}
// azuriteWellKnownAccount and azuriteWellKnownKey are Azurite's published
// development credentials. They are not secrets - they are documented in the
// Azurite README and ship hardcoded in every Azurite distribution.
const (
azuriteWellKnownAccount = "devstoreaccount1"
azuriteWellKnownKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
)
// TestAzureFileBackendAppendRefusesNonBlockBlob exercises the safety
// check in AppendFile: when a blob exists with content but no committed
// block list (i.e. it was uploaded via Put Blob by another tool), the
// backend must refuse the append rather than silently destroy the
// existing content.
func TestAzureFileBackendAppendRefusesNonBlockBlob(t *testing.T) {
be := newAzuriteBackend(t)
path := "append-refusal-test.bin"
t.Cleanup(func() { _ = be.RemoveFile(path) })
// Write the blob via the high-level Upload helper, which calls the
// Put Blob REST endpoint and leaves the committed-block list empty.
original := []byte("planted-by-another-tool")
bb := be.newBlockBlobClient(path)
_, err := bb.Upload(context.Background(), nopReadSeekCloser{bytes.NewReader(original)}, nil)
require.NoError(t, err)
_, err = be.AppendFile(bytes.NewReader([]byte("would-overwrite")), path)
require.Error(t, err)
require.Contains(t, err.Error(), "no committed block list")
// The original content must still be intact.
got, err := be.ReadFile(path)
require.NoError(t, err)
require.Equal(t, original, got)
}
// TestAzureFileBackendMakeContainerIdempotent ensures that calling
// MakeContainer twice on the same backend is a no-op the second time.
// Two nodes can race through TestConnection plus MakeContainer at boot;
// the loser must converge instead of returning an error.
func TestAzureFileBackendMakeContainerIdempotent(t *testing.T) {
be := newAzuriteBackend(t)
require.NoError(t, be.MakeContainer())
require.NoError(t, be.MakeContainer())
}
type nopReadSeekCloser struct {
*bytes.Reader
}
func (nopReadSeekCloser) Close() error { return nil }
// newAzuriteBackend builds an Azure backend pointed at the Azurite emulator
// and ensures the container exists. Standalone Azure tests should use this
// instead of calling NewAzureFileBackend + TestConnection directly; the
// shared FileBackendTestSuite handles provisioning itself in SetupTest.
func newAzuriteBackend(t *testing.T) *AzureFileBackend {
t.Helper()
be, err := NewAzureFileBackend(azuriteSettings(t))
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if err := be.TestConnection(); errors.As(err, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, err)
}
return be
}
func azuriteSettings(t *testing.T) FileBackendSettings {
t.Helper()
host := os.Getenv("CI_AZURITE_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("CI_AZURITE_PORT")
if port == "" {
port = "10000"
}
return FileBackendSettings{
DriverName: driverAzure,
AzureStorageAccount: azuriteWellKnownAccount,
AzureAuthMode: model.AzureAuthModeSharedKey,
AzureAccessKey: azuriteWellKnownKey,
AzureContainer: "mattermost-test",
AzureCloud: model.AzureCloudCustom,
AzureEndpoint: "http://" + net.JoinHostPort(host, port) + "/" + azuriteWellKnownAccount + "/",
// The emulator runs on an internal address, so its host must be allowed.
AllowedUntrustedInternalConnections: host,
AzureRequestTimeoutMilliseconds: 30000,
}
}
func TestNewAzureFileBackendAuthMode(t *testing.T) {
base := FileBackendSettings{
DriverName: driverAzure,
AzureStorageAccount: "anaccount",
AzureContainer: "acontainer",
AzureEndpoint: "localhost:10000",
AzureSSL: false,
AzureRequestTimeoutMilliseconds: 30000,
}
t.Run("shared_key constructs a client", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeSharedKey
s.AzureAccessKey = azuriteWellKnownKey
be, err := NewAzureFileBackend(s)
require.NoError(t, err)
require.NotNil(t, be.client)
})
t.Run("default_credential constructs a client without an access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
// Intentionally no AzureAccessKey - default credential reads
// identity from the host environment, not config.
be, err := NewAzureFileBackend(s)
require.NoError(t, err)
require.NotNil(t, be.client)
})
t.Run("empty AuthMode is rejected", func(t *testing.T) {
s := base
s.AzureAuthMode = ""
s.AzureAccessKey = azuriteWellKnownKey
_, err := NewAzureFileBackend(s)
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
t.Run("unknown AuthMode is rejected", func(t *testing.T) {
s := base
s.AzureAuthMode = "oauth2"
s.AzureAccessKey = azuriteWellKnownKey
_, err := NewAzureFileBackend(s)
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
}
func TestCheckMandatoryAzureFieldsAuthMode(t *testing.T) {
base := FileBackendSettings{
AzureStorageAccount: "anaccount",
AzureContainer: "acontainer",
}
t.Run("shared_key requires access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeSharedKey
s.AzureAccessKey = ""
require.Error(t, s.CheckMandatoryAzureFields())
s.AzureAccessKey = "somekey"
require.NoError(t, s.CheckMandatoryAzureFields())
})
t.Run("default_credential does not require access key", func(t *testing.T) {
s := base
s.AzureAuthMode = model.AzureAuthModeDefaultCredential
s.AzureAccessKey = ""
require.NoError(t, s.CheckMandatoryAzureFields())
})
}
func TestAzureFileBackendTestSuite(t *testing.T) {
suite.Run(t, &FileBackendTestSuite{settings: azuriteSettings(t)})
}
// TestAzureGeneratePublicLink exercises the Service SAS path against Azurite.
// The user-delegation SAS path requires Entra ID and cannot run in CI; it is
// covered by the manual verification recipe in the PR description.
func TestAzureGeneratePublicLink(t *testing.T) {
t.Run("refuses to issue a link when presign expiration is unset", func(t *testing.T) {
settings := azuriteSettings(t)
// Deliberately leave AzurePresignExpiresSeconds at zero - the
// shape an admin would hit if they enabled the export-direct-
// download feature without ever configuring the field.
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
_, _, err = be.GeneratePublicLink("any/path")
require.Error(t, err)
require.Contains(t, err.Error(), "presign expiration is not configured")
})
t.Run("shared-key SAS round-trips through Azurite", func(t *testing.T) {
settings := azuriteSettings(t)
settings.AzurePresignExpiresSeconds = 300
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, connErr)
}
path := "tests/presign-" + model.NewId() + ".bin"
payload := bytes.Repeat([]byte("mattermost"), 4096) // 40 KiB
_, writeErr := be.WriteFile(bytes.NewReader(payload), path)
require.NoError(t, writeErr)
t.Cleanup(func() { _ = be.RemoveFile(path) })
link, expiry, err := be.GeneratePublicLink(path)
require.NoError(t, err)
require.NotEmpty(t, link)
require.Equal(t, 300*time.Second, expiry)
// SAS query parameters Azurite expects to see.
u, err := url.Parse(link)
require.NoError(t, err)
require.NotEmpty(t, u.Query().Get("sig"))
require.Equal(t, "r", u.Query().Get("sp"))
require.Equal(t, "attachment", u.Query().Get("rscd"))
resp, err := http.Get(link)
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
require.Equal(t, http.StatusOK, resp.StatusCode)
require.Equal(t, "attachment", resp.Header.Get("Content-Disposition"))
got, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, payload, got)
})
t.Run("tampered SAS is rejected by the storage service", func(t *testing.T) {
settings := azuriteSettings(t)
settings.AzurePresignExpiresSeconds = 300
be, err := NewAzureFileBackend(settings)
require.NoError(t, err)
var noBucket *FileBackendNoBucketError
if connErr := be.TestConnection(); errors.As(connErr, &noBucket) {
require.NoError(t, be.MakeContainer())
} else {
require.NoError(t, connErr)
}
path := "tests/tamper-" + model.NewId() + ".bin"
_, writeErr := be.WriteFile(bytes.NewReader([]byte("contents")), path)
require.NoError(t, writeErr)
t.Cleanup(func() { _ = be.RemoveFile(path) })
link, _, err := be.GeneratePublicLink(path)
require.NoError(t, err)
// Flip the last character of the signature so the link no longer
// matches what Azurite signed.
u, err := url.Parse(link)
require.NoError(t, err)
q := u.Query()
sig := q.Get("sig")
require.NotEmpty(t, sig)
swapped := sig[:len(sig)-1] + flipBase64Char(sig[len(sig)-1])
q.Set("sig", swapped)
u.RawQuery = q.Encode()
resp, err := http.Get(u.String())
require.NoError(t, err)
t.Cleanup(func() { _ = resp.Body.Close() })
require.Equal(t, http.StatusForbidden, resp.StatusCode)
})
t.Run("unknown auth mode is rejected", func(t *testing.T) {
// Build a backend by hand so we can exercise the default-branch
// guard. Going through NewAzureFileBackend would be rejected by
// CheckMandatoryAzureFields first.
be := &AzureFileBackend{
authMode: "totally-not-real",
presignExpires: time.Minute,
}
_, _, err := be.GeneratePublicLink("any/path")
require.Error(t, err)
require.Contains(t, err.Error(), "unknown azure auth mode")
})
}
// flipBase64Char swaps a base64 character for another in the same alphabet
// to produce a still-valid-but-wrong signature character. Used only by the
// tamper test above.
func flipBase64Char(c byte) string {
if c == 'A' {
return "B"
}
return "A"
}

View File

@@ -0,0 +1,44 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
// FileBackendAuthError is returned when testing a connection and authentication
// against the file storage backend fails. Backends should wrap the underlying
// auth failure in this type so the admin Test Connection flow can surface a
// useful message regardless of which driver is configured.
type FileBackendAuthError struct {
// Err is the underlying driver error, if any.
Err error
// DetailedError is a human-readable message describing the failure.
// Kept for compatibility with the previous S3-specific type.
DetailedError string
}
func (e *FileBackendAuthError) Error() string {
if e.DetailedError != "" {
return e.DetailedError
}
if e.Err != nil {
return e.Err.Error()
}
return "authentication failed"
}
func (e *FileBackendAuthError) Unwrap() error { return e.Err }
// FileBackendNoBucketError is returned when testing a connection and the
// configured bucket / container does not exist.
type FileBackendNoBucketError struct {
// Err is the underlying driver error, if any.
Err error
}
func (e *FileBackendNoBucketError) Error() string {
if e.Err != nil {
return e.Err.Error()
}
return "no such bucket or container"
}
func (e *FileBackendNoBucketError) Unwrap() error { return e.Err }

View File

@@ -0,0 +1,250 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"io"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/pkg/errors"
)
const (
driverS3 = "amazons3"
driverLocal = "local"
driverAzure = "azureblob"
)
type ReadCloseSeeker interface {
io.ReadCloser
io.Seeker
}
type FileBackend interface {
DriverName() string
TestConnection() error
Reader(path string) (ReadCloseSeeker, error)
ReadFile(path string) ([]byte, error)
FileExists(path string) (bool, error)
FileSize(path string) (int64, error)
CopyFile(oldPath, newPath string) error
MoveFile(oldPath, newPath string) error
WriteFile(fr io.Reader, path string) (int64, error)
AppendFile(fr io.Reader, path string) (int64, error)
RemoveFile(path string) error
FileModTime(path string) (time.Time, error)
ListDirectory(path string) ([]string, error)
ListDirectoryRecursively(path string) ([]string, error)
RemoveDirectory(path string) error
ZipReader(path string, deflate bool) (io.ReadCloser, error)
}
type FileBackendWithLinkGenerator interface {
GeneratePublicLink(path string) (string, time.Duration, error)
}
type FileBackendSettings struct {
DriverName string
Directory string
AmazonS3AccessKeyId string
AmazonS3SecretAccessKey string
AmazonS3Bucket string
AmazonS3PathPrefix string
AmazonS3Region string
AmazonS3Endpoint string
AmazonS3SSL bool
AmazonS3SignV2 bool
AmazonS3SSE bool
AmazonS3Trace bool
SkipVerify bool
AmazonS3RequestTimeoutMilliseconds int64
AmazonS3PresignExpiresSeconds int64
AmazonS3UploadPartSizeBytes int64
AmazonS3StorageClass string
AzureStorageAccount string
AzureAuthMode string
AzureAccessKey string
AzureContainer string
AzurePathPrefix string
AzureCloud string
AzureEndpoint string
AzureSSL bool
AzureRequestTimeoutMilliseconds int64
AzurePresignExpiresSeconds int64
// AllowedUntrustedInternalConnections carries the value of the
// ServiceSettings.AllowedUntrustedInternalConnections config setting, used
// by the Azure custom-cloud backend.
AllowedUntrustedInternalConnections string
}
func NewFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
if *fileSettings.DriverName == model.ImageDriverLocal {
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
Directory: *fileSettings.Directory,
}
}
if *fileSettings.DriverName == model.ImageDriverAzure {
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
AzureStorageAccount: *fileSettings.AzureStorageAccount,
AzureAuthMode: *fileSettings.AzureAuthMode,
AzureAccessKey: *fileSettings.AzureAccessKey,
AzureContainer: *fileSettings.AzureContainer,
AzurePathPrefix: *fileSettings.AzurePathPrefix,
AzureCloud: *fileSettings.AzureCloud,
AzureEndpoint: *fileSettings.AzureEndpoint,
AzureSSL: fileSettings.AzureSSL == nil || *fileSettings.AzureSSL,
AzureRequestTimeoutMilliseconds: *fileSettings.AzureRequestTimeoutMilliseconds,
SkipVerify: skipVerify,
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
}
}
return FileBackendSettings{
DriverName: *fileSettings.DriverName,
AmazonS3AccessKeyId: *fileSettings.AmazonS3AccessKeyId,
AmazonS3SecretAccessKey: *fileSettings.AmazonS3SecretAccessKey,
AmazonS3Bucket: *fileSettings.AmazonS3Bucket,
AmazonS3PathPrefix: *fileSettings.AmazonS3PathPrefix,
AmazonS3Region: *fileSettings.AmazonS3Region,
AmazonS3Endpoint: *fileSettings.AmazonS3Endpoint,
AmazonS3SSL: fileSettings.AmazonS3SSL == nil || *fileSettings.AmazonS3SSL,
AmazonS3SignV2: fileSettings.AmazonS3SignV2 != nil && *fileSettings.AmazonS3SignV2,
AmazonS3SSE: fileSettings.AmazonS3SSE != nil && *fileSettings.AmazonS3SSE && enableComplianceFeature,
AmazonS3Trace: fileSettings.AmazonS3Trace != nil && *fileSettings.AmazonS3Trace,
AmazonS3RequestTimeoutMilliseconds: *fileSettings.AmazonS3RequestTimeoutMilliseconds,
SkipVerify: skipVerify,
AmazonS3UploadPartSizeBytes: *fileSettings.AmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.AmazonS3StorageClass,
}
}
func NewExportFileBackendSettingsFromConfig(fileSettings *model.FileSettings, enableComplianceFeature bool, skipVerify bool, allowedUntrustedInternalConnections string) FileBackendSettings {
if *fileSettings.ExportDriverName == model.ImageDriverLocal {
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
Directory: *fileSettings.ExportDirectory,
}
}
if *fileSettings.ExportDriverName == model.ImageDriverAzure {
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
AzureStorageAccount: *fileSettings.ExportAzureStorageAccount,
AzureAuthMode: *fileSettings.ExportAzureAuthMode,
AzureAccessKey: *fileSettings.ExportAzureAccessKey,
AzureContainer: *fileSettings.ExportAzureContainer,
AzurePathPrefix: *fileSettings.ExportAzurePathPrefix,
AzureCloud: *fileSettings.ExportAzureCloud,
AzureEndpoint: *fileSettings.ExportAzureEndpoint,
AzureSSL: fileSettings.ExportAzureSSL == nil || *fileSettings.ExportAzureSSL,
AzureRequestTimeoutMilliseconds: *fileSettings.ExportAzureRequestTimeoutMilliseconds,
AzurePresignExpiresSeconds: *fileSettings.ExportAzurePresignExpiresSeconds,
SkipVerify: skipVerify,
AllowedUntrustedInternalConnections: allowedUntrustedInternalConnections,
}
}
return FileBackendSettings{
DriverName: *fileSettings.ExportDriverName,
AmazonS3AccessKeyId: *fileSettings.ExportAmazonS3AccessKeyId,
AmazonS3SecretAccessKey: *fileSettings.ExportAmazonS3SecretAccessKey,
AmazonS3Bucket: *fileSettings.ExportAmazonS3Bucket,
AmazonS3PathPrefix: *fileSettings.ExportAmazonS3PathPrefix,
AmazonS3Region: *fileSettings.ExportAmazonS3Region,
AmazonS3Endpoint: *fileSettings.ExportAmazonS3Endpoint,
AmazonS3SSL: fileSettings.ExportAmazonS3SSL == nil || *fileSettings.ExportAmazonS3SSL,
AmazonS3SignV2: fileSettings.ExportAmazonS3SignV2 != nil && *fileSettings.ExportAmazonS3SignV2,
AmazonS3SSE: fileSettings.ExportAmazonS3SSE != nil && *fileSettings.ExportAmazonS3SSE && enableComplianceFeature,
AmazonS3Trace: fileSettings.ExportAmazonS3Trace != nil && *fileSettings.ExportAmazonS3Trace,
AmazonS3RequestTimeoutMilliseconds: *fileSettings.ExportAmazonS3RequestTimeoutMilliseconds,
AmazonS3PresignExpiresSeconds: *fileSettings.ExportAmazonS3PresignExpiresSeconds,
AmazonS3UploadPartSizeBytes: *fileSettings.ExportAmazonS3UploadPartSizeBytes,
AmazonS3StorageClass: *fileSettings.ExportAmazonS3StorageClass,
SkipVerify: skipVerify,
}
}
func (settings *FileBackendSettings) CheckMandatoryS3Fields() error {
if settings.AmazonS3Bucket == "" {
return errors.New("missing s3 bucket settings")
}
// if S3 endpoint is not set call the set defaults to set that
if settings.AmazonS3Endpoint == "" {
settings.AmazonS3Endpoint = "s3.amazonaws.com"
}
return nil
}
func (settings *FileBackendSettings) CheckMandatoryAzureFields() error {
if settings.AzureStorageAccount == "" {
return errors.New("missing azure storage account setting")
}
if settings.AzureContainer == "" {
return errors.New("missing azure container setting")
}
// AzureAccessKey is only meaningful for shared-key auth. Default credential
// reads identity from the host environment (managed identity / workload
// identity / service principal env vars / az login), so an empty access key
// is the expected configuration in that mode.
if settings.AzureAuthMode == model.AzureAuthModeSharedKey && settings.AzureAccessKey == "" {
return errors.New("missing azure access key setting")
}
return nil
}
// NewFileBackend creates a new file backend
func NewFileBackend(settings FileBackendSettings) (FileBackend, error) {
return newFileBackend(settings, true)
}
// NewExportFileBackend creates a new file backend for exports, that will not attempt to use bifrost.
func NewExportFileBackend(settings FileBackendSettings) (FileBackend, error) {
return newFileBackend(settings, false)
}
func newFileBackend(settings FileBackendSettings, canBeCloud bool) (FileBackend, error) {
switch settings.DriverName {
case driverS3:
newBackendFn := NewS3FileBackend
if !canBeCloud {
newBackendFn = NewS3FileBackendWithoutBifrost
}
backend, err := newBackendFn(settings)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the s3 backend")
}
return backend, nil
case driverLocal:
return &LocalFileBackend{
directory: settings.Directory,
}, nil
case driverAzure:
backend, err := NewAzureFileBackend(settings)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the azure backend")
}
return backend, nil
}
return nil, errors.New("no valid filestorage driver found")
}
// TryWriteFileContext checks if the file backend supports context writes and passes the context in that case.
// Should the file backend not support contexts, it just calls WriteFile instead. This can be used to disable
// the timeouts for long writes (like exports).
func TryWriteFileContext(ctx context.Context, fb FileBackend, fr io.Reader, path string) (int64, error) {
type ContextWriter interface {
WriteFileContext(context.Context, io.Reader, string) (int64, error)
}
if cw, ok := fb.(ContextWriter); ok {
return cw.WriteFileContext(ctx, fr, path)
}
return fb.WriteFile(fr, path)
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,347 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"time"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
const (
TestFilePath = "/testfile"
MaxRecursionDepth = 50
)
type LocalFileBackend struct {
directory string
}
// copyFile will copy a file from src path to dst path.
// Overwrites any existing files at dst.
// Permissions are copied from file at src to the new file at dst.
func copyFile(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
return
}
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
if e := out.Close(); e != nil {
err = e
}
}()
_, err = io.Copy(out, in)
if err != nil {
return
}
err = out.Sync()
if err != nil {
return
}
stat, err := os.Stat(src)
if err != nil {
return
}
err = os.Chmod(dst, stat.Mode())
if err != nil {
return
}
return
}
func (b *LocalFileBackend) DriverName() string {
return driverLocal
}
func (b *LocalFileBackend) TestConnection() error {
f := bytes.NewReader([]byte("testingwrite"))
if _, err := writeFileLocally(f, filepath.Join(b.directory, TestFilePath)); err != nil {
return errors.Wrap(err, "unable to write to the local filesystem storage")
}
os.Remove(filepath.Join(b.directory, TestFilePath))
mlog.Debug("Able to write files to local storage.")
return nil
}
func (b *LocalFileBackend) Reader(path string) (ReadCloseSeeker, error) {
f, err := os.Open(filepath.Join(b.directory, path))
if err != nil {
return nil, errors.Wrapf(err, "unable to open file %s", path)
}
return f, nil
}
func (b *LocalFileBackend) ReadFile(path string) ([]byte, error) {
f, err := os.ReadFile(filepath.Join(b.directory, path))
if err != nil {
return nil, errors.Wrapf(err, "unable to read file %s", path)
}
return f, nil
}
func (b *LocalFileBackend) FileExists(path string) (bool, error) {
_, err := os.Stat(filepath.Join(b.directory, path))
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
}
return true, nil
}
func (b *LocalFileBackend) FileSize(path string) (int64, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
}
return info.Size(), nil
}
func (b *LocalFileBackend) FileModTime(path string) (time.Time, error) {
info, err := os.Stat(filepath.Join(b.directory, path))
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.ModTime(), nil
}
func (b *LocalFileBackend) CopyFile(oldPath, newPath string) error {
if err := copyFile(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
}
return nil
}
func (b *LocalFileBackend) MoveFile(oldPath, newPath string) error {
if err := os.MkdirAll(filepath.Dir(filepath.Join(b.directory, newPath)), 0750); err != nil {
return errors.Wrapf(err, "unable to create the new destination directory %s", filepath.Dir(newPath))
}
if err := os.Rename(filepath.Join(b.directory, oldPath), filepath.Join(b.directory, newPath)); err != nil {
return errors.Wrapf(err, "unable to move the file to %s to the destination directory", newPath)
}
return nil
}
func (b *LocalFileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
return writeFileLocally(fr, filepath.Join(b.directory, path))
}
func writeFileLocally(fr io.Reader, path string) (int64, error) {
if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil {
directory, _ := filepath.Abs(filepath.Dir(path))
return 0, errors.Wrapf(err, "unable to create the directory %s for the file %s", directory, path)
}
fw, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return 0, errors.Wrapf(err, "unable to open the file %s to write the data", path)
}
defer fw.Close()
written, err := io.Copy(fw, fr)
if err != nil {
return written, errors.Wrapf(err, "unable write the data in the file %s", path)
}
return written, nil
}
func (b *LocalFileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
fp := filepath.Join(b.directory, path)
if _, err := os.Stat(fp); err != nil {
return 0, errors.Wrapf(err, "unable to find the file %s to append the data", path)
}
fw, err := os.OpenFile(fp, os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return 0, errors.Wrapf(err, "unable to open the file %s to append the data", path)
}
defer fw.Close()
written, err := io.Copy(fw, fr)
if err != nil {
return written, errors.Wrapf(err, "unable append the data in the file %s", path)
}
return written, nil
}
func (b *LocalFileBackend) RemoveFile(path string) error {
if err := os.Remove(filepath.Join(b.directory, path)); err != nil {
return errors.Wrapf(err, "unable to remove the file %s", path)
}
return nil
}
// basePath: path to get to the file but won't be added to the end result
// path: basePath+path current directory we are looking at
// maxDepth: parameter to prevent infinite recursion, once this is reached we won't look any further
func appendRecursively(basePath, path string, maxDepth int) ([]string, error) {
results := []string{}
dirEntries, err := os.ReadDir(filepath.Join(basePath, path))
if err != nil {
if os.IsNotExist(err) {
return results, nil
}
return results, errors.Wrapf(err, "unable to list the directory %s", path)
}
for _, dirEntry := range dirEntries {
entryName := dirEntry.Name()
entryPath := filepath.Join(path, entryName)
if entryName == "." || entryName == ".." || entryPath == path {
continue
}
if dirEntry.IsDir() {
if maxDepth <= 0 {
mlog.Warn("Max depth reached, skipping any further directories", mlog.Int("depth", maxDepth), mlog.String("path", entryPath))
results = append(results, entryPath)
continue // we'll ignore it if max depth is reached.
}
nestedResults, err := appendRecursively(basePath, entryPath, maxDepth-1)
if err != nil {
return results, err
}
results = append(results, nestedResults...)
} else {
results = append(results, entryPath)
}
}
return results, nil
}
func (b *LocalFileBackend) ListDirectory(path string) ([]string, error) {
results := []string{}
dirEntries, err := os.ReadDir(filepath.Join(b.directory, path))
if err != nil {
if os.IsNotExist(err) {
// ideally os.ErrNotExist should've been returned but to keep the
// consistency, leaving it as is before.
return results, nil
}
// same here, ideally we shouldn't return the empty slice
return results, errors.Wrapf(err, "unable to list the directory %s", path)
}
for _, dirEntry := range dirEntries {
results = append(results, filepath.Join(path, dirEntry.Name()))
}
return results, nil
}
func (b *LocalFileBackend) ListDirectoryRecursively(path string) ([]string, error) {
return appendRecursively(b.directory, path, MaxRecursionDepth)
}
func (b *LocalFileBackend) RemoveDirectory(path string) error {
if err := os.RemoveAll(filepath.Join(b.directory, path)); err != nil {
return errors.Wrapf(err, "unable to remove the directory %s", path)
}
return nil
}
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
func (b *LocalFileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
deflateMethod := zip.Store
if deflate {
deflateMethod = zip.Deflate
}
fullPath := filepath.Join(b.directory, path)
baseInfo, err := os.Stat(fullPath)
if err != nil {
return nil, errors.Wrapf(err, "unable to stat path %s", path)
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
zipWriter := zip.NewWriter(pw)
defer zipWriter.Close()
err = filepath.Walk(fullPath, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Handle single file case
baseDir := fullPath
if !baseInfo.IsDir() {
baseDir = filepath.Dir(baseDir)
}
// Get the relative path from the base directory
relPath, err := filepath.Rel(baseDir, filePath)
if err != nil {
return errors.Wrapf(err, "unable to get relative path for %s", filePath)
}
// Skip the root directory itself
if relPath == "." {
return nil
}
// Create zip header
header, err := zip.FileInfoHeader(info)
if err != nil {
return errors.Wrapf(err, "unable to create zip header for %s", relPath)
}
// Ensure consistent forward slashes in paths
header.Name = filepath.ToSlash(relPath)
// Skip directories - we don't need to create entries for them
if info.IsDir() {
return nil
}
// Create file entry
header.Method = deflateMethod
header.SetMode(0644) // rw-r--r-- permissions
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return errors.Wrapf(err, "unable to create zip entry for %s", relPath)
}
file, err := os.Open(filePath)
if err != nil {
return errors.Wrapf(err, "unable to open file %s", filePath)
}
defer file.Close()
if _, err := io.Copy(writer, file); err != nil {
return errors.Wrapf(err, "unable to copy file content for %s", relPath)
}
return nil
})
if err != nil {
pw.CloseWithError(errors.Wrap(err, "error walking directory"))
}
}()
return pr, nil
}

View File

@@ -0,0 +1,430 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
io "io"
time "time"
filestore "github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
mock "github.com/stretchr/testify/mock"
)
// FileBackend is an autogenerated mock type for the FileBackend type
type FileBackend struct {
mock.Mock
}
// AppendFile provides a mock function with given fields: fr, path
func (_m *FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
ret := _m.Called(fr, path)
if len(ret) == 0 {
panic("no return value specified for AppendFile")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
return rf(fr, path)
}
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
r0 = rf(fr, path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
r1 = rf(fr, path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CopyFile provides a mock function with given fields: oldPath, newPath
func (_m *FileBackend) CopyFile(oldPath string, newPath string) error {
ret := _m.Called(oldPath, newPath)
if len(ret) == 0 {
panic("no return value specified for CopyFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(oldPath, newPath)
} else {
r0 = ret.Error(0)
}
return r0
}
// DriverName provides a mock function with no fields
func (_m *FileBackend) DriverName() string {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for DriverName")
}
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// FileExists provides a mock function with given fields: path
func (_m *FileBackend) FileExists(path string) (bool, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileExists")
}
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func(string) (bool, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) bool); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileModTime provides a mock function with given fields: path
func (_m *FileBackend) FileModTime(path string) (time.Time, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileModTime")
}
var r0 time.Time
var r1 error
if rf, ok := ret.Get(0).(func(string) (time.Time, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) time.Time); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(time.Time)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// FileSize provides a mock function with given fields: path
func (_m *FileBackend) FileSize(path string) (int64, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for FileSize")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (int64, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) int64); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListDirectory provides a mock function with given fields: path
func (_m *FileBackend) ListDirectory(path string) ([]string, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ListDirectory")
}
var r0 []string
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []string); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ListDirectoryRecursively provides a mock function with given fields: path
func (_m *FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ListDirectoryRecursively")
}
var r0 []string
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]string, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []string); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MoveFile provides a mock function with given fields: oldPath, newPath
func (_m *FileBackend) MoveFile(oldPath string, newPath string) error {
ret := _m.Called(oldPath, newPath)
if len(ret) == 0 {
panic("no return value specified for MoveFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, string) error); ok {
r0 = rf(oldPath, newPath)
} else {
r0 = ret.Error(0)
}
return r0
}
// ReadFile provides a mock function with given fields: path
func (_m *FileBackend) ReadFile(path string) ([]byte, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for ReadFile")
}
var r0 []byte
var r1 error
if rf, ok := ret.Get(0).(func(string) ([]byte, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) []byte); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Reader provides a mock function with given fields: path
func (_m *FileBackend) Reader(path string) (filestore.ReadCloseSeeker, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for Reader")
}
var r0 filestore.ReadCloseSeeker
var r1 error
if rf, ok := ret.Get(0).(func(string) (filestore.ReadCloseSeeker, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
r0 = rf(path)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(filestore.ReadCloseSeeker)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// RemoveDirectory provides a mock function with given fields: path
func (_m *FileBackend) RemoveDirectory(path string) error {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for RemoveDirectory")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(path)
} else {
r0 = ret.Error(0)
}
return r0
}
// RemoveFile provides a mock function with given fields: path
func (_m *FileBackend) RemoveFile(path string) error {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for RemoveFile")
}
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(path)
} else {
r0 = ret.Error(0)
}
return r0
}
// TestConnection provides a mock function with no fields
func (_m *FileBackend) TestConnection() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for TestConnection")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// WriteFile provides a mock function with given fields: fr, path
func (_m *FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
ret := _m.Called(fr, path)
if len(ret) == 0 {
panic("no return value specified for WriteFile")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(io.Reader, string) (int64, error)); ok {
return rf(fr, path)
}
if rf, ok := ret.Get(0).(func(io.Reader, string) int64); ok {
r0 = rf(fr, path)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(io.Reader, string) error); ok {
r1 = rf(fr, path)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ZipReader provides a mock function with given fields: path, deflate
func (_m *FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
ret := _m.Called(path, deflate)
if len(ret) == 0 {
panic("no return value specified for ZipReader")
}
var r0 io.ReadCloser
var r1 error
if rf, ok := ret.Get(0).(func(string, bool) (io.ReadCloser, error)); ok {
return rf(path, deflate)
}
if rf, ok := ret.Get(0).(func(string, bool) io.ReadCloser); ok {
r0 = rf(path, deflate)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(io.ReadCloser)
}
}
if rf, ok := ret.Get(1).(func(string, bool) error); ok {
r1 = rf(path, deflate)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewFileBackend creates a new instance of FileBackend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewFileBackend(t interface {
mock.TestingT
Cleanup(func())
}) *FileBackend {
mock := &FileBackend{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,65 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
time "time"
mock "github.com/stretchr/testify/mock"
)
// FileBackendWithLinkGenerator is an autogenerated mock type for the FileBackendWithLinkGenerator type
type FileBackendWithLinkGenerator struct {
mock.Mock
}
// GeneratePublicLink provides a mock function with given fields: path
func (_m *FileBackendWithLinkGenerator) GeneratePublicLink(path string) (string, time.Duration, error) {
ret := _m.Called(path)
if len(ret) == 0 {
panic("no return value specified for GeneratePublicLink")
}
var r0 string
var r1 time.Duration
var r2 error
if rf, ok := ret.Get(0).(func(string) (string, time.Duration, error)); ok {
return rf(path)
}
if rf, ok := ret.Get(0).(func(string) string); ok {
r0 = rf(path)
} else {
r0 = ret.Get(0).(string)
}
if rf, ok := ret.Get(1).(func(string) time.Duration); ok {
r1 = rf(path)
} else {
r1 = ret.Get(1).(time.Duration)
}
if rf, ok := ret.Get(2).(func(string) error); ok {
r2 = rf(path)
} else {
r2 = ret.Error(2)
}
return r0, r1, r2
}
// NewFileBackendWithLinkGenerator creates a new instance of FileBackendWithLinkGenerator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewFileBackendWithLinkGenerator(t interface {
mock.TestingT
Cleanup(func())
}) *FileBackendWithLinkGenerator {
mock := &FileBackendWithLinkGenerator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,100 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import mock "github.com/stretchr/testify/mock"
// ReadCloseSeeker is an autogenerated mock type for the ReadCloseSeeker type
type ReadCloseSeeker struct {
mock.Mock
}
// Close provides a mock function with no fields
func (_m *ReadCloseSeeker) Close() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for Close")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Read provides a mock function with given fields: p
func (_m *ReadCloseSeeker) Read(p []byte) (int, error) {
ret := _m.Called(p)
if len(ret) == 0 {
panic("no return value specified for Read")
}
var r0 int
var r1 error
if rf, ok := ret.Get(0).(func([]byte) (int, error)); ok {
return rf(p)
}
if rf, ok := ret.Get(0).(func([]byte) int); ok {
r0 = rf(p)
} else {
r0 = ret.Get(0).(int)
}
if rf, ok := ret.Get(1).(func([]byte) error); ok {
r1 = rf(p)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Seek provides a mock function with given fields: offset, whence
func (_m *ReadCloseSeeker) Seek(offset int64, whence int) (int64, error) {
ret := _m.Called(offset, whence)
if len(ret) == 0 {
panic("no return value specified for Seek")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(int64, int) (int64, error)); ok {
return rf(offset, whence)
}
if rf, ok := ret.Get(0).(func(int64, int) int64); ok {
r0 = rf(offset, whence)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(int64, int) error); ok {
r1 = rf(offset, whence)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// NewReadCloseSeeker creates a new instance of ReadCloseSeeker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewReadCloseSeeker(t interface {
mock.TestingT
Cleanup(func())
}) *ReadCloseSeeker {
mock := &ReadCloseSeeker{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,60 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
context "context"
blob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
mock "github.com/stretchr/testify/mock"
)
// blobDownloader is an autogenerated mock type for the blobDownloader type
type blobDownloader struct {
mock.Mock
}
// DownloadStream provides a mock function with given fields: ctx, opts
func (_m *blobDownloader) DownloadStream(ctx context.Context, opts *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error) {
ret := _m.Called(ctx, opts)
if len(ret) == 0 {
panic("no return value specified for DownloadStream")
}
var r0 blob.DownloadStreamResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) (blob.DownloadStreamResponse, error)); ok {
return rf(ctx, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, *blob.DownloadStreamOptions) blob.DownloadStreamResponse); ok {
r0 = rf(ctx, opts)
} else {
r0 = ret.Get(0).(blob.DownloadStreamResponse)
}
if rf, ok := ret.Get(1).(func(context.Context, *blob.DownloadStreamOptions) error); ok {
r1 = rf(ctx, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// newBlobDownloader creates a new instance of blobDownloader. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func newBlobDownloader(t interface {
mock.TestingT
Cleanup(func())
}) *blobDownloader {
mock := &blobDownloader{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,122 @@
// Code generated by mockery v2.53.4. DO NOT EDIT.
// Regenerate this file using `make filestore-mocks`.
package mocks
import (
context "context"
minio "github.com/minio/minio-go/v7"
mock "github.com/stretchr/testify/mock"
)
// s3CopyClient is an autogenerated mock type for the s3CopyClient type
type s3CopyClient struct {
mock.Mock
}
// ComposeObject provides a mock function with given fields: ctx, dst, srcs
func (_m *s3CopyClient) ComposeObject(ctx context.Context, dst minio.CopyDestOptions, srcs ...minio.CopySrcOptions) (minio.UploadInfo, error) {
_va := make([]interface{}, len(srcs))
for _i := range srcs {
_va[_i] = srcs[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, dst)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for ComposeObject")
}
var r0 minio.UploadInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
return rf(ctx, dst, srcs...)
}
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) minio.UploadInfo); ok {
r0 = rf(ctx, dst, srcs...)
} else {
r0 = ret.Get(0).(minio.UploadInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, ...minio.CopySrcOptions) error); ok {
r1 = rf(ctx, dst, srcs...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// CopyObject provides a mock function with given fields: ctx, dst, src
func (_m *s3CopyClient) CopyObject(ctx context.Context, dst minio.CopyDestOptions, src minio.CopySrcOptions) (minio.UploadInfo, error) {
ret := _m.Called(ctx, dst, src)
if len(ret) == 0 {
panic("no return value specified for CopyObject")
}
var r0 minio.UploadInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) (minio.UploadInfo, error)); ok {
return rf(ctx, dst, src)
}
if rf, ok := ret.Get(0).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) minio.UploadInfo); ok {
r0 = rf(ctx, dst, src)
} else {
r0 = ret.Get(0).(minio.UploadInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, minio.CopyDestOptions, minio.CopySrcOptions) error); ok {
r1 = rf(ctx, dst, src)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// StatObject provides a mock function with given fields: ctx, bucketName, objectName, opts
func (_m *s3CopyClient) StatObject(ctx context.Context, bucketName string, objectName string, opts minio.GetObjectOptions) (minio.ObjectInfo, error) {
ret := _m.Called(ctx, bucketName, objectName, opts)
if len(ret) == 0 {
panic("no return value specified for StatObject")
}
var r0 minio.ObjectInfo
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) (minio.ObjectInfo, error)); ok {
return rf(ctx, bucketName, objectName, opts)
}
if rf, ok := ret.Get(0).(func(context.Context, string, string, minio.GetObjectOptions) minio.ObjectInfo); ok {
r0 = rf(ctx, bucketName, objectName, opts)
} else {
r0 = ret.Get(0).(minio.ObjectInfo)
}
if rf, ok := ret.Get(1).(func(context.Context, string, string, minio.GetObjectOptions) error); ok {
r1 = rf(ctx, bucketName, objectName, opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// newS3CopyClient creates a new instance of s3CopyClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func newS3CopyClient(t interface {
mock.TestingT
Cleanup(func())
}) *s3CopyClient {
mock := &s3CopyClient{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

View File

@@ -0,0 +1,61 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"net/http"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// customTransport is used to point the request to a different server.
// This is helpful in situations where a different service is handling AWS S3 requests
// from multiple Mattermost applications, and the Mattermost service itself does not
// have any S3 credentials.
type customTransport struct {
host string
scheme string
client http.Client
}
// RoundTrip implements the http.Roundtripper interface.
func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Roundtrippers should not modify the original request.
newReq := req.Clone(context.Background())
*newReq.URL = *req.URL
req.URL.Scheme = t.scheme
req.URL.Host = t.host
return t.client.Do(req)
}
// customProvider is a dummy credentials provider for the minio client to work
// without actually providing credentials. This is needed with a custom transport
// in cases where the minio client does not actually have credentials with itself,
// rather needs responses from another entity.
//
// It satisfies the credentials.Provider interface.
type customProvider struct {
isSignV2 bool
}
// RetrieveWithCredContext simply calls Retrieve, because we don't need to use
// the param.
func (cp customProvider) RetrieveWithCredContext(_ *credentials.CredContext) (credentials.Value, error) {
return cp.Retrieve()
}
// Retrieve just returns empty credentials.
func (cp customProvider) Retrieve() (credentials.Value, error) {
sign := credentials.SignatureV4
if cp.isSignV2 {
sign = credentials.SignatureV2
}
return credentials.Value{
SignerType: sign,
}, nil
}
// IsExpired always returns false.
func (cp customProvider) IsExpired() bool { return false }

View File

@@ -0,0 +1,877 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"archive/zip"
"bytes"
"context"
"crypto/tls"
"fmt"
"io"
"io/fs"
"mime"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
// S3FileBackend contains all necessary information to communicate with
// an AWS S3 compatible API backend.
type S3FileBackend struct {
endpoint string
accessKey string
secretKey string
secure bool
signV2 bool
region string
bucket string
pathPrefix string
encrypt bool
trace bool
client *s3.Client
skipVerify bool
timeout time.Duration
presignExpires time.Duration
isCloud bool // field to indicate whether this is running under Mattermost cloud or not.
uploadPartSize int64
storageClass string
}
// S3FileBackendAuthError and S3FileBackendNoBucketError are aliases for the
// generic backend errors. They are kept so external code (plugins,
// historically-typed consumers) continues to compile.
type (
S3FileBackendAuthError = FileBackendAuthError
S3FileBackendNoBucketError = FileBackendNoBucketError
)
const (
// This is not exported by minio. See: https://github.com/minio/minio-go/issues/1339
bucketNotFound = "NoSuchBucket"
invalidBucket = "InvalidBucketName"
)
var (
// Ensure that the ReaderAt interface is implemented.
_ io.ReaderAt = (*s3WithCancel)(nil)
_ FileBackendWithLinkGenerator = (*S3FileBackend)(nil)
)
func getContentType(ext string) string {
mimeType := mime.TypeByExtension(strings.ToLower(ext))
if mimeType == "" {
mimeType = "application/octet-stream"
}
return mimeType
}
// NewS3FileBackend returns an instance of an S3FileBackend and determine if we are in Mattermost cloud or not.
func NewS3FileBackend(settings FileBackendSettings) (*S3FileBackend, error) {
return newS3FileBackend(settings, os.Getenv("MM_CLOUD_FILESTORE_BIFROST") != "")
}
// NewS3FileBackendWithoutBifrost returns an instance of an S3FileBackend that will not use bifrost.
func NewS3FileBackendWithoutBifrost(settings FileBackendSettings) (*S3FileBackend, error) {
return newS3FileBackend(settings, false)
}
func newS3FileBackend(settings FileBackendSettings, isCloud bool) (*S3FileBackend, error) {
timeout := time.Duration(settings.AmazonS3RequestTimeoutMilliseconds) * time.Millisecond
backend := &S3FileBackend{
endpoint: settings.AmazonS3Endpoint,
accessKey: settings.AmazonS3AccessKeyId,
secretKey: settings.AmazonS3SecretAccessKey,
secure: settings.AmazonS3SSL,
signV2: settings.AmazonS3SignV2,
region: settings.AmazonS3Region,
bucket: settings.AmazonS3Bucket,
pathPrefix: settings.AmazonS3PathPrefix,
encrypt: settings.AmazonS3SSE,
trace: settings.AmazonS3Trace,
skipVerify: settings.SkipVerify,
timeout: timeout,
presignExpires: time.Duration(settings.AmazonS3PresignExpiresSeconds) * time.Second,
uploadPartSize: settings.AmazonS3UploadPartSizeBytes,
storageClass: settings.AmazonS3StorageClass,
}
cli, err := backend.s3New(isCloud)
if err != nil {
return nil, err
}
backend.client = cli
backend.isCloud = isCloud
return backend, nil
}
type s3Trace struct {
}
func (*s3Trace) Write(in []byte) (int, error) {
mlog.Debug(string(in))
return len(in), nil
}
// Similar to s3.New() but allows initialization of signature v2 or signature v4 client.
// If signV2 input is false, function always returns signature v4.
//
// Additionally this function also takes a user defined region, if set
// disables automatic region lookup.
func (b *S3FileBackend) s3New(isCloud bool) (*s3.Client, error) {
var creds *credentials.Credentials
if isCloud {
creds = credentials.New(customProvider{isSignV2: b.signV2})
} else if b.accessKey == "" && b.secretKey == "" {
creds = credentials.NewIAM("")
} else if b.signV2 {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV2)
} else {
creds = credentials.NewStatic(b.accessKey, b.secretKey, "", credentials.SignatureV4)
}
opts := s3.Options{
Creds: creds,
Secure: b.secure,
Region: b.region,
}
tr, err := s3.DefaultTransport(b.secure)
if err != nil {
return nil, err
}
if b.skipVerify {
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
opts.Transport = tr
// If this is a cloud installation, we override the default transport.
if isCloud {
scheme := "http"
if b.secure {
scheme = "https"
}
newTransport := http.DefaultTransport.(*http.Transport).Clone()
newTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: b.skipVerify}
opts.Transport = &customTransport{
host: b.endpoint,
scheme: scheme,
client: http.Client{Transport: newTransport},
}
}
s3Clnt, err := s3.New(b.endpoint, &opts)
if err != nil {
return nil, err
}
if b.trace {
s3Clnt.TraceOn(&s3Trace{})
}
if tr, ok := opts.Transport.(*http.Transport); ok {
if tr.TLSClientConfig == nil {
tr.TLSClientConfig = &tls.Config{}
}
tr.TLSClientConfig.MinVersion = tls.VersionTLS12
}
return s3Clnt, nil
}
func (b *S3FileBackend) DriverName() string {
return driverS3
}
func (b *S3FileBackend) TestConnection() error {
exists := true
var err error
// If a path prefix is present, we attempt to test the bucket by listing objects under the path
// and just checking the first response. This is because the BucketExists call is only at a bucket level
// and sometimes the user might only be allowed access to the specified path prefix.
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if b.pathPrefix != "" {
obj := <-b.client.ListObjects(ctx, b.bucket, s3.ListObjectsOptions{Prefix: b.pathPrefix})
if obj.Err != nil {
typedErr := s3.ToErrorResponse(obj.Err)
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to list objects in the S3 bucket: %v", typedErr)}
}
exists = false
}
} else {
exists, err = b.client.BucketExists(ctx, b.bucket)
if err != nil {
typedErr := s3.ToErrorResponse(err)
if typedErr.Code != bucketNotFound && typedErr.Code != invalidBucket {
return &S3FileBackendAuthError{DetailedError: fmt.Sprintf("unable to check if the S3 bucket exists: %v", typedErr)}
}
}
}
if !exists {
return &S3FileBackendNoBucketError{}
}
mlog.Debug("Connection to S3 or minio is good. Bucket exists.")
return nil
}
func (b *S3FileBackend) MakeBucket() error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
err := b.client.MakeBucket(ctx, b.bucket, s3.MakeBucketOptions{Region: b.region})
if err != nil {
return errors.Wrap(err, "unable to create the s3 bucket")
}
return nil
}
// s3WithCancel is a wrapper struct which cancels the context
// when the object is closed.
type s3WithCancel struct {
*s3.Object
timer *time.Timer
cancel context.CancelFunc
}
func (sc *s3WithCancel) Close() error {
sc.timer.Stop()
sc.cancel()
return sc.Object.Close()
}
// CancelTimeout attempts to cancel the timeout for this reader. It allows calling
// code to ignore the timeout in case of longer running operations. The methods returns
// false if the timeout has already fired.
func (sc *s3WithCancel) CancelTimeout() bool {
return sc.timer.Stop()
}
// Caller must close the first return value
func (b *S3FileBackend) Reader(path string) (ReadCloseSeeker, error) {
path, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithCancel(context.Background())
minioObject, err := b.client.GetObject(ctx, b.bucket, path, s3.GetObjectOptions{})
if err != nil {
cancel()
return nil, errors.Wrapf(err, "unable to open file %s", path)
}
sc := &s3WithCancel{
Object: minioObject,
timer: time.AfterFunc(b.timeout, cancel),
cancel: cancel,
}
return sc, nil
}
func (b *S3FileBackend) ReadFile(path string) ([]byte, error) {
encodedPath, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
minioObject, err := b.client.GetObject(ctx, b.bucket, encodedPath, s3.GetObjectOptions{})
if err != nil {
return nil, errors.Wrapf(err, "unable to open file %s", encodedPath)
}
defer minioObject.Close()
f, err := io.ReadAll(minioObject)
if err != nil {
return nil, errors.Wrapf(err, "unable to read file %s", encodedPath)
}
return f, nil
}
func (b *S3FileBackend) FileExists(path string) (bool, error) {
path, err := b.prefixedPath(path)
if err != nil {
return false, errors.Wrapf(err, "unable to prefix path %s", path)
}
return b._fileExists(path)
}
func (b *S3FileBackend) _fileExists(path string) (bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
_, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err == nil {
return true, nil
}
var s3Err s3.ErrorResponse
if errors.As(err, &s3Err); s3Err.Code == "NoSuchKey" {
return false, nil
}
return false, errors.Wrapf(err, "unable to know if file %s exists", path)
}
func (b *S3FileBackend) FileSize(path string) (int64, error) {
path, err := b.prefixedPath(path)
if err != nil {
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return 0, errors.Wrapf(err, "unable to get file size for %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) FileModTime(path string) (time.Time, error) {
path, err := b.prefixedPath(path)
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
info, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err != nil {
return time.Time{}, errors.Wrapf(err, "unable to get modification time for file %s", path)
}
return info.LastModified, nil
}
// maxS3SingleCopySize is the largest source object (5GiB) that S3 allows for a
// single server-side copy (CopyObject). Sources larger than this must be copied
// with a server-side multipart copy (UploadPartCopy).
const maxS3SingleCopySize = 5 * 1024 * 1024 * 1024
// s3CopyClient is the subset of the S3 client used to route a server-side copy.
// It exists so copyObjectWithClient can be unit tested with a mock client.
type s3CopyClient interface {
StatObject(ctx context.Context, bucketName, objectName string, opts s3.StatObjectOptions) (s3.ObjectInfo, error)
CopyObject(ctx context.Context, dst s3.CopyDestOptions, src s3.CopySrcOptions) (s3.UploadInfo, error)
ComposeObject(ctx context.Context, dst s3.CopyDestOptions, srcs ...s3.CopySrcOptions) (s3.UploadInfo, error)
}
// copyObject performs a server-side copy of a single object from srcOpts to
// dstOpts, using the backend's request timeout.
func (b *S3FileBackend) copyObject(srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return copyObjectWithClient(ctx, b.client, srcOpts, dstOpts)
}
// copyObjectWithClient uses CopyObject for sources up to 5GiB, and falls back to
// ComposeObject — which performs a server-side multipart copy — for larger
// sources, which S3's single-operation CopyObject rejects. The size is checked
// explicitly rather than relying on ComposeObject's own single-copy fast path,
// which is only taken when the source range Start is -1 (a value its input
// validation rejects, so it is unreachable here).
func copyObjectWithClient(ctx context.Context, client s3CopyClient, srcOpts s3.CopySrcOptions, dstOpts s3.CopyDestOptions) error {
stat, err := client.StatObject(ctx, srcOpts.Bucket, srcOpts.Object, s3.StatObjectOptions{})
if err != nil {
return errors.Wrapf(err, "unable to stat source file %s", srcOpts.Object)
}
if stat.Size > maxS3SingleCopySize {
_, err = client.ComposeObject(ctx, dstOpts, srcOpts)
} else {
_, err = client.CopyObject(ctx, dstOpts, srcOpts)
}
return err
}
func (b *S3FileBackend) CopyFile(oldPath, newPath string) error {
oldPath, err := b.prefixedPath(oldPath)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
}
newPath = filepath.Join(b.pathPrefix, newPath)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy file from %s to %s", oldPath, newPath)
}
return nil
}
// DecodeFilePathIfNeeded is a special method to URL decode all older
// file paths. It is only needed for the migration, and will be removed
// as soon as the migration is complete.
func (b *S3FileBackend) DecodeFilePathIfNeeded(path string) error {
// Encode and check if file path changes.
// If there is no change, then there is no need to do anything.
if path == s3utils.EncodePath(path) {
return nil
}
// Check if encoded path exists.
exists, err := b.lookupOriginalPath(s3utils.EncodePath(path))
if err != nil {
return err
}
if !exists {
return nil
}
// If yes, then it needs to be migrated.
// This is basically a copy of MoveFile without the path encoding.
// We avoid any further refactoring because this method will be removed anyways.
oldPath := filepath.Join(b.pathPrefix, s3utils.EncodePath(path))
newPath := filepath.Join(b.pathPrefix, path)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
}
return nil
}
func (b *S3FileBackend) MoveFile(oldPath, newPath string) error {
oldPath, err := b.prefixedPath(oldPath)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", oldPath)
}
newPath = filepath.Join(b.pathPrefix, newPath)
srcOpts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: oldPath,
}
if b.encrypt {
srcOpts.Encryption = encrypt.NewSSE()
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: newPath,
}
if b.encrypt {
dstOpts.Encryption = encrypt.NewSSE()
}
if err := b.copyObject(srcOpts, dstOpts); err != nil {
return errors.Wrapf(err, "unable to copy the file to %s to the new destination", newPath)
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
if err := b.client.RemoveObject(ctx2, b.bucket, oldPath, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file old file %s", oldPath)
}
return nil
}
func (b *S3FileBackend) WriteFile(fr io.Reader, path string) (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
return b.WriteFileContext(ctx, fr, path)
}
func (b *S3FileBackend) WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, error) {
var contentType string
path = filepath.Join(b.pathPrefix, path)
ext := filepath.Ext(path)
contentType = getContentType(ext)
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
objSize := int64(-1)
if b.isCloud {
options.DisableContentSha256 = true
} else {
// We pass an object size only in situations where bifrost is not
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
switch t := fr.(type) {
case *bytes.Buffer:
objSize = int64(t.Len())
case *os.File:
if s, err2 := t.Stat(); err2 == nil {
objSize = s.Size()
}
}
}
info, err := b.client.PutObject(ctx, b.bucket, path, fr, objSize, options)
if err != nil {
return info.Size, errors.Wrapf(err, "unable write the data in the file %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) AppendFile(fr io.Reader, path string) (int64, error) {
fp, err := b.prefixedPath(path)
if err != nil {
return 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if _, err2 := b.client.StatObject(ctx, b.bucket, fp, s3.StatObjectOptions{}); err2 != nil {
return 0, errors.Wrapf(err2, "unable to find the file %s to append the data", path)
}
contentType := getContentType(filepath.Ext(fp))
options := s3PutOptions(b.encrypt, contentType, b.uploadPartSize, b.storageClass)
sse := options.ServerSideEncryption
partName := fp + ".part"
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
objSize := -1
if b.isCloud {
options.DisableContentSha256 = true
}
// We pass an object size only in situations where bifrost is not
// used. Bifrost needs to run in HTTPS, which is not yet deployed.
if buf, ok := fr.(*bytes.Buffer); ok && !b.isCloud {
objSize = buf.Len()
}
info, err := b.client.PutObject(ctx2, b.bucket, partName, fr, int64(objSize), options)
if err != nil {
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
}
defer func() {
ctx4, cancel4 := context.WithTimeout(context.Background(), b.timeout)
defer cancel4()
b.client.RemoveObject(ctx4, b.bucket, partName, s3.RemoveObjectOptions{})
}()
src1Opts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: fp,
}
src2Opts := s3.CopySrcOptions{
Bucket: b.bucket,
Object: partName,
}
dstOpts := s3.CopyDestOptions{
Bucket: b.bucket,
Object: fp,
Encryption: sse,
}
ctx3, cancel3 := context.WithTimeout(context.Background(), b.timeout)
defer cancel3()
_, err = b.client.ComposeObject(ctx3, dstOpts, src1Opts, src2Opts)
if err != nil {
return 0, errors.Wrapf(err, "unable append the data in the file %s", path)
}
return info.Size, nil
}
func (b *S3FileBackend) RemoveFile(path string) error {
path, err := b.prefixedPath(path)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
if err := b.client.RemoveObject(ctx, b.bucket, path, s3.RemoveObjectOptions{}); err != nil {
return errors.Wrapf(err, "unable to remove the file %s", path)
}
return nil
}
func (b *S3FileBackend) listDirectory(path string, recursion bool) ([]string, error) {
path, err := b.prefixedPath(path)
if err != nil {
return nil, errors.Wrapf(err, "unable to prefix path %s", path)
}
if !strings.HasSuffix(path, "/") && path != "" {
// s3Clnt returns only the path itself when "/" is not present
// appending "/" to make it consistent across all filestores
path = path + "/"
}
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: recursion,
}
var paths []string
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
var count int
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
if object.Err != nil {
return nil, errors.Wrapf(object.Err, "unable to list the directory %s", path)
}
// We strip the path prefix that gets applied,
// so that it remains transparent to the application.
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
trimmed := strings.Trim(object.Key, "/")
if trimmed != "" {
paths = append(paths, trimmed)
}
count++
}
// Check if only one item was returned and it matches the path prefix
if count == 1 && len(paths) > 0 && strings.TrimRight(path, "/") == paths[0] {
// Return a fs.PathError to maintain consistency
return nil, &fs.PathError{Op: "readdir", Path: path, Err: fs.ErrNotExist}
}
return paths, nil
}
func (b *S3FileBackend) ListDirectory(path string) ([]string, error) {
return b.listDirectory(path, false)
}
func (b *S3FileBackend) ListDirectoryRecursively(path string) ([]string, error) {
return b.listDirectory(path, true)
}
func (b *S3FileBackend) RemoveDirectory(path string) error {
path, err := b.prefixedPath(path)
if err != nil {
return errors.Wrapf(err, "unable to prefix path %s", path)
}
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: true,
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
// List all objects in the directory
for object := range b.client.ListObjects(ctx, b.bucket, opts) {
if object.Err != nil {
return errors.Wrapf(object.Err, "unable to list the directory %s", path)
}
// Remove each object individually to avoid MD5 usage
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
err := b.client.RemoveObject(ctx2, b.bucket, object.Key, s3.RemoveObjectOptions{})
if err != nil {
return errors.Wrapf(err, "unable to remove object %s from directory %s", object.Key, path)
}
}
return nil
}
// ZipReader will create a zip of path. If path is a single file, it will zip the single file.
// If deflate is true, the contents will be compressed. It will stream the zip to io.ReadCloser.
func (b *S3FileBackend) ZipReader(path string, deflate bool) (io.ReadCloser, error) {
deflateMethod := zip.Store
if deflate {
deflateMethod = zip.Deflate
}
path, err := b.prefixedPath(path)
if err != nil {
return nil, err
}
pr, pw := io.Pipe()
go func() {
defer pw.Close()
zipWriter := zip.NewWriter(pw)
defer zipWriter.Close()
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
// Is path a single file?
object, err := b.client.StatObject(ctx, b.bucket, path, s3.StatObjectOptions{})
if err == nil {
// We want the zipped file to be at the root of the zip. E.g., given a path of
// "path/to/file.sh" we want the zip to have one file: "file.sh", not "path/to/file.sh".
stripPath := filepath.Dir(path)
if stripPath != "" {
stripPath += "/"
}
if err = b._copyObjectToZipWriter(zipWriter, object, stripPath, deflateMethod); err != nil {
pw.CloseWithError(err)
}
return
}
// Is path a directory?
path = path + "/"
opts := s3.ListObjectsOptions{
Prefix: path,
Recursive: true,
}
ctx2, cancel2 := context.WithTimeout(context.Background(), b.timeout)
defer cancel2()
for object := range b.client.ListObjects(ctx2, b.bucket, opts) {
if object.Err != nil {
pw.CloseWithError(errors.Wrapf(object.Err, "unable to list the directory %s", path))
return
}
if err = b._copyObjectToZipWriter(zipWriter, object, path, deflateMethod); err != nil {
pw.CloseWithError(err)
return
}
}
}()
return pr, nil
}
func (b *S3FileBackend) _copyObjectToZipWriter(zipWriter *zip.Writer, object s3.ObjectInfo, stripPath string, deflateMethod uint16) error {
// We strip the path prefix that gets applied,
// so that it remains transparent to the application.
object.Key = strings.TrimPrefix(object.Key, b.pathPrefix)
// We strip the path prefix + path so the zip file is relative to the root of the requested path
relPath := strings.TrimPrefix(object.Key, stripPath)
header := &zip.FileHeader{
Name: relPath,
Method: deflateMethod,
Modified: object.LastModified,
}
header.SetMode(0644) // rw-r--r-- permissions
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return errors.Wrapf(err, "unable to create zip entry for %s", object.Key)
}
reader, err := b.Reader(object.Key)
if err != nil {
return errors.Wrapf(err, "unable to create reader for %s", object.Key)
}
defer reader.Close()
_, err = io.Copy(writer, reader)
if err != nil {
return errors.Wrapf(err, "unable to copy content for %s", object.Key)
}
return nil
}
func (b *S3FileBackend) GeneratePublicLink(path string) (string, time.Duration, error) {
path, err := b.prefixedPath(path)
if err != nil {
return "", 0, errors.Wrapf(err, "unable to prefix path %s", path)
}
ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
defer cancel()
reqParams := make(url.Values)
reqParams.Set("response-content-disposition", "attachment")
req, err := b.client.PresignedGetObject(ctx, b.bucket, path, b.presignExpires, reqParams)
if err != nil {
return "", 0, errors.Wrapf(err, "unable to generate public link for %s", path)
}
return req.String(), b.presignExpires, nil
}
func (b *S3FileBackend) lookupOriginalPath(s string) (bool, error) {
exists, err := b._fileExists(filepath.Join(b.pathPrefix, s))
if err != nil {
var s3Err s3.ErrorResponse
// Sometimes S3 will not allow to access other paths.
// In that case, we consider them as not exists.
if errors.As(err, &s3Err); s3Err.Code == "AccessDenied" {
return false, nil
}
return false, errors.Wrapf(err, "unable to check for file path %s", s)
}
return exists, nil
}
func (b *S3FileBackend) prefixedPath(s string) (string, error) {
if b.isCloud {
// We do a lookup of the original path for compatibility purposes.
exists, err := b.lookupOriginalPath(s)
if err != nil {
return "", err
}
// If it exists, then we don't want to encode it
// because it's an old path.
if !exists {
// If the request is routed via bifrost, then we need to encode the path
// to avoid signature validation errors.
// This happens because in bifrost, we are signing the URL outside the SDK
// and therefore the signature sent from the bifrost client
// will contain the encoded path, whereas the original path is sent
// un-encoded.
// More info at: https://github.com/aws/aws-sdk-go-v2/blob/1e4148ac334a4ea7abe31bd984a31dc761bb289d/aws/signer/v4/v4.go#L20
s = s3utils.EncodePath(s)
}
}
return filepath.Join(b.pathPrefix, s), nil
}
func s3PutOptions(encrypted bool, contentType string, uploadPartSize int64, storageClass string) s3.PutObjectOptions {
options := s3.PutObjectOptions{}
if encrypted {
options.ServerSideEncryption = encrypt.NewSSE()
}
options.ContentType = contentType
options.PartSize = uint64(uploadPartSize)
options.StorageClass = storageClass
return options
}

View File

@@ -0,0 +1,79 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"context"
"errors"
"testing"
s3 "github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ensure the real S3 client satisfies the interface used by copyObjectWithClient.
var _ s3CopyClient = (*s3.Client)(nil)
// mockS3CopyClient is a minimal s3CopyClient that records which copy operation
// was invoked and lets StatObject's result be controlled per test.
type mockS3CopyClient struct {
statInfo s3.ObjectInfo
statErr error
copyCalled bool
composeCalled bool
}
func (m *mockS3CopyClient) StatObject(_ context.Context, _, _ string, _ s3.StatObjectOptions) (s3.ObjectInfo, error) {
return m.statInfo, m.statErr
}
func (m *mockS3CopyClient) CopyObject(_ context.Context, _ s3.CopyDestOptions, _ s3.CopySrcOptions) (s3.UploadInfo, error) {
m.copyCalled = true
return s3.UploadInfo{}, nil
}
func (m *mockS3CopyClient) ComposeObject(_ context.Context, _ s3.CopyDestOptions, _ ...s3.CopySrcOptions) (s3.UploadInfo, error) {
m.composeCalled = true
return s3.UploadInfo{}, nil
}
var (
testCopySrc = s3.CopySrcOptions{Bucket: "bucket", Object: "src"}
testCopyDst = s3.CopyDestOptions{Bucket: "bucket", Object: "dst"}
)
func Test_copyObject_UsesCompose_ForLarge(t *testing.T) {
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize + 1}}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.NoError(t, err)
assert.True(t, mock.composeCalled, "expected ComposeObject (multipart copy) for sources larger than 5GiB")
assert.False(t, mock.copyCalled, "expected CopyObject not to be used for sources larger than 5GiB")
}
func Test_copyObject_UsesCopy_ForSmall(t *testing.T) {
// A source exactly at the 5GiB limit must still use the single CopyObject.
mock := &mockS3CopyClient{statInfo: s3.ObjectInfo{Size: maxS3SingleCopySize}}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.NoError(t, err)
assert.True(t, mock.copyCalled, "expected CopyObject for sources up to 5GiB")
assert.False(t, mock.composeCalled, "expected ComposeObject not to be used for sources up to 5GiB")
}
func Test_copyObject_PropagatesStatError(t *testing.T) {
statErr := errors.New("stat failed")
mock := &mockS3CopyClient{statErr: statErr}
err := copyObjectWithClient(context.Background(), mock, testCopySrc, testCopyDst)
require.Error(t, err)
assert.ErrorIs(t, err, statErr, "StatObject error must be propagated")
assert.False(t, mock.copyCalled, "no copy should be attempted when StatObject fails")
assert.False(t, mock.composeCalled, "no compose should be attempted when StatObject fails")
}

View File

@@ -0,0 +1,456 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package filestore
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"errors"
"fmt"
"io"
"io/fs"
"net/http/httptest"
"net/http/httputil"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/mattermost/mattermost/server/public/model"
s3 "github.com/minio/minio-go/v7"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckMandatoryS3Fields(t *testing.T) {
cfg := FileBackendSettings{}
err := cfg.CheckMandatoryS3Fields()
require.Error(t, err)
require.Equal(t, err.Error(), "missing s3 bucket settings", "should've failed with missing s3 bucket")
cfg.AmazonS3Bucket = "test-mm"
err = cfg.CheckMandatoryS3Fields()
require.NoError(t, err)
cfg.AmazonS3Endpoint = ""
err = cfg.CheckMandatoryS3Fields()
require.NoError(t, err)
require.Equal(t, "s3.amazonaws.com", cfg.AmazonS3Endpoint, "should've set the endpoint to the default")
}
func TestMakeBucket(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
SkipVerify: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
}
func TestTimeout(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
SkipVerify: false,
AmazonS3RequestTimeoutMilliseconds: 0,
}
t.Run("MakeBucket", func(t *testing.T) {
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.True(t, errors.Is(err, context.DeadlineExceeded))
})
t.Run("WriteFile", func(t *testing.T) {
cfg.AmazonS3RequestTimeoutMilliseconds = 1000
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
r, w := io.Pipe()
go func() {
defer w.Close()
for range 10 {
_, writeErr := w.Write([]byte("data"))
require.NoError(t, writeErr)
time.Sleep(time.Millisecond * 200)
}
}()
_, err = fileBackend.WriteFile(r, "tests/"+randomString()+".png")
require.True(t, errors.Is(err, context.DeadlineExceeded))
})
}
func TestInsecureMakeBucket(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
proxySelfSignedHTTPS := newTLSProxyServer(&url.URL{Scheme: "http", Host: s3Endpoint})
defer proxySelfSignedHTTPS.Close()
enableInsecure, secure := true, false
testCases := []struct {
description string
skipVerify bool
expectedAllowed bool
}{
{"allow self-signed HTTPS when insecure enabled", enableInsecure, true},
{"reject self-signed HTTPS when secured", secure, false},
}
for _, testCase := range testCases {
t.Run(testCase.description, func(t *testing.T) {
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: proxySelfSignedHTTPS.URL[8:],
AmazonS3Region: "",
AmazonS3PathPrefix: "",
AmazonS3SSL: true,
SkipVerify: testCase.skipVerify,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
if testCase.expectedAllowed {
require.NoError(t, err)
} else {
require.Error(t, err)
}
})
}
}
func newTLSProxyServer(backend *url.URL) *httptest.Server {
return httptest.NewTLSServer(httputil.NewSingleHostReverseProxy(backend))
}
func TestS3WithCancel(t *testing.T) {
// Some of these tests use time.Sleep to wait for the timeout to expire.
// They are run in parallel to reduce wait times.
t.Run("zero timeout", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(0, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
t.Run("timeout", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(100 * time.Millisecond) // give the context time to cancel
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
t.Run("timeout cancel", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
require.False(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("timeout closed", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, ctx.Err())
require.NoError(t, r.Close())
time.Sleep(100 * time.Millisecond) // wait for the original (canceled) timeout to expire
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("close cancel close", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, nil)
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.True(t, r.CancelTimeout())
require.NoError(t, r.Close())
require.Error(t, ctx.Err())
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
require.NoError(t, r.Close())
})
t.Run("close error", func(t *testing.T) {
t.Parallel()
r, ctx := newMockS3WithCancel(50*time.Millisecond, errors.New("test error"))
time.Sleep(10 * time.Millisecond) // give the context time to cancel
require.NoError(t, ctx.Err())
require.Error(t, r.Close())
require.False(t, r.CancelTimeout())
require.Error(t, ctx.Err())
})
}
func newMockS3WithCancel(timeout time.Duration, closeErr error) (*fauxCloser, context.Context) {
ctx, cancel := context.WithCancel(context.Background())
return &fauxCloser{
s3WithCancel: &s3WithCancel{
Object: &s3.Object{},
timer: time.AfterFunc(timeout, cancel),
cancel: cancel,
},
closeErr: closeErr,
}, ctx
}
type fauxCloser struct {
*s3WithCancel
closeErr error
}
func (fc fauxCloser) Close() error {
fc.s3WithCancel.timer.Stop()
fc.s3WithCancel.cancel()
return fc.closeErr
}
func TestListDirectory(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
cfg := FileBackendSettings{
DriverName: driverS3,
AmazonS3AccessKeyId: "minioaccesskey",
AmazonS3SecretAccessKey: "miniosecretkey",
AmazonS3Bucket: "mattermost-test-1",
AmazonS3Region: "",
AmazonS3Endpoint: s3Endpoint,
AmazonS3PathPrefix: "",
AmazonS3SSL: false,
AmazonS3SSE: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
found, err := fileBackend.client.BucketExists(context.Background(), cfg.AmazonS3Bucket)
require.NoError(t, err)
if !found {
err = fileBackend.MakeBucket()
require.NoError(t, err)
}
fileBackend.pathPrefix = "19700101/"
require.NoError(t, err)
b := []byte("test")
path1 := "19700101/" + randomString() + ".txt"
_, err = fileBackend.WriteFile(bytes.NewReader(b), path1)
require.NoError(t, err, "Failed to write file1 to S3")
_, err = fileBackend.ListDirectory("")
var pErr *fs.PathError
assert.True(t, errors.As(err, &pErr), "error is not of type fs.PathError")
err = fileBackend.RemoveFile(path1)
require.NoError(t, err, "Failed to remove file1 from S3")
err = fileBackend.RemoveDirectory("19700101")
require.NoError(t, err)
}
func TestWriteFileVideoMimeTypes(t *testing.T) {
s3Host := os.Getenv("CI_MINIO_HOST")
if s3Host == "" {
s3Host = "localhost"
}
s3Port := os.Getenv("CI_MINIO_PORT")
if s3Port == "" {
s3Port = "9000"
}
s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
// Generate a random bucket name
b := make([]byte, 30)
rand.Read(b)
bucketName := base64.StdEncoding.EncodeToString(b)
bucketName = strings.ToLower(bucketName)
bucketName = strings.Replace(bucketName, "+", "", -1)
bucketName = strings.Replace(bucketName, "/", "", -1)
cfg := FileBackendSettings{
DriverName: model.ImageDriverS3,
AmazonS3AccessKeyId: model.MinioAccessKey,
AmazonS3SecretAccessKey: model.MinioSecretKey,
AmazonS3Bucket: bucketName,
AmazonS3Endpoint: s3Endpoint,
AmazonS3SSL: false,
AmazonS3RequestTimeoutMilliseconds: 5000,
}
fileBackend, err := NewS3FileBackend(cfg)
require.NoError(t, err)
err = fileBackend.MakeBucket()
require.NoError(t, err)
// Test video types
// Test video types with multiple valid MIME types
testContent := []byte("test-video-content")
videoTypes := map[string][]string{
".avi": {"video/vnd.avi", "video/x-msvideo"},
".mpeg": {"video/mpeg"},
".mp4": {"video/mp4"},
}
for ext, validMimeTypes := range videoTypes {
t.Run(strings.TrimPrefix(ext, "."), func(t *testing.T) {
path := "test" + ext
reader := bytes.NewReader(testContent)
written, err := fileBackend.WriteFile(reader, path)
require.NoError(t, err)
require.Equal(t, int64(len(testContent)), written)
// Verify the file exists with correct mime type
props, err := fileBackend.client.StatObject(
context.Background(),
bucketName,
path,
s3.StatObjectOptions{},
)
require.NoError(t, err)
// Ensure the MIME type is one of the expected values
assert.Contains(t, validMimeTypes, props.ContentType, "Unexpected MIME type: %s", props.ContentType)
defer func() {
err = fileBackend.RemoveFile(path)
require.NoError(t, err)
}()
})
}
}

View File

@@ -0,0 +1,183 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package mail
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"strings"
"time"
)
const (
InbucketAPI = "/api/v1/mailbox/"
)
// OutputJSONHeader holds the received Header to test sending emails (inbucket)
type JSONMessageHeaderInbucket []struct {
Mailbox string
ID string `json:"Id"`
From, Subject, Date string
To []string
Size int
}
// OutputJSONMessage holds the received Message fto test sending emails (inbucket)
type JSONMessageInbucket struct {
Mailbox string
ID string `json:"Id"`
From, Subject, Date string
Size int
Header map[string][]string
Body struct {
Text string
HTML string `json:"Html"`
}
Attachments []struct {
Filename string
ContentType string `json:"content-type"`
DownloadLink string `json:"download-link"`
Bytes []byte `json:"-"`
}
}
func ParseEmail(email string) string {
pos := strings.Index(email, "@")
parsedEmail := email[0:pos]
return parsedEmail
}
func GetMailBox(email string) (results JSONMessageHeaderInbucket, err error) {
parsedEmail := ParseEmail(email)
url := fmt.Sprintf("%s%s%s", getInbucketHost(), InbucketAPI, parsedEmail)
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer func() {
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
if resp.Body == nil {
return nil, fmt.Errorf("no mailbox")
}
var record JSONMessageHeaderInbucket
err = json.NewDecoder(resp.Body).Decode(&record)
if err != nil {
return nil, fmt.Errorf("error: %w", err)
}
if len(record) == 0 {
return nil, fmt.Errorf("no mailbox")
}
return record, nil
}
func GetMessageFromMailbox(email, id string) (JSONMessageInbucket, error) {
parsedEmail := ParseEmail(email)
var record JSONMessageInbucket
url := fmt.Sprintf("%s%s%s/%s", getInbucketHost(), InbucketAPI, parsedEmail, id)
emailResponse, err := http.Get(url)
if err != nil {
return record, err
}
defer func() {
io.Copy(io.Discard, emailResponse.Body)
emailResponse.Body.Close()
}()
if err = json.NewDecoder(emailResponse.Body).Decode(&record); err != nil {
return record, err
}
// download attachments
if len(record.Attachments) > 0 {
for i := range record.Attachments {
var bytes []byte
bytes, err = downloadAttachment(record.Attachments[i].DownloadLink)
if err != nil {
return record, err
}
record.Attachments[i].Bytes = make([]byte, len(bytes))
copy(record.Attachments[i].Bytes, bytes)
}
}
return record, err
}
func downloadAttachment(url string) ([]byte, error) {
attachmentResponse, err := http.Get(url)
if err != nil {
return nil, err
}
defer attachmentResponse.Body.Close()
buf := new(bytes.Buffer)
io.Copy(buf, attachmentResponse.Body)
return buf.Bytes(), nil
}
func DeleteMailBox(email string) (err error) {
parsedEmail := ParseEmail(email)
url := fmt.Sprintf("%s%s%s", getInbucketHost(), InbucketAPI, parsedEmail)
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func RetryInbucket(attempts int, callback func() error) (err error) {
for i := 0; ; i++ {
err = callback()
if err == nil {
return nil
}
if i >= (attempts - 1) {
break
}
time.Sleep(5 * time.Second)
fmt.Println("retrying...")
}
return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}
func getInbucketHost() (host string) {
inbucket_host := os.Getenv("CI_INBUCKET_HOST")
if inbucket_host == "" {
inbucket_host = "localhost"
}
inbucket_port := os.Getenv("CI_INBUCKET_PORT")
if inbucket_port == "" {
inbucket_port = "9001"
}
return "http://" + net.JoinHostPort(inbucket_host, inbucket_port)
}

View File

@@ -0,0 +1,402 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package mail
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/mail"
"net/smtp"
"slices"
"time"
"github.com/jaytaylor/html2text"
"github.com/pkg/errors"
gomail "github.com/wneessen/go-mail"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
)
const (
TLS = "TLS"
StartTLS = "STARTTLS"
)
type SMTPConfig struct {
ConnectionSecurity string
SkipServerCertificateVerification bool
Hostname string
ServerName string
Server string
Port string
ServerTimeout int
Username string
Password string
EnableSMTPAuth bool
SendEmailNotifications bool
FeedbackName string
FeedbackEmail string
ReplyToAddress string
}
type mailData struct {
mimeTo string
smtpTo string
from mail.Address
cc string
replyTo mail.Address
subject string
htmlBody string
embeddedFiles map[string]io.Reader
mimeHeaders map[string]string
messageID string
inReplyTo string
references string
category string
}
// smtpClient is implemented by an smtp.Client. See https://golang.org/pkg/net/smtp/#Client.
type smtpClient interface {
Mail(string) error
Rcpt(string) error
Data() (io.WriteCloser, error)
}
type authChooser struct {
smtp.Auth
config *SMTPConfig
}
func (a *authChooser) Start(server *smtp.ServerInfo) (string, []byte, error) {
smtpAddress := a.config.ServerName + ":" + a.config.Port
a.Auth = LoginAuth(a.config.Username, a.config.Password, smtpAddress)
if slices.Contains(server.Auth, "PLAIN") {
a.Auth = smtp.PlainAuth("", a.config.Username, a.config.Password, a.config.ServerName+":"+a.config.Port)
}
return a.Auth.Start(server)
}
type loginAuth struct {
username, password, host string
}
func LoginAuth(username, password, host string) smtp.Auth {
return &loginAuth{username, password, host}
}
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
if !server.TLS {
return "", nil, errors.New("unencrypted connection")
}
if server.Name != a.host {
return "", nil, errors.New("wrong host name")
}
return "LOGIN", []byte{}, nil
}
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
if more {
switch string(fromServer) {
case "Username:":
return []byte(a.username), nil
case "Password:":
return []byte(a.password), nil
default:
return nil, errors.New("Unknown fromServer")
}
}
return nil, nil
}
func ConnectToSMTPServerAdvanced(config *SMTPConfig) (net.Conn, error) {
var conn net.Conn
var err error
smtpAddress := config.Server + ":" + config.Port
dialer := &net.Dialer{
Timeout: time.Duration(config.ServerTimeout) * time.Second,
}
if config.ConnectionSecurity == TLS {
tlsconfig := &tls.Config{
InsecureSkipVerify: config.SkipServerCertificateVerification,
ServerName: config.ServerName,
}
conn, err = tls.DialWithDialer(dialer, "tcp", smtpAddress, tlsconfig)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the SMTP server through TLS")
}
} else {
conn, err = dialer.Dial("tcp", smtpAddress)
if err != nil {
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
}
}
return conn, nil
}
func ConnectToSMTPServer(config *SMTPConfig) (net.Conn, error) {
return ConnectToSMTPServerAdvanced(config)
}
func NewSMTPClientAdvanced(ctx context.Context, conn net.Conn, config *SMTPConfig) (*smtp.Client, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var c *smtp.Client
ec := make(chan error)
go func() {
var err error
c, err = smtp.NewClient(conn, config.ServerName+":"+config.Port)
if err != nil {
ec <- err
return
}
cancel()
}()
select {
case <-ctx.Done():
err := ctx.Err()
if err != nil && err.Error() != "context canceled" {
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
}
case err := <-ec:
return nil, errors.Wrap(err, "unable to connect to the SMTP server")
}
if config.Hostname != "" {
err := c.Hello(config.Hostname)
if err != nil {
return nil, errors.Wrap(err, "unable to send hello message")
}
}
if config.ConnectionSecurity == StartTLS {
tlsconfig := &tls.Config{
InsecureSkipVerify: config.SkipServerCertificateVerification,
ServerName: config.ServerName,
}
c.StartTLS(tlsconfig)
}
if config.EnableSMTPAuth {
if err := c.Auth(&authChooser{config: config}); err != nil {
return nil, errors.Wrap(err, "authentication failed")
}
}
return c, nil
}
func NewSMTPClient(ctx context.Context, conn net.Conn, config *SMTPConfig) (*smtp.Client, error) {
return NewSMTPClientAdvanced(
ctx,
conn,
config,
)
}
func TestConnection(config *SMTPConfig) error {
conn, err := ConnectToSMTPServer(config)
if err != nil {
return errors.Wrap(err, "unable to connect")
}
defer conn.Close()
sec := config.ServerTimeout
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, time.Duration(sec)*time.Second)
defer cancel()
c, err := NewSMTPClient(ctx, conn, config)
if err != nil {
return errors.Wrap(err, "unable to connect")
}
c.Close()
c.Quit()
return nil
}
func SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody string, embeddedFiles map[string]io.Reader, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail string, category string) error {
fromMail := mail.Address{Name: config.FeedbackName, Address: config.FeedbackEmail}
replyTo := mail.Address{Name: config.FeedbackName, Address: config.ReplyToAddress}
mail := mailData{
mimeTo: to,
smtpTo: to,
from: fromMail,
cc: ccMail,
replyTo: replyTo,
subject: subject,
htmlBody: htmlBody,
embeddedFiles: embeddedFiles,
messageID: messageID,
inReplyTo: inReplyTo,
references: references,
category: category,
}
return sendMailUsingConfigAdvanced(mail, config)
}
func SendMailUsingConfig(to, subject, htmlBody string, config *SMTPConfig, enableComplianceFeatures bool, messageID string, inReplyTo string, references string, ccMail, category string) error {
return SendMailWithEmbeddedFilesUsingConfig(to, subject, htmlBody, nil, config, enableComplianceFeatures, messageID, inReplyTo, references, ccMail, category)
}
// allows for sending an email with differing MIME/SMTP recipients
func sendMailUsingConfigAdvanced(mail mailData, config *SMTPConfig) error {
if config.Server == "" {
return nil
}
conn, err := ConnectToSMTPServer(config)
if err != nil {
return err
}
defer conn.Close()
sec := config.ServerTimeout
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, time.Duration(sec)*time.Second)
defer cancel()
c, err := NewSMTPClient(ctx, conn, config)
if err != nil {
return err
}
defer c.Quit()
defer c.Close()
return sendMail(c, mail, time.Now(), config)
}
const SendGridXSMTPAPIHeader = "X-SMTPAPI"
func generateMessageID(hostname string) string {
return fmt.Sprintf("<%s-%d@%s>", model.NewRandomString(16), time.Now().Unix(), hostname)
}
// validateSingleAddress rejects a value that resolves to anything other than
// exactly one address. This package only ever sends to a single recipient per
// header; a multi-address value would otherwise be rejected deep inside the
// SMTP library (or, worse, silently mishandled), so we fail early with a clear
// error rather than adding multi-recipient support. The caller wraps the error
// with the relevant header name.
func validateSingleAddress(value string) error {
addresses, err := mail.ParseAddressList(value)
if err != nil {
return errors.Wrap(err, "failed to parse address")
}
if len(addresses) != 1 {
return errors.Errorf("must contain exactly one address, got %d", len(addresses))
}
return nil
}
func sendMail(c smtpClient, mail mailData, date time.Time, config *SMTPConfig) error {
mlog.Info("sending mail", mlog.String("to", mail.smtpTo))
htmlMessage := mail.htmlBody
text, err := html2text.FromString(htmlMessage)
if err != nil {
mlog.Warn("Unable to convert html body to text", mlog.Err(err))
text = ""
}
m := gomail.NewMsg()
m.SetAddrHeaderFromMailAddress(gomail.HeaderFrom, &mail.from)
if err = validateSingleAddress(mail.mimeTo); err != nil {
return errors.Wrap(err, "invalid To header")
}
if err = m.SetAddrHeader(gomail.HeaderTo, mail.mimeTo); err != nil {
return errors.Wrap(err, "failed to set To address")
}
m.SetGenHeader(gomail.HeaderSubject, mail.subject)
m.SetGenHeader(gomail.Header("Content-Transfer-Encoding"), "8bit")
m.SetGenHeader(gomail.Header("Auto-Submitted"), "auto-generated")
m.SetGenHeader(gomail.Header("Precedence"), "bulk")
if mail.category != "" {
m.SetGenHeader(gomail.Header(SendGridXSMTPAPIHeader), fmt.Sprintf(`{"category": %q}`, mail.category))
}
if mail.replyTo.Address != "" {
m.SetAddrHeaderFromMailAddress(gomail.HeaderReplyTo, &mail.replyTo)
}
if mail.cc != "" {
if err = validateSingleAddress(mail.cc); err != nil {
return errors.Wrap(err, "invalid Cc header")
}
if err = m.SetAddrHeader(gomail.HeaderCc, mail.cc); err != nil {
return errors.Wrap(err, "failed to set Cc address")
}
}
msgID := mail.messageID
if msgID == "" {
msgID = generateMessageID(config.Hostname)
}
// Must be SetGenHeader, not SetGenHeaderPreformatted: only genHeader suppresses go-mail's
// auto-generated Message-ID, otherwise we emit two and strict servers reject with a 554.
m.SetGenHeader(gomail.HeaderMessageID, msgID)
if mail.inReplyTo != "" {
m.SetGenHeaderPreformatted(gomail.HeaderInReplyTo, mail.inReplyTo)
}
if mail.references != "" {
m.SetGenHeaderPreformatted(gomail.HeaderReferences, mail.references)
}
for k, v := range mail.mimeHeaders {
m.SetGenHeader(gomail.Header(k), v)
}
m.SetDateWithValue(date)
m.SetBodyString(gomail.TypeTextPlain, text)
m.AddAlternativeString(gomail.TypeTextHTML, htmlMessage)
for name, reader := range mail.embeddedFiles {
if err = m.EmbedReader(name, reader); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to embed file %q", name))
}
}
if err = c.Mail(mail.from.Address); err != nil {
return errors.Wrap(err, "failed to set the from address")
}
if err = c.Rcpt(mail.smtpTo); err != nil {
return errors.Wrap(err, "failed to set the to address")
}
w, err := c.Data()
if err != nil {
return errors.Wrap(err, "failed to add email message data")
}
_, err = m.WriteTo(w)
if err != nil {
return errors.Wrap(err, "failed to write the email message")
}
err = w.Close()
if err != nil {
return errors.Wrap(err, "failed to close connection to the SMTP server")
}
return nil
}

View File

@@ -0,0 +1,895 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package mail
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime"
"net"
"net/mail"
"net/smtp"
"net/textproto"
"os"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const embeddedFileMinSize = 1500
func getConfig() *SMTPConfig {
server := os.Getenv("MM_EMAILSETTINGS_SMTPSERVER")
if server == "" {
server = "localhost"
}
port := os.Getenv("MM_EMAILSETTINGS_SMTPPORT")
if port == "" {
port = "10025"
}
return &SMTPConfig{
ConnectionSecurity: "",
SkipServerCertificateVerification: false,
Hostname: "localhost",
ServerName: server,
Server: server,
Port: port,
ServerTimeout: 10,
Username: "",
Password: "",
EnableSMTPAuth: false,
SendEmailNotifications: true,
FeedbackName: "",
FeedbackEmail: "test@example.com",
ReplyToAddress: "test@example.com",
}
}
func TestMailConnectionFromConfig(t *testing.T) {
cfg := getConfig()
conn, err := ConnectToSMTPServer(cfg)
require.NoError(t, err, "Should connect to the SMTP Server %v", err)
_, err = NewSMTPClient(context.Background(), conn, cfg)
require.NoError(t, err, "Should get new SMTP client")
cfg.Server = "wrongServer"
cfg.Port = "553"
_, err = ConnectToSMTPServer(cfg)
require.Error(t, err, "Should not connect to the SMTP Server")
}
func TestMailConnectionAdvanced(t *testing.T) {
cfg := getConfig()
conn, err := ConnectToSMTPServerAdvanced(cfg)
require.NoError(t, err, "Should connect to the SMTP Server")
defer conn.Close()
_, err2 := NewSMTPClientAdvanced(context.Background(), conn, cfg)
require.NoError(t, err2, "Should get new SMTP client")
l, err3 := net.Listen("tcp", "localhost:") // emulate nc -l <random-port>
require.NoError(t, err3, "Should've open a network socket and listen")
defer l.Close()
cfg = getConfig()
cfg.Server = strings.Split(l.Addr().String(), ":")[0]
cfg.Port = strings.Split(l.Addr().String(), ":")[1]
cfg.ServerTimeout = 1
conn2, err := ConnectToSMTPServerAdvanced(cfg)
require.NoError(t, err, "Should connect to the SMTP Server")
defer conn2.Close()
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
cfg = getConfig()
cfg.Server = strings.Split(l.Addr().String(), ":")[0]
cfg.Port = strings.Split(l.Addr().String(), ":")[1]
cfg.ServerTimeout = 1
_, err4 := NewSMTPClientAdvanced(
ctx,
conn2,
cfg,
)
require.Error(t, err4, "Should get a timeout get while creating a new SMTP client")
assert.Contains(t, err4.Error(), "unable to connect to the SMTP server")
cfg = getConfig()
cfg.Server = "wrongServer"
cfg.Port = "553"
cfg.ServerTimeout = 1
_, err5 := ConnectToSMTPServerAdvanced(cfg)
require.Error(t, err5, "Should not connect to the SMTP Server")
}
func TestSendMailUsingConfig(t *testing.T) {
cfg := getConfig()
emailTo := "test@example.com"
emailSubject := "Testing this email"
emailBody := "This is a test from autobot"
emailCC := "test@example.com"
// Delete all the messages before check the sample email
DeleteMailBox(emailTo)
err2 := SendMailUsingConfig(emailTo, emailSubject, emailBody, cfg, true, "", "", "", emailCC, "")
require.NoError(t, err2, "Should connect to the SMTP Server")
// Check if the email was send to the right email address
var resultsMailbox JSONMessageHeaderInbucket
err3 := RetryInbucket(5, func() error {
var err error
resultsMailbox, err = GetMailBox(emailTo)
return err
})
if err3 != nil {
t.Log(err3)
t.Log("No email was received, maybe due load on the server. Skipping this verification")
} else {
if len(resultsMailbox) > 0 {
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
require.NoError(t, err, "Could not get message from mailbox")
require.Contains(t, emailBody, resultsEmail.Body.Text, "Wrong received message %s", resultsEmail.Body.Text)
}
}
}
func TestSendMailPlainText(t *testing.T) {
cfg := getConfig()
emailSubject := "Testing this email"
tests := []struct {
name string
emailBodyHTML string
expectedBodyText string
}{
{
name: "Heading",
emailBodyHTML: "<h1>This is a test from autobot</h1><h2>This is a subheading</h2>",
expectedBodyText: "***************************\nThis is a test from autobot\n***************************\n\n--------------------\nThis is a subheading\n--------------------",
},
{
name: "List",
emailBodyHTML: "<ul><li>Item 1</li><li>Item 2</li></ul>",
expectedBodyText: "* Item 1\n* Item 2",
},
{
name: "Inline formatting",
emailBodyHTML: "<p><strong>Strong</strong> and <a href='https://example.com'>link</a>",
expectedBodyText: "*Strong* and link ( https://example.com )",
},
{
// Regression for MM-69721: a link-only body previously yielded an
// empty text/plain part (docconv readability returned "MAIN: P is nil").
name: "Link-only body",
emailBodyHTML: "<a href='https://example.com/reset'>Reset your password</a>",
expectedBodyText: "Reset your password ( https://example.com/reset )",
},
{
// Regression for MM-69721: templated transactional emails previously
// yielded an empty text/plain part (docconv readability extracted no paragraphs).
name: "Templated email",
emailBodyHTML: "<h1>Reset Your Password</h1><p>Click the link below to reset your password.</p><p><a href='https://example.com/reset?token=abc'>Reset Password</a></p>",
expectedBodyText: "Reset Password ( https://example.com/reset?token=abc )",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
emailTo := fmt.Sprintf("plain-text-%d@example.com", time.Now().UnixNano())
emailCC := emailTo
DeleteMailBox(emailTo)
err := SendMailUsingConfig(emailTo, emailSubject, test.emailBodyHTML, cfg, true, "", "", "", emailCC, "")
require.NoError(t, err, "Should connect to the SMTP Server")
var resultsMailbox JSONMessageHeaderInbucket
err = RetryInbucket(5, func() error {
var err2 error
resultsMailbox, err2 = GetMailBox(emailTo)
return err2
})
if err != nil {
t.Log("No email was received, maybe due load on the server. Failing this test")
t.Error(err)
} else {
require.NotEmpty(t, resultsMailbox, "Mailbox should contain at least one message")
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
resultsEmail, err := GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
require.NoError(t, err, "Could not get message from mailbox")
require.Contains(t, test.emailBodyHTML, resultsEmail.Body.HTML, "Wrong received message %s", resultsEmail.Body.Text)
require.NotEmpty(t, resultsEmail.Body.Text, "text/plain part should not be empty (MM-69721)")
require.Contains(t, resultsEmail.Body.Text, test.expectedBodyText, "Wrong message plain text conversion %s", resultsEmail.Body.Text)
}
})
}
}
func TestSendMailWithEmbeddedFilesUsingConfig(t *testing.T) {
cfg := getConfig()
emailTo := fmt.Sprintf("embedded-files-%d@example.com", time.Now().UnixNano())
emailSubject := "Testing this email"
emailBody := "This is a test from autobot"
emailCC := emailTo
// Delete all the messages before check the sample email
DeleteMailBox(emailTo)
embeddedFiles := map[string]io.Reader{
"test1.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
"test2.png": bytes.NewReader([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")),
}
err2 := SendMailWithEmbeddedFilesUsingConfig(emailTo, emailSubject, emailBody, embeddedFiles, cfg, true, "", "", "", emailCC, "")
require.NoError(t, err2, "Should connect to the SMTP Server")
// Check if the email was send to the right email address
var resultsMailbox JSONMessageHeaderInbucket
var resultsEmail JSONMessageInbucket
err3 := RetryInbucket(10, func() error {
var err error
resultsMailbox, err = GetMailBox(emailTo)
if err != nil {
return err
}
if len(resultsMailbox) == 0 {
return fmt.Errorf("no messages in mailbox")
}
resultsEmail, err = GetMessageFromMailbox(emailTo, resultsMailbox[0].ID)
if err != nil {
return err
}
if resultsEmail.Size <= embeddedFileMinSize {
return fmt.Errorf("message size %d does not yet reflect embedded attachments", resultsEmail.Size)
}
return nil
})
if err3 != nil {
t.Log(err3)
t.Log("No email was received, maybe due load on the server. Skipping this verification")
} else {
if len(resultsMailbox) > 0 {
require.Contains(t, resultsMailbox[0].To[0], emailTo, "Wrong To: recipient")
require.Contains(t, emailBody, resultsEmail.Body.Text, "Wrong received message %s", resultsEmail.Body.Text)
// Usign the message size because the inbucket API doesn't return embedded attachments through the API
require.Greater(t, resultsEmail.Size, embeddedFileMinSize, "the file size should be more because the embedded attachments")
}
}
}
func TestSendMailUsingConfigAdvanced(t *testing.T) {
cfg := getConfig()
// Delete all the messages before check the sample email
DeleteMailBox("test2@example.com")
// create two files with the same name that will both be attached to the email
file1, err := os.CreateTemp("", "*")
require.NoError(t, err)
defer os.Remove(file1.Name())
file1.WriteString("hello world")
file1.Close()
file2, err := os.CreateTemp("", "*")
require.NoError(t, err)
defer os.Remove(file2.Name())
file2.WriteString("foo bar")
file2.Close()
embeddedFiles := map[string]io.Reader{
"test": bytes.NewReader([]byte("test data")),
}
headers := make(map[string]string)
headers["TestHeader"] = "TestValue"
md := mailData{
mimeTo: "test@example.com",
smtpTo: "test2@example.com",
from: mail.Address{Name: "Nobody", Address: "nobody@mattermost.com"},
replyTo: mail.Address{Name: "ReplyTo", Address: "reply_to@mattermost.com"},
subject: "Testing this email",
htmlBody: "This is a test from autobot",
embeddedFiles: embeddedFiles,
mimeHeaders: headers,
}
err = sendMailUsingConfigAdvanced(md, cfg)
require.NoError(t, err, "Should connect to the SMTP Server: %v", err)
// Check if the email was send to the right email address
var resultsMailbox JSONMessageHeaderInbucket
err = RetryInbucket(5, func() error {
var mailErr error
resultsMailbox, mailErr = GetMailBox(md.smtpTo)
return mailErr
})
require.NoError(t, err, "No emails found for address %s. error: %v", md.smtpTo, err)
require.NotEqual(t, len(resultsMailbox), 0)
require.Contains(t, resultsMailbox[0].To[0], md.mimeTo, "Wrong To recipient")
resultsEmail, err := GetMessageFromMailbox(md.smtpTo, resultsMailbox[0].ID)
require.NoError(t, err)
require.Contains(t, md.htmlBody, resultsEmail.Body.Text, "Wrong received message")
// verify that the To header of the email message is set to the MIME recipient, even though we got it out of the SMTP recipient's email inbox
require.NotEmpty(t, resultsEmail.Header["To"], "missing To header")
resultEmail, err := mail.ParseAddress(resultsEmail.Header["To"][0])
require.NoError(t, err, "failed to parse To header")
assert.Equal(t, md.mimeTo, resultEmail.Address)
// verify that the MIME from address is correct - unfortunately, we can't verify the SMTP from address
assert.Equal(t, md.from.String(), resultsEmail.Header["From"][0])
// check that the custom mime headers came through - header case seems to get mutated
assert.Equal(t, "TestValue", resultsEmail.Header["Testheader"][0])
}
func TestAuthMethods(t *testing.T) {
auth := &authChooser{
config: &SMTPConfig{
Username: "test",
Password: "fakepass",
ServerName: "fakeserver",
Server: "fakeserver",
Port: "25",
},
}
tests := []struct {
desc string
server *smtp.ServerInfo
err string
}{
{
desc: "auth PLAIN success",
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: true},
},
{
desc: "auth PLAIN unencrypted connection fail",
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"PLAIN"}, TLS: false},
err: "unencrypted connection",
},
{
desc: "auth PLAIN wrong host name",
server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"PLAIN"}, TLS: true},
err: "wrong host name",
},
{
desc: "auth LOGIN success",
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: true},
},
{
desc: "auth LOGIN unencrypted connection fail",
server: &smtp.ServerInfo{Name: "wrongServer:999", Auth: []string{"LOGIN"}, TLS: true},
err: "wrong host name",
},
{
desc: "auth LOGIN wrong host name",
server: &smtp.ServerInfo{Name: "fakeserver:25", Auth: []string{"LOGIN"}, TLS: false},
err: "unencrypted connection",
},
}
for i, test := range tests {
t.Run(test.desc, func(t *testing.T) {
_, _, err := auth.Start(test.server)
got := ""
if err != nil {
got = err.Error()
}
assert.True(t, got == test.err, "%d. got error = %q; want %q", i, got, test.err)
})
}
}
func TestLoginAuthNext(t *testing.T) {
auth := &loginAuth{username: "user", password: "pass", host: "host:25"}
resp, err := auth.Next([]byte("Username:"), true)
require.NoError(t, err)
assert.Equal(t, []byte("user"), resp)
resp, err = auth.Next([]byte("Password:"), true)
require.NoError(t, err)
assert.Equal(t, []byte("pass"), resp)
_, err = auth.Next([]byte("Unknown:"), true)
require.Error(t, err)
resp, err = auth.Next(nil, false)
require.NoError(t, err)
assert.Nil(t, resp)
}
type errReader struct {
err error
payload []byte
}
func (r *errReader) Read(p []byte) (int, error) {
if len(r.payload) > 0 {
n := copy(p, r.payload)
r.payload = r.payload[n:]
return n, nil
}
return 0, r.err
}
func TestSendMailEmbedReaderError(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
embeddedFiles: map[string]io.Reader{
"attachment.txt": &errReader{payload: []byte("partial data"), err: errors.New("read failure")},
},
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.Error(t, err)
require.Contains(t, err.Error(), "failed to embed file")
}
func TestSendMailRejectsMultipleAddresses(t *testing.T) {
multiAddress := []string{
"a@example.com, b@example.com",
"a@example.com,b@example.com",
`"A" <a@example.com>, "B" <b@example.com>`,
}
t.Run("rejects multiple To addresses", func(t *testing.T) {
for _, to := range multiAddress {
mocm := &mockMailer{}
m := mailData{
mimeTo: to,
smtpTo: to,
from: mail.Address{Address: "from@example.com"},
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.Error(t, err, "expected %q to be rejected", to)
assert.Contains(t, err.Error(), "invalid To header: must contain exactly one address")
assert.Empty(t, mocm.data, "no message should be written when the To address is rejected")
}
})
t.Run("rejects multiple Cc addresses", func(t *testing.T) {
for _, cc := range multiAddress {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
cc: cc,
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.Error(t, err, "expected %q to be rejected", cc)
assert.Contains(t, err.Error(), "invalid Cc header: must contain exactly one address")
assert.Empty(t, mocm.data, "no message should be written when the Cc address is rejected")
}
})
t.Run("accepts a single To and Cc address", func(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
cc: "cc@example.com",
htmlBody: "<p>hello</p>",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
assert.NotEmpty(t, mocm.data)
})
}
type mockMailer struct {
data []byte
}
func (m *mockMailer) Mail(string) error { return nil }
func (m *mockMailer) Rcpt(string) error { return nil }
func (m *mockMailer) Data() (io.WriteCloser, error) { return m, nil }
func (m *mockMailer) Write(p []byte) (int, error) {
m.data = append(m.data, p...)
return len(p), nil
}
func (m *mockMailer) Close() error { return nil }
func TestSendMail(t *testing.T) {
dir, err := os.MkdirTemp(".", "mail-test-")
require.NoError(t, err)
defer os.RemoveAll(dir)
mocm := &mockMailer{}
testCases := map[string]struct {
replyTo mail.Address
messageID string
inReplyTo string
references string
contains string
notContains string
}{
"adds reply-to header": {
mail.Address{Address: "foo@test.com"},
"",
"",
"",
"\r\nReply-To: <foo@test.com>\r\n",
"",
},
"doesn't add reply-to header": {
mail.Address{},
"",
"",
"",
"",
"\r\nReply-To:",
},
"adds message-id header": {
mail.Address{},
"<abc123@mattermost.com>",
"",
"",
"\r\nMessage-ID: <abc123@mattermost.com>\r\n",
"",
},
"always adds message-id header": {
mail.Address{},
"",
"",
"",
"\r\nMessage-ID: <",
"",
},
"adds in-reply-to header": {
mail.Address{},
"",
"<defg456@mattermost.com>",
"",
"\r\nIn-Reply-To: <defg456@mattermost.com>\r\n",
"",
},
"doesn't add in-reply-to header": {
mail.Address{},
"",
"",
"",
"",
"\r\nIn-Reply-To:",
},
"adds references header": {
mail.Address{},
"",
"",
"<ghi789@mattermost.com>",
"\r\nReferences: <ghi789@mattermost.com>\r\n",
"",
},
"doesn't add references header": {
mail.Address{},
"",
"",
"",
"",
"\r\nReferences:",
},
}
t.Run("sets exactly one message-id header when provided", func(t *testing.T) {
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
messageID: "<abc123@mattermost.com>",
}
err = sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(string(mocm.data), "Message-ID:"), "expected exactly one Message-ID header, got:\n%s", string(mocm.data))
assert.Contains(t, string(mocm.data), "\r\nMessage-ID: <abc123@mattermost.com>\r\n")
mocm.data = []byte{}
})
t.Run("sets exactly one message-id header when generated", func(t *testing.T) {
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
}
err = sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(string(mocm.data), "Message-ID:"), "expected exactly one Message-ID header, got:\n%s", string(mocm.data))
mocm.data = []byte{}
})
t.Run("adds cc header", func(t *testing.T) {
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
cc: "cc@example.com",
}
err = sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
require.Contains(t, string(mocm.data), "\r\nCc: <cc@example.com>\r\n")
mocm.data = []byte{}
})
t.Run("adds sendgrid category header", func(t *testing.T) {
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
category: "transactional",
}
err = sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
require.Contains(t, string(mocm.data), SendGridXSMTPAPIHeader)
require.Contains(t, string(mocm.data), `"transactional"`)
mocm.data = []byte{}
})
for testName, tc := range testCases {
t.Run(testName, func(t *testing.T) {
md := mailData{"test@example.com", "test@example.com", mail.Address{Address: "from@example.com"}, "", tc.replyTo, "", "", nil, nil, tc.messageID, tc.inReplyTo, tc.references, ""}
cfg := getConfig()
err = sendMail(mocm, md, time.Now(), cfg)
require.NoError(t, err)
if tc.contains != "" {
require.Contains(t, string(mocm.data), tc.contains)
}
if tc.notContains != "" {
require.NotContains(t, string(mocm.data), tc.notContains)
}
mocm.data = []byte{}
})
}
}
// mailHeaderValue returns the value of the named header (case-insensitive) from
// the raw RFC 5322 message, unfolding any continuation lines. It fails the test
// if the message is not parseable, and returns "" and false if the header is
// absent.
func mailHeaderValue(t *testing.T, raw, name string) (string, bool) {
t.Helper()
msg, err := mail.ReadMessage(strings.NewReader(raw))
require.NoError(t, err, "message must be RFC 5322 parseable")
if _, ok := msg.Header[textproto.CanonicalMIMEHeaderKey(name)]; !ok {
return "", false
}
return msg.Header.Get(name), true
}
// TestSendMailSubjectEncoding verifies that a non-ASCII subject is emitted as
// an RFC 2047 encoded-word, never as raw UTF-8 bytes in the header. A library
// swap that stopped encoding the subject would corrupt it on the wire.
func TestSendMailSubjectEncoding(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
subject: "Café ☕ 你好",
htmlBody: "hi",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
subject, ok := mailHeaderValue(t, string(mocm.data), "Subject")
require.True(t, ok, "Subject header must be present")
// The value must be an RFC 2047 encoded-word, not the raw UTF-8 string.
assert.Contains(t, strings.ToLower(subject), "=?utf-8?", "Subject must be RFC 2047 encoded")
assert.NotContains(t, subject, "Café", "raw UTF-8 subject must not leak into the header")
assert.NotContains(t, subject, "你好", "raw UTF-8 subject must not leak into the header")
// Decoding the encoded-word(s) must round-trip back to the original subject,
// without locking in the exact encoding go-mail happens to emit.
decoded, err := new(mime.WordDecoder).DecodeHeader(subject)
require.NoError(t, err)
assert.Equal(t, m.subject, decoded)
}
// TestSendMailMIMEStructure verifies the multipart container layout: with text
// and HTML bodies a multipart/alternative is produced, and with an embedded
// file the whole thing is wrapped in a multipart/related whose embedded part
// carries a Content-ID and inline disposition.
func TestSendMailMIMEStructure(t *testing.T) {
t.Run("text and html without embed produce multipart/alternative", func(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
htmlBody: "<p>Hello <b>world</b></p>",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
data := string(mocm.data)
ct, ok := mailHeaderValue(t, data, "Content-Type")
require.True(t, ok, "Content-Type header must be present")
assert.Contains(t, ct, "multipart/alternative", "top-level container should be multipart/alternative")
assert.NotContains(t, ct, "multipart/related", "no embedded files means no multipart/related wrapper")
assert.Contains(t, data, "text/plain", "plain text part must be present")
assert.Contains(t, data, "text/html", "html part must be present")
})
t.Run("embedded file produces multipart/related with inline Content-ID part", func(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
htmlBody: "<p>Hello <b>world</b></p>",
embeddedFiles: map[string]io.Reader{
"logo.png": bytes.NewReader([]byte("PNGDATA")),
},
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
data := string(mocm.data)
ct, ok := mailHeaderValue(t, data, "Content-Type")
require.True(t, ok, "Content-Type header must be present")
assert.Contains(t, ct, "multipart/related", "an embedded file must be wrapped in multipart/related")
// Both alternatives still exist inside the related container.
assert.Contains(t, data, "multipart/alternative", "multipart/alternative must remain nested inside multipart/related")
assert.Contains(t, data, "text/plain")
assert.Contains(t, data, "text/html")
lower := strings.ToLower(data)
assert.Contains(t, lower, "content-id: <logo.png>", "embedded part must carry a Content-ID")
assert.Contains(t, lower, "content-disposition: inline", "embedded part must be inline")
assert.Contains(t, data, `filename="logo.png"`, "embedded part must reference its filename")
})
}
// TestSendMailDateHeader verifies a Date header is present and RFC 5322 shaped
// (parseable by net/mail). SetDateWithValue regressions would break this.
func TestSendMailDateHeader(t *testing.T) {
mocm := &mockMailer{}
date := time.Date(2026, 7, 6, 12, 0, 0, 0, time.UTC)
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
htmlBody: "hi",
}
err := sendMail(mocm, m, date, getConfig())
require.NoError(t, err)
value, ok := mailHeaderValue(t, string(mocm.data), "Date")
require.True(t, ok, "Date header must be present")
parsed, err := mail.ParseDate(value)
require.NoError(t, err, "Date header %q must be RFC 5322 parseable", value)
assert.True(t, parsed.Equal(date), "parsed Date %s should match the supplied time %s", parsed, date)
}
// TestSendMailComplianceHeaders verifies the bulk/auto-generated headers used to
// signal automated mail are present.
func TestSendMailComplianceHeaders(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
htmlBody: "hi",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
autoSubmitted, ok := mailHeaderValue(t, string(mocm.data), "Auto-Submitted")
require.True(t, ok, "Auto-Submitted header must be present")
assert.Equal(t, "auto-generated", autoSubmitted)
precedence, ok := mailHeaderValue(t, string(mocm.data), "Precedence")
require.True(t, ok, "Precedence header must be present")
assert.Equal(t, "bulk", precedence)
}
// TestSendMailBodyCharset verifies the body parts declare a UTF-8 charset,
// tolerant of casing and quoting differences between libraries.
func TestSendMailBodyCharset(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
htmlBody: "<p>hi</p>",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
lower := strings.ToLower(string(mocm.data))
// Be tolerant of charset=UTF-8 vs charset="UTF-8".
hasCharset := strings.Contains(lower, "charset=utf-8") || strings.Contains(lower, `charset="utf-8"`)
assert.True(t, hasCharset, "body parts must declare a UTF-8 charset")
}
// TestSendMailFromDisplayName verifies that a From display name renders and that
// a non-ASCII display name is RFC 2047 encoded rather than emitted raw.
func TestSendMailFromDisplayName(t *testing.T) {
t.Run("ASCII display name renders literally", func(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Name: "Plain Name", Address: "from@example.com"},
htmlBody: "hi",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
from, ok := mailHeaderValue(t, string(mocm.data), "From")
require.True(t, ok, "From header must be present")
assert.Contains(t, from, "Plain Name")
assert.Contains(t, from, "<from@example.com>")
assert.NotContains(t, from, "=?", "an ASCII display name must not be RFC 2047 encoded")
})
t.Run("non-ASCII display name is RFC 2047 encoded", func(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Name: "Café Admin ☕", Address: "from@example.com"},
htmlBody: "hi",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
from, ok := mailHeaderValue(t, string(mocm.data), "From")
require.True(t, ok, "From header must be present")
assert.Contains(t, strings.ToLower(from), "=?utf-8?", "non-ASCII display name must be RFC 2047 encoded")
assert.NotContains(t, from, "Café", "raw UTF-8 display name must not leak into the header")
assert.Contains(t, from, "<from@example.com>")
})
}
// TestSendMailCcDisplayName verifies a Cc address with a display name renders
// correctly through the address parser.
func TestSendMailCcDisplayName(t *testing.T) {
mocm := &mockMailer{}
m := mailData{
mimeTo: "test@example.com",
smtpTo: "test@example.com",
from: mail.Address{Address: "from@example.com"},
cc: `"Cc Person" <cc@example.com>`,
htmlBody: "hi",
}
err := sendMail(mocm, m, time.Now(), getConfig())
require.NoError(t, err)
cc, ok := mailHeaderValue(t, string(mocm.data), "Cc")
require.True(t, ok, "Cc header must be present")
assert.Contains(t, cc, "Cc Person")
assert.Contains(t, cc, "<cc@example.com>")
}

View File

@@ -0,0 +1,166 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package mfa
import (
"crypto/rand"
"encoding/base32"
"fmt"
"net/url"
"strings"
"github.com/dgryski/dgoogauth"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/rsc/qr"
"github.com/pkg/errors"
)
// InvalidToken indicates the case where the token validation has failed.
var InvalidToken = errors.New("invalid mfa token")
const (
// This will result in 160 bits of entropy (base32 encoded), as recommended by rfc4226.
mfaSecretSize = 20
)
type Store interface {
UpdateMfaActive(userId string, active bool) error
UpdateMfaSecret(userId, secret string) error
StoreMfaUsedTimestamps(userId string, ts []int) error
GetMfaUsedTimestamps(userId string) ([]int, error)
}
type MFA struct {
store Store
}
func New(store Store) *MFA {
return &MFA{store}
}
// newRandomBase32String returns a base32 encoded string of a random slice
// of bytes of the given size. The resulting entropy will be (8 * size) bits.
func newRandomBase32String(size int) string {
data := make([]byte, size)
rand.Read(data)
return base32.StdEncoding.EncodeToString(data)
}
func getIssuerFromURL(uri string) string {
issuer := "Mattermost"
siteURL := strings.TrimSpace(uri)
if siteURL != "" {
siteURL = strings.TrimPrefix(siteURL, "https://")
siteURL = strings.TrimPrefix(siteURL, "http://")
issuer = strings.TrimPrefix(siteURL, "www.")
}
return url.QueryEscape(issuer)
}
// GenerateSecret generates a new user mfa secret and store it with the StoreSecret function provided
func (m *MFA) GenerateSecret(siteURL, userEmail, userID string) (string, []byte, error) {
issuer := getIssuerFromURL(siteURL)
secret := newRandomBase32String(mfaSecretSize)
authLink := fmt.Sprintf("otpauth://totp/%s:%s?secret=%s&issuer=%s", issuer, userEmail, secret, issuer)
code, err := qr.Encode(authLink, qr.H)
if err != nil {
return "", nil, errors.Wrap(err, "unable to generate qr code")
}
img := code.PNG()
if err := m.store.UpdateMfaSecret(userID, secret); err != nil {
return "", nil, errors.Wrap(err, "unable to store mfa secret")
}
return secret, img, nil
}
// Activate set the mfa as active and store it with the StoreActive function provided
func (m *MFA) Activate(userMfaSecret, userID string, token string) error {
usedTs, err := m.store.GetMfaUsedTimestamps(userID)
if err != nil {
return errors.Wrap(err, "unable to retrieve the DisallowReuse slice")
}
otpConfig, err := m.authenticate(userMfaSecret, usedTs, token)
if err != nil {
return errors.Wrap(err, "unable to authenticate the token")
}
if err = m.store.UpdateMfaActive(userID, true); err != nil {
return errors.Wrap(err, "unable to store mfa active")
}
err = m.store.StoreMfaUsedTimestamps(userID, otpConfig.DisallowReuse)
if err != nil {
return errors.Wrap(err, "unable to store the DisallowReuse slice")
}
return nil
}
// Deactivate set the mfa as deactivated, remove the mfa secret, store it with the StoreActive and StoreSecret functions provided
func (m *MFA) Deactivate(userId string) error {
if err := m.store.UpdateMfaActive(userId, false); err != nil {
return errors.Wrap(err, "unable to store mfa active")
}
if err := m.store.UpdateMfaSecret(userId, ""); err != nil {
return errors.Wrap(err, "unable to store mfa secret")
}
return nil
}
// Validate the provide token using the secret provided
func (m *MFA) ValidateToken(user *model.User, token string) (bool, error) {
usedTs, err := m.store.GetMfaUsedTimestamps(user.Id)
if err != nil {
return false, errors.Wrap(err, "unable to retrieve the DisallowReuse slice")
}
otpConfig, err := m.authenticate(user.MfaSecret, usedTs, token)
if err != nil {
if err == InvalidToken {
return false, nil
}
return false, errors.Wrap(err, "unable to parse the token")
}
err = m.store.StoreMfaUsedTimestamps(user.Id, otpConfig.DisallowReuse)
if err != nil {
return true, errors.Wrap(err, "unable to store the DisallowReuse slice")
}
return true, nil
}
func (*MFA) authenticate(userMfaSecret string, usedTs []int, token string) (*dgoogauth.OTPConfig, error) {
trimmedToken := strings.TrimSpace(token)
otpConfig := &dgoogauth.OTPConfig{
Secret: userMfaSecret,
WindowSize: 3,
HotpCounter: 0,
DisallowReuse: usedTs,
}
ok, err := otpConfig.Authenticate(trimmedToken)
if err != nil {
return nil, errors.Wrap(err, "unable to parse the token")
}
if !ok {
return nil, InvalidToken
}
return otpConfig, nil
}

View File

@@ -0,0 +1,237 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package mfa
import (
"encoding/base32"
"errors"
"fmt"
"net/url"
"testing"
"time"
"github.com/dgryski/dgoogauth"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/plugin/plugintest/mock"
"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
)
func TestGenerateSecret(t *testing.T) {
userID := "user-id"
userEmail := "sample@sample.com"
siteURL := "http://localhost:8065"
t.Run("fail on store action fail", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("UpdateMfaSecret", userID, mock.AnythingOfType("string")).Return(func(userId string, secret string) error {
return errors.New("failed to update mfa secret")
})
_, _, err := New(&storeMock).GenerateSecret(siteURL, userEmail, userID)
require.Error(t, err)
require.Contains(t, err.Error(), "unable to store mfa secret")
})
t.Run("Successful generate secret", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("UpdateMfaSecret", userID, mock.AnythingOfType("string")).Return(func(userId string, secret string) error {
return nil
})
secret, img, err := New(&storeMock).GenerateSecret(siteURL, userEmail, userID)
require.NoError(t, err)
assert.Len(t, secret, 32)
require.NotEmpty(t, img, "no image set")
})
}
func TestGetIssuerFromURL(t *testing.T) {
cases := []struct {
Input string
Expected string
}{
{"http://somewebsite.com", url.QueryEscape("somewebsite.com")},
{"https://somewebsite.com", url.QueryEscape("somewebsite.com")},
{"https://some.website.com", url.QueryEscape("some.website.com")},
{" https://www.somewebsite.com", url.QueryEscape("somewebsite.com")},
{"http://somewebsite.com/chat", url.QueryEscape("somewebsite.com/chat")},
{"somewebsite.com ", url.QueryEscape("somewebsite.com")},
{"http://localhost:8065", url.QueryEscape("localhost:8065")},
{"", "Mattermost"},
{" ", "Mattermost"},
}
for _, c := range cases {
assert.Equal(t, c.Expected, getIssuerFromURL(c.Input))
}
}
func TestActivate(t *testing.T) {
userID := "user-id"
userMfaSecret := newRandomBase32String(mfaSecretSize)
token := dgoogauth.ComputeCode(userMfaSecret, time.Now().UTC().Unix()/30)
t.Run("fail on wrongly formatted token", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
err := New(&storeMock).Activate(userMfaSecret, userID, "invalid-token")
require.Error(t, err)
require.Contains(t, err.Error(), "unable to parse the token")
})
t.Run("fail on invalid token", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
err := New(&storeMock).Activate(userMfaSecret, userID, "000000")
require.Error(t, err)
require.Contains(t, err.Error(), "invalid mfa token")
})
t.Run("fail on store action fail", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
storeMock.On("UpdateMfaActive", userID, true).Return(func(userId string, active bool) error {
return errors.New("failed to update mfa active")
})
err := New(&storeMock).Activate(userMfaSecret, userID, fmt.Sprintf("%06d", token))
require.Error(t, err)
require.Contains(t, err.Error(), "unable to store mfa active")
})
t.Run("Successful activate", func(t *testing.T) {
userID := model.NewId()
secret := newRandomBase32String(mfaSecretSize)
t0 := time.Now().UTC().Unix() / 30
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
usMock := mocks.UserStore{}
usMock.On("GetMfaUsedTimestamps", userID).Return([]int{}, nil).Once()
usMock.On("UpdateMfaActive", userID, true).Return(nil).Once()
usMock.On("StoreMfaUsedTimestamps", userID, mock.AnythingOfType("[]int")).Return(nil).Once()
err := New(&usMock).Activate(secret, userID, code)
require.NoError(t, err)
})
t.Run("disallow reuse of totp", func(t *testing.T) {
userID := model.NewId()
secret := newRandomBase32String(mfaSecretSize)
t0 := time.Now().UTC().Unix() / 30
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
usMock := mocks.UserStore{}
usMock.On("GetMfaUsedTimestamps", userID).Return([]int{int(t0)}, nil).Once()
err := New(&usMock).Activate(secret, userID, code)
require.Error(t, err)
})
}
func TestDeactivate(t *testing.T) {
userID := "user-id"
t.Run("fail on store UpdateMfaActive action fail", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
return errors.New("failed to update mfa active")
})
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
return errors.New("failed to update mfa secret")
})
err := New(&storeMock).Deactivate(userID)
require.Error(t, err)
require.Contains(t, err.Error(), "unable to store mfa active")
})
t.Run("fail on store UpdateMfaSecret action fail", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
return nil
})
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
return errors.New("failed to update mfa secret")
})
err := New(&storeMock).Deactivate(userID)
require.Error(t, err)
require.Contains(t, err.Error(), "unable to store mfa secret")
})
t.Run("Successful deactivate", func(t *testing.T) {
storeMock := mocks.UserStore{}
storeMock.On("UpdateMfaActive", userID, false).Return(func(userId string, active bool) error {
return nil
})
storeMock.On("UpdateMfaSecret", userID, "").Return(func(userId string, secret string) error {
return nil
})
err := New(&storeMock).Deactivate(userID)
require.NoError(t, err)
})
}
func TestValidateToken(t *testing.T) {
t.Run("fail on wrongly formatted token", func(t *testing.T) {
id := model.NewId()
secret := newRandomBase32String(mfaSecretSize)
u := &model.User{Id: id, MfaSecret: secret}
usMock := mocks.UserStore{}
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{}, nil).Once()
ok, err := New(&usMock).ValidateToken(u, "invalid-token")
require.Error(t, err)
require.False(t, ok)
require.Contains(t, err.Error(), "unable to parse the token")
})
t.Run("successful validation", func(t *testing.T) {
id := model.NewId()
secret := newRandomBase32String(mfaSecretSize)
u := &model.User{Id: id, MfaSecret: secret}
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, time.Now().UTC().Unix()/30))
usMock := mocks.UserStore{}
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{}, nil).Once()
usMock.On("StoreMfaUsedTimestamps", u.Id, mock.AnythingOfType("[]int")).Return(nil).Once()
ok, err := New(&usMock).ValidateToken(u, code)
require.NoError(t, err)
require.True(t, ok)
})
t.Run("disallow reuse of totp", func(t *testing.T) {
id := model.NewId()
secret := newRandomBase32String(mfaSecretSize)
u := &model.User{Id: id, MfaSecret: secret}
t0 := time.Now().UTC().Unix() / 30
code := fmt.Sprintf("%06d", dgoogauth.ComputeCode(secret, t0))
usMock := mocks.UserStore{}
usMock.On("GetMfaUsedTimestamps", u.Id).Return([]int{int(t0)}, nil).Once()
ok, err := New(&usMock).ValidateToken(u, code)
require.False(t, ok)
require.NoError(t, err)
})
}
func TestRandomBase32String(t *testing.T) {
for i := range 1000 {
str := newRandomBase32String(i)
require.Len(t, str, base32.StdEncoding.EncodedLen(i))
}
}

View File

@@ -0,0 +1,74 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package templates
import (
"bytes"
"html/template"
"io"
"path/filepath"
"sync"
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
)
// Container represents a set of templates that can be render
type Container struct {
templates *template.Template
mutex sync.RWMutex
}
// Data contains the data used to populate the template variables, it has Props
// that can be of any type and HTML that only can be `template.HTML` types.
type Data struct {
Props map[string]any
HTML map[string]template.HTML
}
func GetTemplateDirectory() (string, bool) {
return fileutils.FindDir("templates")
}
// NewFromTemplates creates a new templates container using a
// `template.Template` object
func NewFromTemplate(templates *template.Template) *Container {
return &Container{templates: templates}
}
// New creates a new templates container scanning a directory.
func New(directory string) (*Container, error) {
c := &Container{}
htmlTemplates, err := template.ParseGlob(filepath.Join(directory, "*.html"))
if err != nil {
return nil, err
}
c.templates = htmlTemplates
return c, nil
}
// RenderToString renders the template referenced with the template name using
// the data provided and return a string with the result
func (c *Container) RenderToString(templateName string, data Data) (string, error) {
var text bytes.Buffer
if err := c.Render(&text, templateName, data); err != nil {
return "", err
}
return text.String(), nil
}
// RenderToString renders the template referenced with the template name using
// the data provided and write it to the writer provided
func (c *Container) Render(w io.Writer, templateName string, data Data) error {
c.mutex.RLock()
htmlTemplates := c.templates
c.mutex.RUnlock()
if err := htmlTemplates.ExecuteTemplate(w, templateName, data); err != nil {
return err
}
return nil
}

View File

@@ -0,0 +1,67 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package templates
import (
"bytes"
"html/template"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNew_BadDirectory(t *testing.T) {
container, err := New("notarealdirectory")
assert.Nil(t, container)
assert.Error(t, err)
}
func TestRender(t *testing.T) {
tpl := template.New("test")
_, err := tpl.Parse(`{{ define "foo" }}foo{{ .Props.Bar }}{{ end }}`)
require.NoError(t, err)
mt := NewFromTemplate(tpl)
data := Data{
Props: map[string]any{
"Bar": "bar",
},
}
text, err := mt.RenderToString("foo", data)
require.NoError(t, err)
assert.Equal(t, "foobar", text)
buf := &bytes.Buffer{}
require.NoError(t, mt.Render(buf, "foo", data))
assert.Equal(t, "foobar", buf.String())
}
func TestRenderError(t *testing.T) {
tpl := template.New("test")
_, err := tpl.Parse(`{{ define "foo" }}foo{{ .Foo.Bar }}bar{{ end }}`)
require.NoError(t, err)
mt := NewFromTemplate(tpl)
text, err := mt.RenderToString("foo", Data{})
require.Error(t, err)
assert.Equal(t, "", text)
buf := &bytes.Buffer{}
assert.Error(t, mt.Render(buf, "foo", Data{}))
assert.Equal(t, "foo", buf.String())
}
func TestRenderUnknownTemplate(t *testing.T) {
tpl := template.New("")
mt := NewFromTemplate(tpl)
text, err := mt.RenderToString("foo", Data{})
require.Error(t, err)
assert.Equal(t, "", text)
buf := &bytes.Buffer{}
assert.Error(t, mt.Render(buf, "foo", Data{}))
assert.Equal(t, "", buf.String())
}

View File

@@ -0,0 +1,114 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package web
import (
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
)
var UnsafeContentTypes = [...]string{
"application/javascript",
"application/ecmascript",
"text/javascript",
"text/ecmascript",
"application/x-javascript",
"text/html",
}
var MediaContentTypes = [...]string{
"image/jpeg",
"image/png",
"image/bmp",
"image/gif",
"image/tiff",
"image/webp",
"video/avi",
"video/mpeg",
"video/mp4",
"audio/mpeg",
"audio/wav",
}
// WriteFileResponse copies the io.ReadSeeker `fileReader` to the ResponseWriter `w`. Use this when you have a
// ReadSeeker.
func WriteFileResponse(filename string, contentType string, contentSize int64, lastModification time.Time, webserverMode string, fileReader io.ReadSeeker, forceDownload bool, w http.ResponseWriter, r *http.Request) {
setHeaders(w, contentType, forceDownload, filename)
if contentSize > 0 {
contentSizeStr := strconv.Itoa(int(contentSize))
if webserverMode == "gzip" {
w.Header().Set("X-Uncompressed-Content-Length", contentSizeStr)
} else {
w.Header().Set("Content-Length", contentSizeStr)
}
}
http.ServeContent(w, r, filename, lastModification, fileReader)
}
// WriteStreamResponse copies the Reader `r` to the ResponseWriter `w`. Use this when you need to stream a response
// to the client that will appear as a file `filename` of type `contentType`.
func WriteStreamResponse(w http.ResponseWriter, r io.Reader, filename string, contentType string, forceDownload bool) error {
setHeaders(w, contentType, forceDownload, filename)
if _, err := io.Copy(w, r); err != nil {
return errors.Wrap(err, "error streaming Reader")
}
return nil
}
func setHeaders(w http.ResponseWriter, contentType string, forceDownload bool, filename string) {
// Only set Cache-Control if it hasn't been set already
if w.Header().Get("Cache-Control") == "" {
w.Header().Set("Cache-Control", "private, max-age=86400")
}
w.Header().Set("X-Content-Type-Options", "nosniff")
if contentType == "" {
contentType = "application/octet-stream"
} else {
for _, unsafeContentType := range UnsafeContentTypes {
if strings.HasPrefix(contentType, unsafeContentType) {
contentType = "text/plain"
break
}
}
}
w.Header().Set("Content-Type", contentType)
var toDownload bool
if forceDownload {
toDownload = true
} else {
isMediaType := false
for _, mediaContentType := range MediaContentTypes {
if strings.HasPrefix(contentType, mediaContentType) {
isMediaType = true
break
}
}
toDownload = !isMediaType
}
filename = url.PathEscape(filename)
if toDownload {
w.Header().Set("Content-Disposition", "attachment;filename=\""+filename+"\"; filename*=UTF-8''"+filename)
} else {
w.Header().Set("Content-Disposition", "inline;filename=\""+filename+"\"; filename*=UTF-8''"+filename)
}
// prevent file links from being embedded in iframes
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Content-Security-Policy", "Frame-ancestors 'none'")
}